text:
from ale_python_interface import ALEInterface
from gym import spaces
from gym import utils
from gym.envs.atari import AtariEnv
from gym.utils import seeding
import numpy as np
import os

def to_ram(ale):
    ram_size = ale.getRAMSize()
    ram = np.zeros((ram_size),dtype=np.uint8)
    ale.getRAM(ram)
    return ram

class UpdatedAtariEnv(AtariEnv):
    def __init__(self, rom_path, obs_type, frameskip=(2,5), repeat_action_probability=0., mode=0, difficulty=0):
        """Frameskip should be either a tuple (indicating a random range to
        choose from, with the top value exclude), or an int."""

        utils.EzPickle.__init__(self, rom_path, obs_type)
        assert obs_type in ('ram', 'image')

        self.rom_path = rom_path
        if not os.path.exists(self.rom_path):
            raise IOError('You asked for ROM %s but path %s does not exist'%(game, self.game_path))
        self._obs_type = obs_type
        self.frameskip = frameskip
        # Load new ALE interface, instead of atari-py
        self.ale = ALEInterface()
        self.viewer = None

        # Tune (or disable) ALE's action repeat:
        # https://github.com/openai/gym/issues/349
        assert isinstance(repeat_action_probability, (float, int)), "Invalid repeat_action_probability: {!r}".format(repeat_action_probability)
        self.ale.setFloat('repeat_action_probability'.encode('utf-8'), repeat_action_probability)

        self.seed()

        # Set mode and difficulty
        self.ale.setMode(mode)
        self.ale.setDifficulty(difficulty)

        self._action_set = self.ale.getMinimalActionSet()
        self.action_space = spaces.Discrete(len(self._action_set))

        (screen_width,screen_height) = self.ale.getScreenDims()
        if self._obs_type == 'ram':
            self.observation_space = spaces.Box(low=0, high=255, shape=(128,))
        elif self._obs_type == 'image':
            self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 3))
        else:
            raise error.Error('Unrecognized observation type: {}'.format(self._obs_type))

    def seed(self, seed=None):
        self.np_random, seed1 = seeding.np_random(seed)
        # Derive a random seed. This gets passed as a uint, but gets
        # checked as an int elsewhere, so we need to keep it below
        # 2**31.
        seed2 = seeding.hash_seed(seed1 + 1) % 2**31
        # Empirically, we need to seed before loading the ROM.
        self.ale.setInt(b'random_seed', seed2)
        # Load game from ROM instead of game path
        self.ale.loadROM(str.encode(self.rom_path))
        return [seed1, seed2]

    def _get_image(self):
        return self.ale.getScreenRGB()

ACTION_MEANING = {
    0 : "NOOP",
    1 : "FIRE",
    2 : "UP",
    3 : "RIGHT",
    4 : "LEFT",
    5 : "DOWN",
    6 : "UPRIGHT",
    7 : "UPLEFT",
    8 : "DOWNRIGHT",
    9 : "DOWNLEFT",
    10 : "UPFIRE",
    11 : "RIGHTFIRE",
    12 : "LEFTFIRE",
    13 : "DOWNFIRE",
    14 : "UPRIGHTFIRE",
    15 : "UPLEFTFIRE",
    16 : "DOWNRIGHTFIRE",
    17 : "DOWNLEFTFIRE",
}

max_stars_repo_name: alexcoda/DeepRL

max_issues_repo_name: alexcoda/DeepRL

max_forks_repo_name: alexcoda/DeepRL


text:
from __future__ import print_function
from random import randrange
import random
from six.moves import cPickle as pickle
import numpy as np
import os
from scipy.misc import imread
import platform

def grad_check(f, x, epsilon=1e-7):
    """
    Implements the numerical gradient for a function with a vector input.
    -------
    Args: f; a function that takes a vector argument
          x; numpy array of shape inpt_size x 1
          epsilon; float; tiny shift to the input to compute approximated gradient
    -------
    Returns: numpy array; the approximated numerical gradient
    """
    # Set-up variables
    xshape = x.shape
    input_size = x.size
    grad = np.zeros((input_size,))
    x = x.ravel()

    # Compute grad
    for i in range(input_size):
        # Compute f_plus[i]
        oldval = x[i]
        x[i] = oldval + epsilon
        f_plus =  f(x.reshape(xshape))

        # Compute f_minus[i]
        x[i] = oldval - epsilon
        f_minus = f(x.reshape(xshape))

        # Restore
        x[i] = oldval

        # Compute gradapprox[i]
        grad[i] = (f_plus - f_minus) / (2 * epsilon)
    return grad.reshape(xshape)

def grad_check_sparse(f, x, analytic_grad, num_checks=10, seed=42, epsilon=1e-5):
    """
    Sample a few random elements and only return the relative distance
    between the numerical and analyitical gradient.
    -------
    Args: f; a function that takes a vector argument
          x; numpy array the parameters we are differentiating w.r.t of the model
          analytic_grad; numpy array; analytically evaluated grad
          num_checks; int; number of coordinates to evaluate
          epsilon; float; tiny shift to the input to compute approximated gradient
          seed; int; random seed value
    -------
    Returns: numpy array; the relative errors between the numerically calculate gradient
                and the analytic one
    """
    random.seed(seed)
    rel_errors = np.empty(num_checks)
    for i in range(num_checks):
        ix = tuple([randrange(m) for m in x.shape])

        oldval = x[ix]
        x[ix] = oldval + epsilon        # increment by epsilon
        f_pos = f(x)                    # evaluate f(x + epsilon)
        x[ix] = oldval - epsilon        # increment by epsilon
        f_minus = f(x)                  # evaluate f(x - epsilon)
        x[ix] = oldval                  # reset

        grad_numerical = (f_pos - f_minus) / (2 * epsilon)
        grad_analytic = analytic_grad[ix]
        rel_error = abs(grad_numerical - grad_analytic) / (abs(grad_numerical) + abs(grad_analytic))
        rel_errors[i] = rel_error
    return rel_errors

def load_pickle(f):
    version = platform.python_version_tuple()
    if version[0] == '2':
        return  pickle.load(f)
    elif version[0] == '3':
        return  pickle.load(f, encoding='latin1')
    raise ValueError("invalid python version: {}".format(version))

def load_CIFAR_batch(filename):
  """ load single batch of cifar """
  with open(filename, 'rb') as f:
    datadict = load_pickle(f)
    X = datadict['data']
    Y = datadict['labels']
    X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
    Y = np.array(Y)
    return X, Y

def load_CIFAR10(ROOT):
  """ load all of cifar """
  xs = []
  ys = []
  for b in range(1,6):
    f = os.path.join(ROOT, 'data_batch_%d' % (b, ))
    X, Y = load_CIFAR_batch(f)
    xs.append(X)
    ys.append(Y)
  Xtr = np.concatenate(xs)
  Ytr = np.concatenate(ys)
  del X, Y
  Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch'))
  return Xtr, Ytr, Xte, Yte

def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000, num_dev=500):
    """
    Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
    it for the linear classifier. These are the same steps as we used for the
    SVM, but condensed to a single function.
    """
    # Load the raw CIFAR-10 data
    cifar10_dir = '../datasets/cifar-10-batches-py'
    X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)

    # subsample the data
    # Validation set
    mask = list(range(num_training, num_training + num_validation))
    X_val = X_train[mask]
    y_val = y_train[mask]
    # Training set
    mask = list(range(num_training))
    X_train = X_train[mask]
    y_train = y_train[mask]
    # Test set
    mask = list(range(num_test))
    X_test = X_test[mask]
    y_test = y_test[mask]
    # Dev data set: just for debugging purposes, it overlaps with the training set,
    # but has a smaller size.
    mask = np.random.choice(num_training, num_dev, replace=False)
    X_dev = X_train[mask]
    y_dev = y_train[mask]

    # Preprocessing: reshape the image data into rows
    X_train = np.reshape(X_train, (X_train.shape[0], -1))
    X_val = np.reshape(X_val, (X_val.shape[0], -1))
    X_test = np.reshape(X_test, (X_test.shape[0], -1))
    X_dev = np.reshape(X_dev, (X_dev.shape[0], -1))

    # Normalize the data: subtract the mean image
    mean_image = np.mean(X_train, axis = 0)
    X_train -= mean_image
    X_val -= mean_image
    X_test -= mean_image
    X_dev -= mean_image

    return X_train, y_train, X_val, y_val, X_test, y_test, X_dev, y_dev

def load_models(models_dir):
  """
  Load saved models from disk. This will attempt to unpickle all files in a
  directory; any files that give errors on unpickling (such as README.txt) will
  be skipped.

  Inputs:
  - models_dir: String giving the path to a directory containing model files.
    Each model file is a pickled dictionary with a 'model' field.

  Returns:
  A dictionary mapping model file names to models.
  """
  models = {}
  for model_file in os.listdir(models_dir):
    with open(os.path.join(models_dir, model_file), 'rb') as f:
      try:
        models[model_file] = load_pickle(f)['model']
      except pickle.UnpicklingError:
        continue
  return models

ext: py

max_stars_repo_name: pblankley/paulnet

max_issues_repo_name: pblankley/paulnet

max_forks_repo_name: pblankley/paulnet

reason: import numpy,from scipy


text:
import numpy as np
import cv2
import argparse

from numpy.ma.core import where

def simple_convert(img):
    img_std = np.std(img, axis=(0, 1))
    img_mean = np.mean(img, axis=(0, 1))

    col_bits = 8 * img_std / img_std.sum()
    col_bits_int = (col_bits * 2 + 1) // 2

    print("R: {} bits, G: {} bits, B: {} bits".format(int(col_bits_int[2]), int(col_bits_int[1]), int(col_bits_int[0])))

    col_interval = 256 / (2 ** col_bits_int)
    dot_img = img / col_interval
    dot_img = (dot_img * 2 + 1) // 2
    dot_img = dot_img * col_interval - 1
    dot_img[dot_img < 0] = 0

    return dot_img

def statistical_convert(img):
    pass

def Dotcam():
    capture = cv2.VideoCapture(-1)
    while(True):
        ret, frame = capture.read()
        cv2.imshow('frame', frame)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    capture.release()
    cv2.destroyAllWindows()

def main(args):
    if args.camera == 1:
        Dotcam()
    else:
        img = cv2.imread(args.fig)
        dot = simple_convert(img)

        cv2.imwrite(args.save, dot)

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Convert RGB each 8bit piccture to Dot picture")
    parser.add_argument("--fig", default="Demos/Lenna_(test_image).png")
    parser.add_argument("--save", default="Demos/outputs.png")
    parser.add_argument("--camera", default=0, type=int)
    args = parser.parse_args()
    main(args)

max_stars_repo_name: hanebarla/DotConverter

max_issues_repo_name: hanebarla/DotConverter

max_forks_repo_name: hanebarla/DotConverter

reason: import numpy,from numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
.. module:: ellipse_fitting
   :platform: Unix, Windows
   :synopsis: Ellipse fitting algorithms and handling of ellipse information.

.. moduleauthor:: hbldh <henrik.blidh@nedomkull.com>

Created on 2013-05-05, 23:22

"""

from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import

import numpy as np

from b2ac.compat import *
from b2ac.matrix.matrix_ref import inverse_symmetric_3by3_double, inverse_symmetric_3by3_int, add_symmetric_matrix

def fit_B2AC(points):
    """Ellipse fitting in Python with numerically unstable algorithm.

    Described `here <http://research.microsoft.com/pubs/67845/ellipse-pami.pdf>`_.

    N_POLYPOINTS.B. Do not use, since it works with almost singular matrix.

    :param points: The [Nx2] array of points to fit ellipse to.
    :type points: :py:class:`numpy.ndarray`
    :return: The conic section array defining the fitted ellipse.
    :rtype: :py:class:`numpy.ndarray`

    """
    import scipy.linalg as scla

    constraint_matrix = np.zeros((6, 6))
    constraint_matrix[0, 2] = 2
    constraint_matrix[1, 1] = -1
    constraint_matrix[2, 0] = 2

    S = _calculate_scatter_matrix_py(points[:, 0], points[:, 1])

    evals, evect = scla.eig(S, constraint_matrix)
    ind = np.where(evals == (evals[evals > 0].min()))[0][0]
    return evect[:, ind]

def fit_improved_B2AC_numpy(points):
    """Ellipse fitting in Python with improved B2AC algorithm as described in
    this `paper <http://autotrace.sourceforge.net/WSCG98.pdf>`_.

    This version of the fitting simply applies NumPy:s methods for calculating
    the conic section, modelled after the Matlab code in the paper:

    .. code-block::

        function a = fit_ellipse(x, y)

        D1 = [x .ˆ 2, x .* y, y .ˆ 2]; % quadratic part of the design matrix
        D2 = [x, y, ones(size(x))]; % linear part of the design matrix
        S1 = D1’ * D1; % quadratic part of the scatter matrix
        S2 = D1’ * D2; % combined part of the scatter matrix
        S3 = D2’ * D2; % linear part of the scatter matrix
        T = - inv(S3) * S2’; % for getting a2 from a1
        M = S1 + S2 * T; % reduced scatter matrix
        M = [M(3, :) ./ 2; - M(2, :); M(1, :) ./ 2]; % premultiply by inv(C1)
        [evec, eval] = eig(M); % solve eigensystem
        cond = 4 * evec(1, :) .* evec(3, :) - evec(2, :) .ˆ 2; % evaluate a’Ca
        a1 = evec(:, find(cond > 0)); % eigenvector for min. pos. eigenvalue
        a = [a1; T * a1]; % ellipse coefficients

    :param points: The [Nx2] array of points to fit ellipse to.
    :type points: :py:class:`numpy.ndarray`
    :return: The conic section array defining the fitted ellipse.
    :rtype: :py:class:`numpy.ndarray`

    """
    x = points[:, 0]
    y = points[:, 1]

    D1 = np.vstack([x ** 2, x * y, y ** 2]).T
    D2 = np.vstack([x, y, np.ones((len(x), ), dtype=x.dtype)]).T
    S1 = D1.T.dot(D1)
    S2 = D1.T.dot(D2)
    S3 = D2.T.dot(D2)
    T = -np.linalg.inv(S3).dot(S2.T)
    M = S1 + S2.dot(T)
    M = np.array([M[2, :] / 2, -M[1, :], M[0, :] / 2])
    eval, evec = np.linalg.eig(M)
    cond = (4 * evec[:, 0] * evec[:, 2]) - (evec[:, 1] ** 2)
    I = np.where(cond > 0)[0]
    a1 = evec[:, I[np.argmin(cond[I])]]
    return np.concatenate([a1, T.dot(a1)])

def fit_improved_B2AC(points):
    """Ellipse fitting in Python with improved B2AC algorithm as described in
    this `paper <http://autotrace.sourceforge.net/WSCG98.pdf>`_.

    This version of the fitting uses float storage during calculations and performs the
    eigensolver on a float array.

    :param points: The [Nx2] array of points to fit ellipse to.
    :type points: :py:class:`numpy.ndarray`
    :return: The conic section array defining the fitted ellipse.
    :rtype: :py:class:`numpy.ndarray`

    """
    points = np.array(points, 'float')
    S = _calculate_scatter_matrix_py(points[:, 0], points[:, 1])
    S3 = S[3:, 3:]
    S3 = np.array([S3[0, 0], S3[0, 1], S3[0, 2], S3[1, 1], S3[1, 2], S3[2, 2]])
    S3_inv = inverse_symmetric_3by3_double(S3).reshape((3, 3))
    S2 = S[:3, 3:]
    T = -np.dot(S3_inv, S2.T)
    M = S[:3, :3] + np.dot(S2, T)
    inv_mat = np.array([[0, 0, 0.5], [0, -1, 0], [0.5, 0, 0]], 'float')
    M = inv_mat.dot(M)

    e_vals, e_vect = np.linalg.eig(M)

    try:
        elliptical_solution_index = np.where(((4 * e_vect[0, :] * e_vect[2, :]) - ((e_vect[1, :] ** 2))) > 0)[0][0]
    except:
        # No positive eigenvalues. Fit was not ellipse.
        raise ArithmeticError("No elliptical solution found.")

    a = e_vect[:, elliptical_solution_index]
    if a[0] < 0:
        a = -a
    return np.concatenate((a, np.dot(T, a)))

def fit_improved_B2AC_int(points):
    """Ellipse fitting in Python with improved B2AC algorithm as described in
    this `paper <http://autotrace.sourceforge.net/WSCG98.pdf>`_.

    This version of the fitting uses int64 storage during calculations and performs the
    eigensolver on an integer array.

    :param points: The [Nx2] array of points to fit ellipse to.
    :type points: :py:class:`numpy.ndarray`
    :return: The conic section array defining the fitted ellipse.
    :rtype: :py:class:`numpy.ndarray`

    """
    S = _calculate_scatter_matrix_c(points[:, 0], points[:, 1])
    S1 = np.array([S[0, 0], S[0, 1], S[0, 2], S[1, 1], S[1, 2], S[2, 2]])
    S3 = np.array([S[3, 3], S[3, 4], S[3, 5], S[4, 4], S[4, 5], S[5, 5]])
    adj_S3, det_S3 = inverse_symmetric_3by3_int(S3)
    S2 = S[:3, 3:]
    T_no_det = - np.dot(np.array(adj_S3.reshape((3, 3)), 'int64'), np.array(S2.T, 'int64'))
    M_term2 = np.dot(np.array(S2, 'int64'), T_no_det) // det_S3
    M = add_symmetric_matrix(M_term2, S1)
    M[[0, 2], :] /= 2
    M[1, :] = -M[1, :]

    e_vals, e_vect = np.linalg.eig(M)

    try:
        elliptical_solution_index = np.where(((4 * e_vect[0, :] * e_vect[2, :]) - ((e_vect[1, :] ** 2))) > 0)[0][0]
    except:
        # No positive eigenvalues. Fit was not ellipse.
        raise ArithmeticError("No elliptical solution found.")
    a = e_vect[:, elliptical_solution_index]
    return np.concatenate((a, np.dot(T_no_det, a) / det_S3))

def _calculate_scatter_matrix_py(x, y):
    """Calculates the complete scatter matrix for the input coordinates.

    :param x: The x coordinates.
    :type x: :py:class:`numpy.ndarray`
    :param y: The y coordinates.
    :type y: :py:class:`numpy.ndarray`
    :return: The complete scatter matrix.
    :rtype: :py:class:`numpy.ndarray`

    """
    D = np.ones((len(x), 6), dtype=x.dtype)
    D[:, 0] = x * x
    D[:, 1] = x * y
    D[:, 2] = y * y
    D[:, 3] = x
    D[:, 4] = y

    return D.T.dot(D)

def _calculate_scatter_matrix_c(x, y):
    """Calculates the upper triangular scatter matrix for the input coordinates.

    :param x: The x coordinates.
    :type x: :py:class:`numpy.ndarray`
    :param y: The y coordinates.
    :type y: :py:class:`numpy.ndarray`
    :return: The upper triangular scatter matrix.
    :rtype: :py:class:`numpy.ndarray`

    """
    S = np.zeros((6, 6), 'int32')

    for i in xrange(len(x)):
        tmp_x2 = x[i] ** 2
        tmp_x3 = tmp_x2 * x[i]
        tmp_y2 = y[i] ** 2
        tmp_y3 = tmp_y2 * y[i]

        S[0, 0] += tmp_x2 * tmp_x2
        S[0, 1] += tmp_x3 * y[i]
        S[0, 2] += tmp_x2 * tmp_y2
        S[0, 3] += tmp_x3
        S[0, 4] += tmp_x2 * y[i]
        S[0, 5] += tmp_x2
        S[1, 2] += tmp_y3 * x[i]
        S[1, 4] += tmp_y2 * x[i]
        S[1, 5] += x[i] * y[i]
        S[2, 2] += tmp_y2 * tmp_y2
        S[2, 4] += tmp_y3
        S[2, 5] += tmp_y2
        S[3, 5] += x[i]
        S[4, 5] += y[i]

    S[5, 5] = len(x)

    # Doubles
    S[1, 1] = S[0, 2]
    S[1, 3] = S[0, 4]
    S[2, 3] = S[1, 4]
    S[3, 3] = S[0, 5]
    S[3, 4] = S[1, 5]
    S[4, 4] = S[2, 5]

    return S

max_stars_repo_name: hbldh/ellipse-fitting

max_issues_repo_name: hbldh/ellipse-fitting

max_forks_repo_name: hbldh/ellipse-fitting

reason: import numpy,import scipy


text:
"""
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

  Licensed under the Apache License, Version 2.0 (the "License").
  You may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""

import os
import pickle
import glob

import numpy as np
import pandas as pd

from sklearn.neighbors import NearestNeighbors

from utils.utils import f1_score, best_f1_linspace, normalize_scores
from utils.utils_data import get_random_occlusion_mask

MACHINES = ['machine-1-1','machine-1-2','machine-1-3','machine-1-4','machine-1-5','machine-1-6','machine-1-7','machine-1-8',
            'machine-2-1', 'machine-2-2','machine-2-3','machine-2-4','machine-2-5','machine-2-6','machine-2-7','machine-2-8','machine-2-9',
            'machine-3-1', 'machine-3-2', 'machine-3-3', 'machine-3-4','machine-3-5','machine-3-6','machine-3-7','machine-3-8', 'machine-3-9','machine-3-10', 'machine-3-11']

# ------------------------------------------------------- SMD -------------------------------------------------------
def smd_load_scores(scores_dir, data_dir, machines):
    labels_list = []
    scores_list = []
    for machine in machines:
        results_file = f'{scores_dir}/SMD/{machine}/results.p'
        results = pickle.load(open(results_file,'rb'))

        scores = results['score']
        labels = np.loadtxt(f'{data_dir}/ServerMachineDataset/test_label/{machine}.txt',delimiter=',')
        scores = scores.repeat(10)[-len(labels):]

        assert scores.shape == labels.shape, 'Wrong dimensions'

        labels_list.append(labels)
        scores_list.append(scores)

    return scores_list, labels_list

def smd_compute_f1(scores_dir, n_splits, data_dir):

    scores, labels = smd_load_scores(scores_dir=scores_dir, data_dir=data_dir, machines=MACHINES)

    scores_normalized = normalize_scores(scores=scores, interval_size=64*4*10) # 64*1*10

    scores_normalized = np.hstack(scores_normalized)
    labels = np.hstack(labels)

    f1, precision, recall, *_ = best_f1_linspace(scores=scores_normalized, labels=labels, n_splits=n_splits, segment_adjust=True) #1000

    return f1, precision, recall

def smd_one_liner(occlusion_intervals, occlusion_prob, data_dir):
    labels_list = []
    scores_list = []
    for machine in MACHINES:
        train_data = np.loadtxt(f'{data_dir}/ServerMachineDataset/train/{machine}.txt',delimiter=',')
        test_data = np.loadtxt(f'{data_dir}/ServerMachineDataset/test/{machine}.txt',delimiter=',')

        # Mask
        np.random.seed(1)
        train_mask = get_random_occlusion_mask(dataset=train_data[:,None,:], n_intervals=occlusion_intervals, occlusion_prob=occlusion_prob)
        train_mask = train_mask[:,0,:]

        train_means = np.average(train_data, axis=0, weights=train_mask)
        train_means = train_means[None, :]

        scores = np.abs(test_data - train_means)
        scores = scores.mean(axis=1)

        labels = np.loadtxt(f'{data_dir}/ServerMachineDataset/test_label/{machine}.txt',delimiter=',')
        labels_list.append(labels)
        scores_list.append(scores)

    scores_normalized = normalize_scores(scores=scores_list, interval_size=64*4*10) # 64*1*10

    scores_normalized = np.hstack(scores_normalized)
    labels = np.hstack(labels_list)

    f1, precision, recall, *_ = best_f1_linspace(scores=scores_normalized, labels=labels, n_splits=100, segment_adjust=True)

    return f1, precision, recall

def smd_nn(occlusion_intervals, occlusion_prob, data_dir):
    downsampling_size = 10
    labels_list = []
    scores_list = []
    for machine in MACHINES:
        train_data = np.loadtxt(f'{data_dir}/ServerMachineDataset/train/{machine}.txt',delimiter=',')
        train_data = train_data[:, None, :]
        test_data = np.loadtxt(f'{data_dir}/ServerMachineDataset/test/{machine}.txt',delimiter=',')
        test_data = test_data[:, None, :]
        len_test = len(test_data)

        # Padding
        dataset = np.vstack([train_data, test_data])

        right_padding = downsampling_size - dataset.shape[0]%downsampling_size
        dataset = np.pad(dataset, ((right_padding, 0), (0,0), (0,0) ))

        right_padding = downsampling_size - len_test%downsampling_size

        dataset = dataset.reshape(dataset.shape[0]//downsampling_size, -1, 1, dataset.shape[2]).max(axis=1)
        len_test_downsampled = int(np.ceil(len_test/downsampling_size))

        train_data = dataset[:-len_test_downsampled, 0, :]
        test_data = dataset[-len_test_downsampled:, 0, :]

        # Mask
        np.random.seed(1)
        train_mask = get_random_occlusion_mask(dataset=train_data[:, None, :], n_intervals=occlusion_intervals, occlusion_prob=occlusion_prob)
        train_mask = train_mask[:, 0, :]

        train_data = train_data*train_mask

        nbrs = NearestNeighbors(n_neighbors=10, algorithm='ball_tree').fit(train_data)
        distances, _ = nbrs.kneighbors(test_data)
        scores = distances.mean(axis=1)

        labels = np.loadtxt(f'{data_dir}/ServerMachineDataset/test_label/{machine}.txt',delimiter=',')
        scores = scores.repeat(10)[-len(labels):]

        labels_list.append(labels)
        scores_list.append(scores)

    scores_normalized = normalize_scores(scores=scores_list, interval_size=64*4*10)
    scores_normalized = np.hstack(scores_normalized)
    labels = np.hstack(labels_list)

    f1, precision, recall, *_ = best_f1_linspace(scores=scores_normalized, labels=labels, n_splits=100, segment_adjust=True)

    return f1, precision, recall

# ------------------------------------------------------- NASA -------------------------------------------------------
def nasa_load_scores(dataset, scores_dir, data_dir):
    meta_data = pd.read_csv(f'{data_dir}/NASA/labeled_anomalies.csv')
    entities = meta_data[meta_data['spacecraft']==dataset]['chan_id'].values

    labels_list = []
    scores_list = []
    for entity in entities:
        results_file = f'{scores_dir}/{dataset}/{entity}/results.p'
        results = pickle.load(open(results_file,'rb'))

        ts_scores = results['ts_score']
        labels = np.load(f'{data_dir}/NASA/labels/{entity}.npy')
        ts_scores = np.swapaxes(ts_scores,0,1)
        ts_scores = ts_scores.reshape(len(ts_scores), -1)
        ts_scores = ts_scores[:, -len(labels):]
        score = ts_scores[0]

        # NaN protection (to remove)
        if np.sum(np.isnan(score)) > 0:
            print('entity', entity)
            print('nans', np.mean(np.isnan(score)))
            # score = score[~np.isnan(score)]
            # labels = labels[~np.isnan(score)]
        else:
            assert score.shape == labels.shape, 'Wrong dimensions'
            labels_list.append(labels)
            scores_list.append(score)

    return scores_list, labels_list

def nasa_compute_f1(dataset, scores_dir, n_splits, data_dir):

    scores, labels = nasa_load_scores(dataset=dataset, scores_dir=scores_dir, data_dir=data_dir)

    if dataset=='SMAP':
        scores_normalized = normalize_scores(scores=scores, interval_size=64*4)
    else:
        scores_normalized = normalize_scores(scores=scores, interval_size=10000)

    labels = np.hstack(labels)
    scores = np.hstack(scores)
    scores_normalized = np.hstack(scores_normalized)

    scores_max = scores_normalized.max()
    if np.isinf(scores_max) or np.isnan(scores_max):
        f1 = None
        precision = None
        recall = None
    else:
        f1, precision, recall, *_ = best_f1_linspace(scores=scores_normalized, labels=labels, n_splits=n_splits, segment_adjust=True)

    return f1, precision, recall

def nasa_one_liner(dataset, occlusion_intervals, occlusion_prob, data_dir):
    meta_data = pd.read_csv(f'{data_dir}/NASA/labeled_anomalies.csv')
    entities = meta_data[meta_data['spacecraft']==dataset]['chan_id'].values

    labels_list = []
    scores_list = []
    for entity in entities:
        train_data = np.load(f'{data_dir}/NASA/train/{entity}.npy')
        test_data = np.load(f'{data_dir}/NASA/test/{entity}.npy')
        labels = np.load(f'{data_dir}/NASA/labels/{entity}.npy')

        # Mask
        np.random.seed(1)
        train_mask = get_random_occlusion_mask(dataset=train_data[:,None,:], n_intervals=occlusion_intervals, occlusion_prob=occlusion_prob)
        train_mask = train_mask[:,0,:]

        train_means = np.average(train_data, axis=0, weights=train_mask)
        train_means = train_means[None, :]

        scores = np.abs(test_data - train_means)[:,0]

        labels_list.append(labels)
        scores_list.append(scores)

    scores_normalized = normalize_scores(scores=scores_list, interval_size=10000)

    scores_normalized = np.hstack(scores_normalized)
    labels = np.hstack(labels_list)

    f1, precision, recall, *_ = best_f1_linspace(scores=scores_normalized, labels=labels, n_splits=100, segment_adjust=True)

    return f1, precision, recall

def nasa_nn(dataset, occlusion_intervals, occlusion_prob, data_dir):
    meta_data = pd.read_csv(f'{data_dir}/NASA/labeled_anomalies.csv')
    entities = meta_data[meta_data['spacecraft']==dataset]['chan_id'].values

    labels_list = []
    scores_list = []
    for entity in entities:
        train_data = np.load(f'{data_dir}/NASA/train/{entity}.npy')
        test_data = np.load(f'{data_dir}/NASA/test/{entity}.npy')
        labels = np.load(f'{data_dir}/NASA/labels/{entity}.npy')

        # Mask
        np.random.seed(1)
        train_mask = get_random_occlusion_mask(dataset=train_data[:, None, :], n_intervals=occlusion_intervals, occlusion_prob=occlusion_prob)
        train_mask = train_mask[:, 0, :]

        train_data = train_data*train_mask

        nbrs = NearestNeighbors(n_neighbors=10, algorithm='ball_tree').fit(train_data)
        distances, _ = nbrs.kneighbors(test_data)
        scores = distances.mean(axis=1)

        labels_list.append(labels)
        scores_list.append(scores)

    scores_normalized = normalize_scores(scores=scores_list, interval_size=10000)

    scores_normalized = np.hstack(scores_normalized)
    labels = np.hstack(labels_list)

    f1, precision, recall, *_ = best_f1_linspace(scores=scores_normalized, labels=labels, n_splits=100, segment_adjust=True)

    return f1, precision, recall

# ------------------------------------------------------- SWAT -------------------------------------------------------
def swat_compute_f1(n_splits, results_dir):

    results_file = f'{results_dir}/SWAT/results.p'
    results = pickle.load(open(results_file,'rb'))

    labels = pd.read_csv(f'../data/SWAT/labels_test.csv').values

    ts_scores = results['ts_score']
    len_test = len(labels)//2
    labels = labels[-len_test:, 0]

    x_train_true = results['x_train_true']
    x_train_hat = results['x_train_hat']
    train_errors = np.square(x_train_hat-x_train_true)
    train_errors = train_errors.mean(axis=1)
    train_errors = 100*train_errors[:,None]

    ts_scores = np.swapaxes(ts_scores,0,1)
    ts_scores = ts_scores.reshape(len(ts_scores),-1)
    ts_scores = ts_scores.repeat(10,axis=1)
    ts_scores = ts_scores[:, -len(labels):]

    scores_new = ts_scores/(train_errors)
    scores_new[10,:] = 0
    scores_new = scores_new.mean(axis=0)

    f1, precision, recall, *_ = best_f1_linspace(scores=scores_new, labels=labels, n_splits=n_splits, segment_adjust=False)

    return f1, precision, recall

def swat_one_liner(occlusion_intervals, occlusion_prob, data_dir):
    train_data = pd.read_csv(f'{data_dir}/SWAT/swat_train.csv').values
    test_data = pd.read_csv(f'{data_dir}/SWAT/swat_test.csv').values
    labels = pd.read_csv(f'{data_dir}/SWAT/labels_test.csv').values
    len_test = len(test_data)

    # Mask
    np.random.seed(1)
    train_mask = get_random_occlusion_mask(dataset=train_data[:,None,:], n_intervals=occlusion_intervals, occlusion_prob=occlusion_prob)
    train_mask = train_mask[:,0,:]

    train_data = train_data*train_mask

    train_means = np.average(train_data, axis=0, weights=train_mask)
    train_means = train_means[None, :]

    train_errors = np.square(train_data-train_means)
    train_errors = train_errors.mean(axis=0)
    train_errors = train_errors[None, :] + 0.01

    scores = np.abs(test_data - train_means)
    scores = scores[len_test//2:]
    scores = scores/(train_errors)
    scores[:, 10] = 0
    scores = scores.mean(axis=1)
    labels = labels[len_test//2:, 0]

    f1, precision, recall, *_ = best_f1_linspace(scores=scores, labels=labels, n_splits=100, segment_adjust=False)

    return f1, precision, recall

def swat_nn(occlusion_intervals, occlusion_prob, data_dir):
    downsampling_size = 10

    train_data = pd.read_csv(f'{data_dir}/SWAT/swat_train.csv').values
    train_data = train_data[:,None,:]
    test_data = pd.read_csv(f'{data_dir}/SWAT/swat_test.csv').values
    test_data = test_data[:,None,:]
    labels = pd.read_csv(f'{data_dir}/SWAT/labels_test.csv').values
    len_test = len(labels)

    # Padding
    dataset = np.vstack([train_data, test_data])

    right_padding = downsampling_size - dataset.shape[0]%downsampling_size
    dataset = np.pad(dataset, ((right_padding, 0), (0,0), (0,0) ))

    right_padding = downsampling_size - len_test%downsampling_size

    dataset = dataset.reshape(dataset.shape[0]//downsampling_size, -1, 1, dataset.shape[2]).max(axis=1)
    len_test_downsampled = int(np.ceil(len_test/downsampling_size))

    train_data = dataset[:-len_test_downsampled, 0, :]
    test_data = dataset[-len_test_downsampled:, 0, :]

    # Mask
    np.random.seed(1)
    train_mask = get_random_occlusion_mask(dataset=train_data[:, None, :], n_intervals=occlusion_intervals, occlusion_prob=occlusion_prob)
    train_mask = train_mask[:, 0, :]
    train_data = train_data*train_mask

    train_max = train_data.max(axis=0, keepdims=True)
    train_data = train_data/train_max
    test_data = test_data/train_max

    nbrs = NearestNeighbors(n_neighbors=10, algorithm='ball_tree').fit(train_data)
    distances, _ = nbrs.kneighbors(test_data)
    scores = distances.mean(axis=1)

    labels = labels[len_test//2:, 0]
    scores = scores.repeat(10)[-len(labels):]

    f1, precision, recall, *_ = best_f1_linspace(scores=scores, labels=labels, n_splits=100, segment_adjust=False)

    return f1, precision, recall

max_stars_repo_name: cchallu/dghl

max_issues_repo_name: cchallu/dghl

max_forks_repo_name: cchallu/dghl

reason: import numpy


text:
import tensorflow as tf
import numpy as np
import time

# num_features = 784

# mnist = tf.keras.datasets.mnist

# (x_train, y_train), (x_test, y_test) = mnist.load_data()

# x_train, x_test = x_train.reshape([-1, num_features]), x_test.reshape([-1, num_features])

from sklearn.datasets import fetch_openml
from keras.utils.np_utils import to_categorical
import numpy as np
from sklearn.model_selection import train_test_split

x, y = fetch_openml('mnist_784', version=1, return_X_y=True)
x = (x/255).astype('float32')
y = to_categorical(y)

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.15, random_state=42)

#print(y_train)

class DeepNeuralNetwork():
	def __init__(self, sizes, epochs=1, l_rate=0.001):
		self.sizes = sizes
		self.epochs = epochs
		self.l_rate = l_rate

		# we save all parameters in the neural network in this dictionary
		self.params = self.initialization()

	def sigmoid(self, x, derivative=False):
		if derivative:
			return (np.exp(-x))/((np.exp(-x)+1)**2)
		return 1/(1 + np.exp(-x))

	def softmax(self, x, derivative=False):
		# Numerically stable with large exponentials
		exps = np.exp(x - x.max())
		if derivative:
			return exps / np.sum(exps, axis=0) * (1 - exps / np.sum(exps, axis=0))
		return exps / np.sum(exps, axis=0)

	def initialization(self):
		# number of nodes in each layer
		input_layer=self.sizes[0]
		#print("INPUT LAYER",input_layer)
		hidden_1=self.sizes[1]
		hidden_2=self.sizes[2]
		output_layer=self.sizes[3]

		params = {
			'W1':np.random.randn(hidden_1, input_layer) * np.sqrt(1. / hidden_1),
			'W2':np.random.randn(hidden_2, hidden_1) * np.sqrt(1. / hidden_2),
			'W3':np.random.randn(output_layer, hidden_2) * np.sqrt(1. / output_layer),
			'B1':np.ones(128, dtype='int')*0.1,
			'B2':np.ones(64, dtype='int')*0.1,
			'B3':np.ones(10, dtype='int')*0.1
		}
		#print(params['W2'].shape)
		return params

	def feedforward(self, x_train):
		params = self.params

		# input layer activations becomes sample
		params['A0'] = x_train
		#print(x_train.shape)
		# input layer to hidden layer 1 and passed through activation function
		params['Z1'] = np.dot(params["W1"], params['A0']) + params['B1']
		params['A1'] = self.sigmoid(params['Z1'])

		# hidden layer 1 to hidden layer 2 and passed through activation function
		params['Z2'] = np.dot(params["W2"], params['A1']) + params['B2']
		params['A2'] = self.sigmoid(params['Z2'])

		# hidden layer 2 to output layer
		params['Z3'] = np.dot(params["W3"], params['A2'])+params['B3']
		params['A3'] = self.softmax(params['Z3'])
		#print(params['A3'].shape)
		return params['A3']

	def backpropagation(self, y_train, output):

		params = self.params
		change_w = {}

		# Calculate W3 update
		error = 2 * (output - y_train) / output.shape[0] * self.softmax(params['Z3'], derivative=True)
		change_w['W3'] = np.outer(error, params['A2'])

		# Calculate W2 update
		error = np.dot(params['W3'].T, error) * self.sigmoid(params['Z2'], derivative=True)
		change_w['W2'] = np.outer(error, params['A1'])

		# Calculate W1 update
		error = np.dot(params['W2'].T, error) * self.sigmoid(params['Z1'], derivative=True)
		change_w['W1'] = np.outer(error, params['A0'])

		return change_w

	def update_weights(self, changes_to_w):

		for key, value in changes_to_w.items():
			#print(key, value.shape)
			self.params[key] -= self.l_rate * value

	def accuracy(self, x_val, y_val):
		predictions = []

		for x, y in zip(x_val, y_val):
			output = self.feedforward(x)
			pred = np.argmax(output)
			predictions.append(pred == np.argmax(y))

		return np.mean(predictions)

	def train(self, x_train, y_train, x_val, y_val):
		start_time = time.time()
		for iteration in range(self.epochs):
			for x,y in zip(x_train, y_train):
				output = self.feedforward(x)
				#print("OUTPUT", output.shape)
				changes_to_w = self.backpropagation(y, output)
				self.update_weights(changes_to_w)

			accuracy = self.accuracy(x_val, y_val)
			print('Epoch: {0}, Time Spent: {1:.2f}s, Accuracy: {2:.2f}%'.format(
				iteration+1, time.time() - start_time, accuracy * 100
			))

dnn = DeepNeuralNetwork(sizes=[784, 128, 64, 10])
dnn.train(x_train, y_train, x_test, y_test)


text:
from __future__ import print_function
import numpy as np
import os
import argparse
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit, leastsq
import simplejson

# Made for the electrowetting case, thus the default.

def parse_args():
    parser = argparse.ArgumentParser(description="Analyze contact angle")
    parser.add_argument("folders", nargs='+', type=str, help="Folders")
    parser.add_argument("-k", "--keys", type=str,
                        default=(
                            "V_top,contact_angle,rad_init,"
                            "concentration_init_s,concentration_init_d,"
                            "permittivity,surface_tension"),
                        help="Keys to get info from.")
    args = parser.parse_args()
    return args

def main():
    args = parse_args()

    for base_folder in args.folders:
        # folder = os.path.join(base_folder, "Analysis", "contour")

        params = simplejson.load(
            open(os.path.join(
                base_folder, "Settings",
                "parameters_from_tstep_0.dat")))

        keys = args.keys.split(",")

        for key in keys:
            if key in params:
                print("{}:\t{}".format(key, params[key]))

if __name__ == "__main__":
    main()

max_stars_repo_name: gautelinga/-BERNAISE

max_issues_repo_name: gautelinga/-BERNAISE

max_forks_repo_name: gautelinga/-BERNAISE


text:
import json
import re

import pandas as pd
import requests
import requests.exceptions
from numpy import asarray

from mootdx.logger import log

def get_adjust_year(symbol=None, year='2021', factor='00'):
    """
    采集同花顺复权数据

    :param symbol: 股票代码
    :param factor: 前后复权 before 或 01 为前复权， after 或 02为后复权
    :param year: 年份
    :return: DataFrame
    """

    # http://d.10jqka.com.cn/v2/line/hs_600036/01/2018.js
    # http://d.10jqka.com.cn/v6/line/hs_600000/00/all.js

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
        'Referer': 'http://stockpage.10jqka.com.cn/',
        'DNT': '1',
    }

    if factor == 'before':
        factor = '01'
    elif factor == 'after':
        factor = '02'

    if factor not in ['01', '02']:
        log.warning('复权参数错误，factor 的值必须是: before, after, 01, 02')
        return pd.DataFrame(data=None)

    try:
        url = f'http://d.10jqka.com.cn/v2/line/hs_{symbol}/{factor}/{year}.js'
        res = requests.get(url, headers=headers)
        res.raise_for_status()

        # 出现 window.location.href 则请求太频繁，需要稍等再采集
        text = re.findall(r'\((.*)\)', res.text)[0]
        text = json.loads(text)

        data = text['data'].split(';')
        data = [item.split(',')[:8] for item in data]

        columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'amount', 'adjust']
        return pd.DataFrame(data, index=list(asarray(data).T[0]), columns=columns)
    except requests.exceptions.RequestException:
        log.warning('网络连接失败，请重试...')
    except IndexError as e:
        log.warning('数据解析错误，请求太频繁，请稍后重试...')
        log.debug(e)

    return pd.DataFrame(data=None)

max_stars_repo_name: wangjild/mootdx

max_issues_repo_name: wangjild/mootdx

max_forks_repo_name: wangjild/mootdx

reason: from numpy


text:
# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""Fever models built from tf.keras.Model module API (eg two tower ranker)."""

from language.serene import layers
import numpy as np
import tensorflow.compat.v2 as tf

def gelu(x):
  """Gaussian Error Linear Unit.

  This is a smoother version of the RELU.
  Original paper: https://arxiv.org/abs/1606.08415
  Args:
    x: float Tensor to perform activation.

  Returns:
    `x` with the GELU activation applied.
  """
  cdf = 0.5 * (1.0 + tf.tanh(
      (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))))
  return x * cdf

def parse_activation(
    name):
  if name == 'gelu':
    return gelu
  else:
    return name

class TwoTowerRanker(tf.keras.Model):
  """Build a model of two towers that are joined with a matching function."""

  # pyformat: disable
  def __init__(
      self,
      vocab_size, *,
      embedder_name,
      tied_encoders,
      matcher_name,
      bidirectional,
      contextualizer,
      context_num_layers,
      activation,
      matcher_hidden_size,
      word_emb_size,
      hidden_size,
      projection_dim,
      bert_model_name,
      bert_model_path,
      bert_trainable,
      bert_dropout,
      dropout,
      use_batch_norm,
      classify_claim):
    # pyformat: enable
    """TwoTower model.

    Args:
      vocab_size: Size of vocab
      embedder_name: Which embedder, basic or bert
      tied_encoders: Whether to tie the encoders
      matcher_name: What kind of matcher to use
      bidirectional: Whether context layer is bidirectional
      contextualizer: Which contextualizer to use (e.g., GRU)
      context_num_layers: Number of layers in contextualizer
      activation: Activation function to use for feed forward layers
      matcher_hidden_size: Hidden size of matcher, if it has one
      word_emb_size: Word embedding hidden size
      hidden_size: Hidden size of contextualizer
      projection_dim: Dimension to project embeddings to
      bert_model_name: Name of bert model (e.g., base vs large)
      bert_model_path: Path to the bert checkpoint
      bert_trainable: Whether bert part of model is trainable
      bert_dropout: Dropout rate on bert embeddings
      dropout: Dropout rate to use
      use_batch_norm: Whether to use batch norm
      classify_claim: Whether to classify the claim
    """
    super().__init__(name='two_tower_ranker')
    self._vocab_size = vocab_size
    self._embedder_name = embedder_name
    self._matcher_name = matcher_name
    self._bidirectional = bidirectional
    self._contextualizer = contextualizer
    self._context_num_layers = context_num_layers
    self._activation = activation
    activation = parse_activation(activation)
    self._matcher_hidden_size = matcher_hidden_size
    self._word_emb_size = word_emb_size
    self._hidden_size = hidden_size
    self._projection_dim = projection_dim
    self._bert_model_name = bert_model_name
    self._bert_model_path = bert_model_path
    self._bert_trainable = bert_trainable
    self._bert_dropout = bert_dropout
    self._dropout = dropout
    self._use_batch_norm = use_batch_norm
    self._classify_claim = classify_claim
    self._tied_encoders = tied_encoders

    if embedder_name == 'classic_embedder':
      if self._tied_encoders:
        self._claim_encoder = layers.ClassicEmbedder(
            vocab_size=vocab_size,
            word_emb_size=word_emb_size,
            use_batch_norm=use_batch_norm,
            hidden_size=hidden_size,
            dropout=dropout,
            bidirectional=bidirectional,
            contextualizer=contextualizer,
            context_num_layers=context_num_layers,
            name='tied_encoder',
        )
        self._evidence_encoder = self._claim_encoder
      else:
        self._claim_encoder = layers.ClassicEmbedder(
            vocab_size=vocab_size,
            word_emb_size=word_emb_size,
            use_batch_norm=use_batch_norm,
            hidden_size=hidden_size,
            dropout=dropout,
            bidirectional=bidirectional,
            contextualizer=contextualizer,
            context_num_layers=context_num_layers,
            name='claim_encoder',
        )
        self._evidence_encoder = layers.ClassicEmbedder(
            vocab_size=vocab_size,
            word_emb_size=word_emb_size,
            use_batch_norm=use_batch_norm,
            hidden_size=hidden_size,
            dropout=dropout,
            bidirectional=bidirectional,
            contextualizer=contextualizer,
            context_num_layers=context_num_layers,
            name='evidence_encoder',
        )
    elif embedder_name == 'bert_embedder':
      if self._tied_encoders:
        self._claim_encoder = layers.BertEmbedder(
            bert_model_name,
            bert_model_path,
            name='tied_encoder',
            bert_trainable=bert_trainable,
            bert_dropout=bert_dropout,
        )
        self._evidence_encoder = self._claim_encoder
      else:
        self._claim_encoder = layers.BertEmbedder(
            bert_model_name,
            bert_model_path,
            name='claim_encoder',
            projection_dim=projection_dim,
            bert_trainable=bert_trainable,
            bert_dropout=bert_dropout,
        )
        self._evidence_encoder = layers.BertEmbedder(
            bert_model_name,
            bert_model_path,
            name='evidence_encoder',
            projection_dim=projection_dim,
            bert_trainable=bert_trainable,
            bert_dropout=bert_dropout,
        )
    else:
      raise ValueError('Invalid embedder')

    matcher = layers.matcher_registry[matcher_name](
        hidden_size=matcher_hidden_size,
        dropout=dropout,
        use_batch_norm=use_batch_norm,
        activation=activation,
        # Relevant or not
        n_classes=1,
    )
    if self._tied_encoders:
      self._claim_projector = tf.keras.layers.Dense(projection_dim)
      self._evidence_projector = self._claim_projector
    else:
      self._claim_projector = tf.keras.layers.Dense(projection_dim)
      self._evidence_projector = tf.keras.layers.Dense(projection_dim)
    self._evidence_classifier = tf.keras.Sequential(name='evidence_classifier')
    self._evidence_classifier.add(matcher)
    self._evidence_classifier.add(tf.keras.layers.Activation('sigmoid'))
    if classify_claim:
      # This doesn't need to preserve property of being findable with dot
      # product, so make it more powerful with bilinear matching.
      claim_hidden = layers.matcher_registry['bilinear_matcher'](
          hidden_size=matcher_hidden_size,
          dropout=dropout,
          use_batch_norm=use_batch_norm,
          activation=activation,
          # support, refute, not enough info
          n_classes=3,
      )
      self._claim_classifier = tf.keras.Sequential(name='claim_classifier')
      self._claim_classifier.add(claim_hidden)
      self._claim_classifier.add(tf.keras.layers.Activation('softmax'))
    else:
      self._claim_classifier = None

  def call(self,
           inputs,
           embed_claim=False,
           embed_evidence=False,
           training=None,
           **kwargs):
    """Model forward pass.

    Args:
      inputs: Input dictionary, dependent on type of embedder used
      embed_claim: Whether to embed the claim
      embed_evidence: Whether to embed the evidence
      training: Whether training mode is enabled
      **kwargs: Satisfies API compat

    Returns:
      Prediction of the model if claim is relevant to evidence
    """
    if embed_claim or embed_evidence:
      # (batch_size, projection_dim), (batch_size, projection_dim)
      return self.embed_only(
          inputs, embed_claim=embed_claim, embed_evidence=embed_evidence)
    else:
      if self._embedder_name == 'classic_embedder':
        # (batch_size, hidden_dim)
        encoded_claim = self._claim_encoder({'tokens': inputs['claim_text']})
        # (batch_size, hidden_dim)
        encoded_evidence = self._evidence_encoder(
            {'tokens': inputs['evidence_text']})
      elif self._embedder_name == 'bert_embedder':
        # (batch_size, hidden_dim)
        encoded_claim = self._claim_encoder({
            'word_ids': inputs['claim_text_word_ids'],
            'mask': inputs['claim_text_mask'],
            'segment_ids': inputs['claim_text_segment_ids'],
        })
        # (batch_size, hidden_dim)
        encoded_evidence = self._evidence_encoder({
            'word_ids': inputs['evidence_text_word_ids'],
            'mask': inputs['evidence_text_mask'],
            'segment_ids': inputs['evidence_text_segment_ids'],
        })
      else:
        raise ValueError('invalid embedder')
      out = {}
      # (batch_size, projection_dim)
      projected_claim = self._claim_projector(encoded_claim)
      # (batch_size, projection_dim)
      projected_evidence = self._evidence_projector(encoded_evidence)
      # (batch_size, 1)
      evidence_out = self._evidence_classifier(
          (projected_claim, projected_evidence))
      out['evidence_matching'] = tf.identity(
          evidence_out, name='evidence_matching')
      if self._classify_claim:
        # The claim takes the full size embedding, not the projected one.
        claim_out = self._claim_classifier((encoded_claim, encoded_evidence))
      else:
        # Predictions have three logits, one for each class
        claim_out = tf.fill((tf.shape(encoded_claim)[0], 3), 0.0)
      out['claim_classification'] = tf.identity(
          claim_out, name='claim_classification')
      return out

  def embed_only(
      self,
      inputs,
      *,
      embed_claim,
      embed_evidence,
      project = True):
    if self._embedder_name == 'classic_embedder':
      if embed_claim:
        # (batch_size, hidden_dim)
        encoded_claim = self._claim_encoder({'tokens': inputs['claim_text']})
      else:
        encoded_claim = None
      if embed_evidence:
        # (batch_size, hidden_dim)
        encoded_evidence = self._evidence_encoder(
            {'tokens': inputs['evidence_text']})
      else:
        encoded_evidence = None
    elif self._embedder_name == 'bert_embedder':
      if embed_claim:
        # (batch_size, hidden_dim)
        encoded_claim = self._claim_encoder({
            'word_ids': inputs['claim_text_word_ids'],
            'mask': inputs['claim_text_mask'],
            'segment_ids': inputs['claim_text_segment_ids'],
        })
      else:
        encoded_claim = None
      if embed_evidence:
        # (batch_size, hidden_dim)
        encoded_evidence = self._evidence_encoder({
            'word_ids': inputs['evidence_text_word_ids'],
            'mask': inputs['evidence_text_mask'],
            'segment_ids': inputs['evidence_text_segment_ids'],
        })
      else:
        encoded_evidence = None
    else:
      raise ValueError('invalid embedder')

    if project:
      if encoded_claim is not None:
        encoded_claim = self._claim_projector(encoded_claim)
      if encoded_evidence is not None:
        encoded_evidence = self._evidence_projector(encoded_evidence)
    return encoded_claim, encoded_evidence

  def get_config(self):
    config = super().get_config()
    config.update({
        'vocab_size': self._vocab_size,
        'embedder_name': self._embedder_name,
        'tied_encoders': self._tied_encoders,
        'matcher_name': self._matcher_name,
        'bidirectional': self._bidirectional,
        'contextualizer': self._contextualizer,
        'context_num_layers': self._context_num_layers,
        'activation': self._activation,
        'matcher_hidden_size': self._matcher_hidden_size,
        'word_emb_size': self._word_emb_size,
        'hidden_size': self._hidden_size,
        'projection_dim': self._projection_dim,
        'bert_model_name': self._bert_model_name,
        'bert_model_path': self._bert_model_path,
        'bert_trainable': self._bert_trainable,
        'bert_dropout': self._bert_dropout,
        'dropout': self._dropout,
        'use_batch_norm': self._use_batch_norm,
        'classify_claim': self._classify_claim,
    })
    return config

max_stars_repo_name: Xtuden-com/language

max_issues_repo_name: Xtuden-com/language

max_forks_repo_name: Xtuden-com/language


text:
import os
import numbers
import numpy as np
from tqdm import trange
import torch
from im2mesh.ptf import models
from im2mesh import encoder
from torch.nn import functional as F
from torch import distributions as dist

from im2mesh.utils.focalloss import FocalLoss

from im2mesh.common import (
    compute_iou, make_3d_grid
)
from im2mesh.training import BaseTrainer
from collections import defaultdict

IPNet_parts = ['head', 'left_foot', 'left_forearm', 'left_leg', 'left_midarm', 'left_upperarm', 'right_foot', 'right_forearm', 'right_leg', 'right_midarm', 'right_upperarm', 'torso', 'upper_left_leg', 'upper_right_leg']

class Trainer(BaseTrainer):
    ''' Trainer object for the Occupancy Network.

    Args:
        model (nn.Module): Occupancy Network model
        optimizer (optimizer): pytorch optimizer object
        skinning_weight (float): weight for skinning loss
        use_corr_loss_pred (bool): whether to use correspondence loss with predicted labels
        corr_weight (float): weight for correspondences loss
        device (device): pytorch device
        input_type (str): input type
        threshold (float): threshold value for evaluating IoU
        num_joints (int): number of joints, should be either 14 or 24
        max_operator (str): max operator used for final occupancy value of IP-Net and PTF.
            For PTF-piecewise, it can be either 'lse' or 'max'; for IP-Net and PTF it must
            be 'softmax'.
        occ_loss_type (str): occupancy loss type, currenlty only support binary cross-entropy (ce)
        occ_loss_type (str): skinning loss type, currenlty only support binary cross-entropy (ce)

    '''

    def __init__(self, model, optimizer, skinning_weight,
                 use_corr_loss_pred, corr_weight=1.0, device=None,
                 input_type='pointcloud', threshold=0.5,
                 num_joints=24, max_operator='lse', occ_loss_type='ce', skin_loss_type='ce',
                 **kwargs):
        self.model = model
        self.optimizer = optimizer
        self.device = device
        self.input_type = input_type
        self.threshold = threshold
        self.num_joints = num_joints

        self.use_corr_loss_pred = use_corr_loss_pred

        self.skinning_weight = skinning_weight
        self.corr_weight = corr_weight

        self.model_counter = defaultdict(int)

        self.max_operator = max_operator
        self.occ_loss_type = occ_loss_type
        self.skin_loss_type = skin_loss_type

        # Focal loss is experimental, not used in current version
        if self.occ_loss_type == 'fl':
            fl_gamma = kwargs['fl_gamma']
            fl_alpha = kwargs['fl_alpha']
            self.focal_loss = FocalLoss(gamma=fl_gamma, alpha=fl_alpha, device=device)

        try:
            self.model_type = self.model.module.model_type
            self.tf_type = self.model.module.tf_type
        except:
            self.model_type = self.model.model_type
            self.tf_type = self.model.tf_type

    def train_step(self, data):
        ''' Performs a training step.

        Args:
            data (dict): data dictionary
        '''
        self.model.train()
        self.optimizer.zero_grad()
        loss_dict = self.compute_loss(data)
        loss_dict['total_loss'].backward()
        self.optimizer.step()
        return {k: v.item() if isinstance(v, torch.Tensor) else v for k, v in loss_dict.items()}

    def eval_step(self, data, model_dict=None):
        ''' Performs an evaluation step.

        Args:
            data (dict): data dictionary
        '''
        self.model.eval()

        device = self.device
        threshold = self.threshold
        eval_dict = {}

        points_iou = data.get('points_iou').to(device)
        occ_iou = data.get('points_iou.occ').to(device)
        root_locs = data.get('points_iou.root_loc').to(device)
        trans = data.get('points_iou.trans').to(device)
        loc = data.get('points_iou.loc').to(device)
        # bone_transforms = data.get('points_iou.bone_transforms').to(device)
        # bone_transforms_inv = data.get('points_iou.bone_transforms_inv').to(device)    # B x num_joints x 4 x 4
        batch_size, T, D = points_iou.size()

        occ_iou = occ_iou[:, :]

        kwargs = {}
        scale = data.get('points_iou.scale').to(device)
        kwargs.update({'scale': scale.view(-1, 1, 1)}) #, 'bone_transforms_inv': bone_transforms_inv})

        with torch.no_grad():
            # Encoder inputs
            inputs = data.get('inputs', torch.empty(1, 1, 0)).to(device)
            mask = torch.ones(batch_size, T, dtype=points_iou.dtype, device=points_iou.device)

            # Decode occupancies
            out_dict = self.model(points_iou, inputs, **kwargs)
            logits = out_dict['logits']

            if len(logits.shape) == 4:
                # PTF-piecewise predictions
                logits = torch.max(logits, dim=1)[0]
                p_out = dist.Multinomial(logits=logits.transpose(1, 2))
            elif len(logits.shape) == 3:
                # IPNet/PTF predictions
                p_out = dist.Multinomial(logits=logits.transpose(1, 2))
            else:
                raise ValueError('Wrong logits shape')

        # Compute iou
        occ_iou_np = ((occ_iou >= 0.5) * mask).cpu().numpy()
        if len(logits.shape) == 3:
            # IoU for outer surface; we just want an easy-to-compute indicator for model selection
            occ_iou_hat_np = ((p_out.probs[:, :, 1:].sum(-1) >= threshold) * mask).cpu().numpy()
        else:
            raise ValueError('Wrong logits shape')

        iou = compute_iou(occ_iou_np, occ_iou_hat_np).mean()
        eval_dict['iou'] = iou

        return eval_dict

    def compute_loss(self, data):
        ''' Computes the loss.

        Args:
            data (dict): data dictionary
        '''
        device = self.device
        p = data.get('points').to(device)
        n_p_occ = p.size(1)

        p_corr = data.get('points.points_corr')

        if p_corr is not None:
            p_corr = p_corr.to(device)
            n_p_corr = p_corr.size(1)
            p = torch.cat([p, p_corr], dim=1)
        else:
            n_p_corr = 0

        batch_size = p.size(0)
        occ = data.get('points.occ').to(device)
        # bone_transforms = data.get('points.bone_transforms').to(device)    # B x num_joints x 4 x 4
        # bone_transforms_inv = data.get('points.bone_transforms_inv').to(device)    # B x num_joints x 4 x 4
        root_locs = data.get('points.root_loc').to(device)
        trans = data.get('points.trans').to(device)
        loc = data.get('points.loc').to(device)

        kwargs = {}
        scale = data.get('points.scale').to(device)
        kwargs.update({'scale': scale.view(-1, 1, 1)}) #, 'bone_transforms_inv': bone_transforms_inv})

        inputs = data.get('inputs', torch.empty(1, 1, 0)).to(device)

        z = None    # no prior

        loss_dict = {}

        if self.model_type in ['ipnet', 'deformable']:
            # Models for IP-Net and PTF
            # Get labels for skinning loss
            s_targets = data.get('points.skinning_inds').to(self.device).long()
            s_targets_ipnet = data.get('points.skinning_inds_ipnet').to(self.device).long()
            s_targets_smpl = data.get('points.skinning_inds_smpl').to(self.device).long()
            # Network computation
            out_dict = self.model(p, inputs, **kwargs)
            logits = out_dict['logits']
            p_hat = out_dict['p_hat']
            p_hat_template = out_dict.get('p_hat_template', None)   # get predicted offset in canonical space, if available

            if n_p_corr > 0:
                logits = logits[:, :, :n_p_occ]
                p_hat = p_hat[:, :, n_p_occ:]
                if p_hat_template is not None:
                    p_hat_template = p_hat_template[:, :, n_p_occ:]

            # Occupancy loss
            if self.max_operator == 'lse':
                occ_logits = torch.logsumexp(logits, dim=1)    # smoothed-max over logits, for PTF-piecewise only
            elif self.max_operator == 'max':
                occ_logits = torch.max(logits, dim=1)[0]    # max over logits, for PTF-piecewise only
            elif self.max_operator == 'softmax':
                occ_logits = logits     # do nothing, softmax was already applied inside IP-Net/PTF decoder
            else:
                raise ValueError('Max operator type {} is not supported'.format(self.max_operator))

            if self.occ_loss_type == 'ce':
                if len(occ_logits.shape) == 3:
                    # Double-layer prediction
                    loss_occ = F.cross_entropy(
                        occ_logits, occ, reduction='none').sum(-1).mean()
                else:
                    # Single-layer prediction
                    loss_occ = F.binary_cross_entropy_with_logits(
                        occ_logits, occ, reduction='none').sum(-1).mean()
            else:
                raise ValueError('Occupancy loss type {} is not supported'.format(self.occ_loss_type))

            # Compute IoUs for this batch
            if len(occ_logits.shape) == 3:
                # Double-layer prediction
                occ_hat_np = ((F.softmax(occ_logits, dim=1)[:, 1:, :].sum(1)) > 0.5).detach().cpu().numpy()
            else:
                # Single-layer prediction
                occ_hat_np = (torch.sigmoid(occ_logits) > 0.5).detach().cpu().numpy()

            occ_np = (occ > 0.5).detach().cpu().numpy()
            ious = compute_iou(occ_np, occ_hat_np).flatten()
            loss_dict['iou'] = ious

            # Skinning loss with PTFs predictions
            if self.max_operator == 'softmax':
                # For IP-Net/PTF
                out_cls = out_dict['out_cls']
                if n_p_corr > 0:
                    out_cls = out_cls[:, :, n_p_occ:]

                if self.skin_loss_type == 'ce':
                    loss_skin = F.cross_entropy(out_cls, s_targets, reduction='none').sum(-1).mean()
                else:
                    raise ValueError('Skinning loss type {} is not supported'.format(self.skin_loss_type))
            elif self.max_operator in ['lse']:
                # For PTF-piecewise
                if len(logits.shape) == 4:
                    # Double layer prediction
                    skin_logits = torch.logsumexp(logits, dim=2)    # smoothed-max over class (i.e. inside, in-between, outside) dimension
                else:
                    # Single layer prediction
                    skin_logits = logits

                if self.skin_loss_type == 'ce':
                    loss_skin = F.cross_entropy(skin_logits, s_targets, reduction='none').sum(-1).mean()
                else:
                    raise ValueError('Skinning loss type {} is not supported'.format(self.skin_loss_type))
            else:
                raise ValueError('Max operator type {} is not supported'.format(self.max_operator))

            if self.use_corr_loss_pred: # and self.tf_type is not None:
                parts_softmax = out_dict.get('parts_softmax', None)
                # Compute p_hat
                if parts_softmax is not None:
                    # For IP-Net/PTF, who has a separate branch to predict part probabilities
                    if n_p_corr > 0:
                        parts_softmax = parts_softmax[:, :, n_p_occ:]

                    p_hat = p_hat.view(batch_size, -1, 3, n_p_corr if n_p_corr > 0 else n_p_occ)
                    p_hat = p_hat * parts_softmax.view(batch_size, -1, 1, n_p_corr if n_p_corr > 0 else n_p_occ)
                    if p_hat_template is not None:
                        p_hat_template = p_hat_template.view(batch_size, -1, 3, n_p_corr if n_p_corr > 0 else n_p_occ)
                        p_hat_template = p_hat_template * parts_softmax.view(batch_size, -1, 1, n_p_corr if n_p_corr > 0 else n_p_occ)
                else:
                    # For PTF-piecewise, who does not predict part probabilities directly
                    p_hat = p_hat.view(batch_size, -1, 3, n_p_corr if n_p_corr > 0 else n_p_occ)
                    p_hat = p_hat * F.softmax(skin_logits, dim=1).view(batch_size, -1, 1, n_p_corr if n_p_corr > 0 else n_p_occ)
                    if p_hat_template is not None:
                        p_hat_template = p_hat_template.view(batch_size, -1, 3, n_p_corr if n_p_corr > 0 else n_p_occ)
                        p_hat_template = p_hat_template * F.softmax(skin_logits, dim=1).view(batch_size, -1, 1, n_p_corr if n_p_corr > 0 else n_p_occ)

                p_hat = p_hat.sum(1).transpose(1, 2)
                if p_hat_template is not None:
                    p_hat_template = p_hat_template.sum(1).transpose(1, 2)

            # Correspondence loss with predicted skinning inds
            loss_corr_dict = {}
            if self.use_corr_loss_pred: # and self.tf_type is not None:
                p_a_pose = data.get('points.pts_a_pose').to(device)
                p_template = data.get('points.pts_template')

                if p_template is not None:
                    p_template = p_template.to(device)

                loss_corr_pred_all = torch.norm(p_hat - p_a_pose, 2, dim=-1)
                loss_corr_pred = loss_corr_pred_all.sum(-1).mean()
                if p_hat_template is not None:
                    loss_template_pred = torch.norm(p_hat_template - p_template, 2, dim=-1).sum(-1).mean()
                else:
                    loss_template_pred = torch.zeros(1, device=self.device)
            else:
                loss_corr_pred = torch.zeros(1, device=self.device)
                loss_template_pred = torch.zeros(1, device=self.device)

            loss_dict['occ_loss'] = loss_occ
            loss_dict['skinning_loss'] = loss_skin
            loss_dict['corr_loss'] = loss_corr_pred
            loss_dict['template_loss'] = loss_template_pred
            loss_dict.update(loss_corr_dict)

            # Total weighted sum
            loss = loss_occ + self.skinning_weight * loss_skin + self.corr_weight * (loss_corr_pred + loss_template_pred)
            loss_dict['total_loss'] = loss
        else:
            raise ValueError('Supported model type: ipnet, deformable, got {}'.format(self.model_type))

        return loss_dict

max_stars_repo_name: taconite/PTF

max_issues_repo_name: taconite/PTF

max_forks_repo_name: taconite/PTF


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

from config import batch_size, num_support, gamma, V_max, V_min

class Distributional_C51(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(Distributional_C51, self).__init__()
        self.num_inputs = num_inputs
        self.num_outputs = num_outputs

        self.dz = float(V_max - V_min) / (num_support - 1)
        self.z = torch.Tensor([V_min + i * self.dz for i in range(num_support)])

        self.fc1 = nn.Linear(num_inputs, 128)
        self.fc2 = nn.Linear(128, num_outputs * num_support)

        for m in self.modules():
            if isinstance(m, nn.Linear):
                nn.init.xavier_uniform(m.weight)

    def forward(self, input):
        x = F.relu(self.fc1(input))
        x = self.fc2(x)
        z = x.view(-1, self.num_outputs, num_support)
        p = nn.Softmax(dim=2)(z)
        return p

    def get_action(self, input):
        p = self.forward(input)
        p = p.squeeze(0)
        z_space = self.z.repeat(self.num_outputs, 1)
        Q = torch.sum(p * z_space, dim=1)
        action = torch.argmax(Q)
        return action.item()

    @classmethod
    def get_m(cls, _rewards, _masks, _prob_next_states_action):
        rewards = _rewards.numpy()
        masks = _masks.numpy()
        prob_next_states_action = _prob_next_states_action.detach().numpy()
        m_prob = np.zeros([batch_size, num_support], dtype=np.float32)

        dz = float(V_max - V_min) / (num_support - 1)
        batch_id = range(batch_size)
        for j in range(num_support):
            Tz = np.clip(rewards + masks * gamma * (V_min + j * dz), V_min, V_max)
            bj = (Tz - V_min) / dz

            lj = np.floor(bj).astype(np.int64)
            uj = np.ceil(bj).astype(np.int64)

            blj = (bj - lj)
            buj = (uj - bj)

            m_prob[batch_id, lj[batch_id]] += ((1 - masks) + masks * (prob_next_states_action[batch_id, j])) * buj[batch_id]
            m_prob[batch_id, uj[batch_id]] += ((1 - masks) + masks * (prob_next_states_action[batch_id, j])) * blj[batch_id]

        return m_prob

    @classmethod
    def train_model(cls, online_net, target_net, optimizer, batch):
        states = torch.stack(batch.state)
        next_states = torch.stack(batch.next_state)
        actions = torch.Tensor(batch.action).int()
        rewards = torch.Tensor(batch.reward)
        masks = torch.Tensor(batch.mask)

        z_space = online_net.z.repeat(batch_size, online_net.num_outputs, 1)
        prob_next_states = target_net(next_states)
        Q_next_state = torch.sum(prob_next_states * z_space, 2)
        next_actions = torch.argmax(Q_next_state, 1)
        prob_next_states_action = torch.stack([prob_next_states[i, action, :] for i, action in enumerate(next_actions)])

        m_prob = cls.get_m(rewards, masks, prob_next_states_action)
        m_prob = torch.tensor(m_prob)

        m_prob = (m_prob / torch.sum(m_prob, dim=1, keepdim=True)).detach()
        expand_dim_action = torch.unsqueeze(actions, -1)
        p = torch.sum(online_net(states) * expand_dim_action.float(), dim=1)
        loss = -torch.sum(m_prob * torch.log(p + 1e-20), 1)
        loss = loss.mean()

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        return loss


text:
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

from typing import Sequence
import numpy as np
import numpy.lib.mixins

from . import ir_utils
from . import tracing

import jax.core

from iree.compiler import (
    ir,)

_BASE_HANDLED_FUNCTIONS = {}

def _base_implements(np_function):
  """Decorator that registers a base class implementation."""

  def decorator(func):
    _BASE_HANDLED_FUNCTIONS[np_function] = func
    return func

  return decorator

class TracedArrayBase(numpy.lib.mixins.NDArrayOperatorsMixin):
  """Base class for tracked arrays."""

  def __init__(self, aval: jax.core.AbstractValue):
    self.aval = aval

  def __array_function__(self, func, types, args, kwargs):
    if func not in _BASE_HANDLED_FUNCTIONS:
      return NotImplemented
    return _BASE_HANDLED_FUNCTIONS[func](*args, **kwargs)

  def __array__(self, dtype=None):
    if dtype is not None:
      assert dtype is self.aval.dtype, "Traced data type cast not yet implemented"
    # assert dtype is None
    return self

@_base_implements(np.shape)
def _(arr: TracedArrayBase):
  return arr.aval.shape

@_base_implements(np.result_type)
def _(arr: TracedArrayBase):
  return arr.aval.dtype

class ExportedGlobalArray(TracedArrayBase, tracing.Intrinsic):
  """Represents an exported global exposed as one array at the Python level."""

  def __init__(self, aval: jax.core.ShapedArray, symbol_name: str,
               ir_type: ir.Type):
    super().__init__(aval)
    self.symbol_name = symbol_name
    self.ir_type = ir_type

  def __repr__(self):
    return f"ExportedGlobalArray(@{self.symbol_name} : {self.ir_type})"

  def resolve_ir_values(
      self, func_trace: tracing.FunctionIrTrace) -> Sequence[ir.Value]:
    return (ir_utils.create_global_load_op(self.symbol_name, self.ir_type),)

class IrValueArray(TracedArrayBase, tracing.Intrinsic):
  """Represents an array that corresponds to an IR value."""

  def __init__(self, aval: jax.core.ShapedArray, ir_value: ir.Value):
    super().__init__(aval)
    self.ir_value = ir_value

  def __repr__(self):
    return f"IrValueArray(@{self.ir_value})"

  def resolve_ir_values(
      self, func_trace: tracing.FunctionIrTrace) -> Sequence[ir.Value]:
    return (self.ir_value,)

max_stars_repo_name: powderluv/iree-samples

max_issues_repo_name: powderluv/iree-samples

max_forks_repo_name: powderluv/iree-samples

reason: import numpy,import jax


text:
import numpy as np
from numpy import random, convolve
from numba import prange, njit, int64, float64
# import time
import matplotlib
# import matplotlib.pyplot as plt
# from pyoteapp.autocorrtools import *
matplotlib.use('Qt5Agg')

@njit(float64[:](int64, float64), cache=True)
def noise_gen_jit(num_points: int, noise_sigma: float) -> np.ndarray:
    out = np.empty(num_points)
    for i in range(num_points):
        out[i] = random.normal(0.0, noise_sigma)
    return out

@njit(cache=True)
def simple_convolve(x, c):
    nx = len(x)
    nc = len(c)
    nout = nx - nc + 1
    out = np.zeros(nout)
    for i in range(nout):
        for j in range(nc):
            out[i] += x[i + j] * c[j]
    return out

@njit(cache=True)
def simple_convolve2(x, c):
    nx = len(x)
    nc = len(c)
    nout = nx - nc + 1
    out = np.zeros(nout)
    i = 0
    while i < nout:
        j = 0
        while j < nc:
            out[i] += x[i + j] * c[j]
            j += 1
        i += 1
    return out

def noise_gen_numpy(num_points: int, noise_sigma: float) -> np.ndarray:
    return random.normal(size=num_points, scale=noise_sigma)

# @jit(float64[:](int64, float64, float64[:]))
def simulated_observation(observation_size: int, noise_sigma: float, corr_array: np.ndarray) -> np.ndarray:
    """Returns a numpy array of floats representing an observation that contains only correlated noise.

    observation_size: number of points in the simulated observation
    noise_sigma:      standard deviation of observation noise
    corr_array:      correlation array needed to produce the correlated noise
    """
    return convolve(noise_gen_jit(observation_size, noise_sigma), corr_array, 'same')
    # noise_array = noise_gen_jit(observation_size+len(corr_array), noise_sigma)
    # out = np.empty(observation_size)
    # for i in prange(observation_size):
    #     out[i] = np.dot(corr_array, noise_array[i:i+len(corr_array)])
    #
    # return out

# noinspection PyPep8Naming
@njit(cache=True)
def max_drop_in_simulated_observation(
        event_duration: int,
        observation_size: int,
        noise_sigma: float,
        corr_array: np.ndarray) -> float:
    """Returns the maximum drop from a simulated observation.

    An observation consisting only of correlated gaussian noise is
    created and exhaustively searched for an 'event' of a specified size.
    The biggest 'drop' (B - A) found is returned.  This drop will always
    be positive (>= 0.0)

    event_duration:   size/duration of event being simulated
    observation_size: number of points in the simulated observation
    noise_sigma:      standard deviation of observation noise
    corr_array:       correlation array needed to produce the correlated noise
    """

    # obs = simulated_observation(observation_size, noise_sigma, corr_array)
    obs = noise_gen_jit(observation_size, noise_sigma)
    obs = simple_convolve(obs, corr_array)

    obs_size = len(obs)

    numA = event_duration
    numB = obs_size - numA

    # Start with the trial 'event' position at the extreme left of the observation
    cumA = np.sum(obs[0:numA])
    cumB = np.sum(obs[numA:])

    A = cumA / numA
    B = cumB / numB

    drop = B - A

    # Use negative values returned by this routine as an indicator that no positive drop could be found
    # best_drop_so_far = drop if drop >= 0.0 else 0.0
    best_drop_so_far = -1.0

    # Calculate new values for B and A iteratively and conditionally update best_drop_so_far
    for i in range(1, numB):
        cumA -= obs[i]
        cumB += obs[i]

        cumA += obs[i + event_duration]
        cumB -= obs[i + event_duration]

        A = cumA / numA
        B = cumB / numB

        drop = B - A
        if drop > best_drop_so_far:
            best_drop_so_far = drop

    return best_drop_so_far

@njit(parallel=True)
def compute_drops(event_duration: int,
                  observation_size: int,
                  noise_sigma: float,
                  corr_array: np.ndarray,
                  num_trials: int) -> np.ndarray:
    """Returns the maximum drops from num_trials simulated observations.

    event_duration:   size/duration of event being simulated
    observation_size: number of points in the simulated observation
    noise_sigma:      standard deviation of observation noise
    corr_array:       correlation array needed to produce the correlated noise
    num_trials:       number of simulated observations
    """

    drops: np.ndarray = np.empty(num_trials)

    for i in prange(num_trials):
        drops[i] = max_drop_in_simulated_observation(
            event_duration=event_duration, observation_size=observation_size,
            noise_sigma=noise_sigma, corr_array=corr_array)

    return drops

# def excercise():
#     noise_gen_jit(1, 1.0)
#     noise_gen_jit(1, 1.0)
#
#     start_time = time.time()
#     _ = noise_gen_jit(4_000, 1.0)
#     stop_time = time.time()
#     print(f'noise_gen_jit time: {stop_time - start_time:.6f} seconds')
#
#     start_time = time.time()
#     _ = noise_gen_numpy(4_000, 1.0)
#     stop_time = time.time()
#     print(f'noise_gen_numpy time: {stop_time - start_time:.6f} seconds')
#
#     # Warm-up calls to allow Numba compiler to do its work
#     print(f'\nDoing Numba warm-up calls...')
#     simulated_observation(100, 1.0, np.ones(10))
#     simulated_observation(100, 1.0, np.ones(10))
#
#     max_drop_in_simulated_observation(10, 100, 1.0, np.ones(10))
#     max_drop_in_simulated_observation(10, 100, 1.0, np.ones(10))
#
#     compute_drops(10, 100, 1.0, np.ones(10), 1)
#     compute_drops(10, 100, 1.0, np.ones(10), 1)
#     print(f'...finished warm-up calls.\n')
#     # End warm-up calls
#
#     dur = 100
#     n_obs = 4000
#     sigma = 2.0
#     # corr = np.ones(10)
#     corr = np.array([0.11653067, 0.29474461, 0.40390159, 0.85814318])
#     n_trials = 10000
#
#     start_time = time.time()
#     drop_array = compute_drops(event_duration=dur, observation_size=n_obs,
#                                noise_sigma=sigma, corr_array=corr, num_trials=n_trials)
#     stop_time = time.time()
#     print(f'{n_trials} drops computed in {stop_time - start_time:.6f} seconds')
#
#     # %%
#     plt.plot(np.sort(drop_array))
#     plt.show()
#
#     # %%
#     hist_values, bin_edges = np.histogram(drop_array, 50)
#     # print(hist_values)
#     # print(bin_edges)
#     plt.hist(drop_array, bins=50)
#     max_drop = np.max(drop_array)
#     plt.vlines(max_drop, 0, max(hist_values), colors='r')
#     plt.show()
#
#     noise_gen = CorrelatedNoiseGenerator(acfcoeffs=[1.0, 0.5, 0.3, 0.1])
#     test_noise = noise_gen.corr_normal(num_values=100000)
#     print(f'sigma: {np.std(test_noise)}')
#     print(autocorr(test_noise, lastlag=6))
#     print(noise_gen.final_coeffs)
#

# if __name__ == "__main__":
#     excercise()

reason: import numpy,from numpy,from numba


text:
from scipy import signal
import numpy as np
import pandas as pd

# TODO: rename s
# TODO: remove dependence on dataframe
def preprocess(s, resample_factor=1.0, freq_range=[0.01, 0.1]):
    """Pre process

    Args:
        s: signal. (nchannel x nsamples)
        resample_factor: resampling factor

    Returns:
        np.array: (nchannel x nsamples)
    """
    # Resampling
    if abs(resample_factor - 1) > 0.01:
        s = signal.resample(s,
                            num=int(np.rint(np.shape(s)[1] * resample_factor)),
                            axis=1)

    # read token and convert to data frame
    df0 = pd.DataFrame(np.array(s).transpose(), columns=['ch' + str(i) for i in range(0, len(s))])  # TODO: use l as label

    # Drop Channels
    df0 = df0.iloc[:, 0:8]

    # Remove DC offset
    df0 = df0.apply(lambda x: x - np.mean(x))

    # Filter with low and high pass
    filter = signal.firwin(400, freq_range, pass_zero=False)
    df0 = df0.apply(lambda x: np.real(signal.convolve(x.to_numpy(), filter, mode='same')))

    return df0.to_numpy().transpose()


text:
# coding=utf-8
# Copyright 2021 The Balloon Learning Environment Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utilities for modeling balloon temperature."""

from balloon_learning_environment.env.balloon import solar
import numpy as np

# View factors are used in thermal modeling and refer to the percentage of the
# balloon involved in the thermal heat transfer (normalized effective area).
# Solar view factor is ~25% since the sun is really far away and can be treated
# as a directional point source acting on a sphere. Earth view factor is ~50%
# since the bottom half of the balloon can see the Earth (we use a slightly
# lower value that assumes the earth is a really large sphere rather than a
# flat plane).
_SOLAR_VIEW_FACTOR = 0.25
_EARTH_VIEW_FACTOR = 0.4605

# PE01 film params.
# - Reflectivity is how much light bounces off the film. It is assumed
#   to be the same for all wavelengths (solar and IR).
# - Solar absorptivity is the film absorption in the solar spectrum.
# - Absorptivity in the IR spectrum is a function of IR black-body
#   temperature. This is modeled as a linear function around a trim
#   condition:
#     A_ir = A_ir_base + A_ir_d_dT * (T_object - T_ref)
# - Specific heat of film.
_PE01_REFLECTIVITY = 0.0291
_PE01_ABSORPTIVITY_SOLAR = 0.01435
_PE01_ABSORPTIVITY_IR_BASE = 0.04587
_PE01_ABSORPTIVITY_IR_D_TEMPERATURE = 0.000232  # [1/K]
_PE01_ABSORPTIVITY_IR_REF_TEMPERATURE = 210  # [K]
_PE01_FILM_SPECIFIC_HEAT = 1500  # [J/(kg.K)]

_STEFAN_BOLTZMAN = 0.000000056704  # [W/(m^2.K^4)]
_UNIVERSAL_GAS_CONSTANT = 8.3144621  # [J/(mol.K)]
_DRY_AIR_MOLAR_MASS = 0.028964922481160  # Dry Air. [kg/mol]

def black_body_temperature_to_flux(temperature_k: float) -> float:
  """Compute corresponding flux given black body temperature.

  Args:
    temperature_k: Black body's temperature [K].

  Returns:
    flux: Black body's flux [W/m^2].
  """
  return _STEFAN_BOLTZMAN * temperature_k**4

def black_body_flux_to_temperature(flux: float) -> float:
  """Compute corresponding temperature given black body flux.

  Args:
    flux: Black body's flux [W/m^2].

  Returns:
    temperature_k: Black body's temperature [K].
  """
  return (flux / _STEFAN_BOLTZMAN)**0.25

def absorptivity_ir(object_temperature_k: float) -> float:
  """Compute balloon IR absorptivity/emissivity given black body temperature.

  This function computes IR absorptivity/emissivity given the radiative object's
  black body temperature. We assume PE01 balloon film and use a linear model.

  Args:
    object_temperature_k: Object's black body temperature [K].

  Returns:
    absorptivity: Absorptivity factor in IR spectrum.
  """
  return (_PE01_ABSORPTIVITY_IR_BASE + _PE01_ABSORPTIVITY_IR_D_TEMPERATURE *
          (object_temperature_k - _PE01_ABSORPTIVITY_IR_REF_TEMPERATURE))

def total_absorptivity(absorptivity: float, reflectivity: float) -> float:
  """Compute total internal balloon absorptivity/emissivity factor.

  This function computes total absorptivity or total emissivity. For the
  absorptivity process, the dynamics are as follows:

  --> From the radiation hitting the surface, R is reflected outward, A is
      absorbed, and the rest, T, is "transmitted" through the surface,
      where T = 1 - R - A.
  --> From the amount transmitted through the surface, T, a portion TA is
      absorbed, a portion TR is reflected back into the sphere, and the
      rest (TT) leaves the sphere.
  --> From the amount reflected into the sphere, TR, a portion TRA is
      absorbed, a portion TRR is reflected back into the sphere, and the
      rest (TRT) is lost.
  --> Continuing this process to infinity and adding up all the aborbed
      amounts gives the following:

         A_total = A + TA + TRA + TR^2 A + TR^3 A + ...
                 = A + TA (1 + R + R^2 + R^3 + ...)
                 = A (1 + T / (1 - R))

  Similarly, we can analyze the emissivity process:

  --> From the radiation emitted by the surface, A is emitted outwards where
      E = A = emissivity, and A is emitted inwards (double radiation).
  --> From the inwards amount, AA is re-absorbed, AR is internally reflected
      and AT is emitted through the film.
  --> From the internally reflected amount, ARA is re-absorbed, ARR is
      internally reflected, and ART is emitted through the film.
  --> Continuing this process to infinity and adding up all the outwards
      emissions gives the following:

         E_total = A + AT + ART + AR^2T + AR^3 T + ...
                 = A + AT (1 + R + R^2 + R^3 + ...)
                 = A (1 + T / (1 - R))

  Noting that E_total and A_total are equivalent, we can use this function
  for both incoming and outgoing emissions.

  Args:
    absorptivity: Balloon film's absorptivity/emissivity.
    reflectivity: Balloon film's reflectivity.

  Returns:
    total_absorptivity_factor: Factor of radiation absorbed/emitted by balloon.
  """
  transmisivity = 1.0 - absorptivity - reflectivity
  total_absorptivity_factor = absorptivity * (1.0 + transmisivity /
                                              (1.0 - reflectivity))
  if total_absorptivity_factor < 0.0 or total_absorptivity_factor > 1.0:
    raise ValueError(
        'total_absorptivity: '
        'Computed total absorptivity factor out of expected range [0, 1].')

  return total_absorptivity_factor

def convective_heat_air_factor(balloon_radius: float,
                               balloon_temperature_k: float,
                               ambient_temperature_k: float,
                               pressure_altitude_pa: float) -> float:
  """Convective heat air factor."""
  viscosity = 1.458e-6 * (ambient_temperature_k**1.5) / (
      ambient_temperature_k + 110.4)
  conductivity = 0.0241 * ((ambient_temperature_k / 273.15)**0.9)
  prandtl = 0.804 - 3.25e-4 * ambient_temperature_k
  air_density = (
      pressure_altitude_pa * _DRY_AIR_MOLAR_MASS /
      (_UNIVERSAL_GAS_CONSTANT * ambient_temperature_k))

  grashof = (9.80665 * (air_density**2) * ((2 * balloon_radius)**3) /
             (ambient_temperature_k *
              (viscosity**2))) * np.abs(ambient_temperature_k -
                                        balloon_temperature_k)
  rayleigh = prandtl * grashof
  nusselt = (2 + 0.457 * (rayleigh**0.25) +
             ((1 + 2.69e-8 * rayleigh)**(1.0 / 12.0)))
  k_heat_transfer = nusselt * conductivity / (2 * balloon_radius)

  return k_heat_transfer * (ambient_temperature_k - balloon_temperature_k)

def d_balloon_temperature_dt(balloon_volume: float,
                             balloon_mass: float,
                             balloon_temperature_k: float,
                             ambient_temperature_k: float,
                             pressure_altitude_pa: float,
                             solar_elevation_deg: float,
                             solar_flux: float,
                             earth_flux: float) -> float:
  """Compute d_balloon_temperature / dt. Assumes PE01 film.

  Args:
    balloon_volume: Balloon volume [m^3].
    balloon_mass: Balloon envelope mass [kg].
    balloon_temperature_k: Balloon temperature [K].
    ambient_temperature_k: Ambient temperature around the balloon [K].
    pressure_altitude_pa: Balloon's pressure altitude [Pa].
    solar_elevation_deg: Solar elevation [deg].
    solar_flux: Solar flux [W/m^2].
    earth_flux: Earth radiation experienced at balloon [W/m^2].

  Returns:
    d_balloon_temperature / dt: Derivative of balloon temperature [K/s].
  """

  # Compute balloon radius and surface area. Assumes spherical balloon.
  balloon_radius = (3 * balloon_volume / (4 * np.pi))**(1 / 3)
  balloon_area = 4 * np.pi * balloon_radius * balloon_radius

  # Compute atmospheric attenuation.
  atm_attenuation = solar.solar_atmospheric_attenuation(solar_elevation_deg,
                                                        pressure_altitude_pa)
  # Compute solar radiative heat.
  q_solar = (
      solar_flux * atm_attenuation * _SOLAR_VIEW_FACTOR * balloon_area *
      total_absorptivity(_PE01_ABSORPTIVITY_SOLAR, _PE01_REFLECTIVITY))

  # Compute earth radiative heat.
  q_earth = (
      earth_flux * _EARTH_VIEW_FACTOR * balloon_area * total_absorptivity(
          absorptivity_ir(black_body_flux_to_temperature(earth_flux)),
          _PE01_REFLECTIVITY))

  # Compute balloon emissions given balloon temperature.
  q_emitted = (
      black_body_temperature_to_flux(balloon_temperature_k) * balloon_area *
      total_absorptivity(absorptivity_ir(balloon_temperature_k),
                         _PE01_REFLECTIVITY))

  # Compute external convective heat given balloon temperature.
  q_convective = balloon_area * convective_heat_air_factor(
      balloon_radius, balloon_temperature_k, ambient_temperature_k,
      pressure_altitude_pa)

  # Compute derivative of balloon temperature give total heat loads and mass.
  return (q_solar + q_earth + q_convective - q_emitted) / (
      _PE01_FILM_SPECIFIC_HEAT * balloon_mass)

max_stars_repo_name: taodav/balloon-learning-environment

max_issues_repo_name: taodav/balloon-learning-environment

max_forks_repo_name: taodav/balloon-learning-environment


text:
import numpy as np
import sys
import random
import operator
import csv
import time
import gzip
import math

from loading_data import LoadData

from timeSVDpp import timeSVDpp
from TVBPR import TVBPR
from timeSVD_VC import timeSVD_VC

if len(sys.argv) != 7:
	  print "Parameters: "
	  print "1. model name (e.g. timeSVDpp/ TVBPR/ timeSVD_VC)"
	  print "2. number of iterations"
	  print "3. number of epochs/bins"
	  print "4. number of non-visual factors"
	  print "5. number of visual factors"
	  print "6. dataset (e.g. 390_actions, 780_actions, 1560_actions, 2340_actions, 4099_actions)"
	  sys.exit()

model = sys.argv[1]
nIterations = int(sys.argv[2])
nEpochs = int(sys.argv[3])
nFactors = int(sys.argv[4])
nVisualFactors = int(sys.argv[5])
dataset = sys.argv[6]

if dataset == '390_actions':
	path = "./datasets/trainingSet_78_users_390_actions.csv"
elif dataset == '780_actions':
	path = "./datasets/trainingSet_78_users_780_actions.csv"
elif dataset == '1560_actions':
	path = "./datasets/trainingSet_78_users_1560_actions.csv"
elif dataset == '2340_actions':
	path = "./datasets/trainingSet_78_users_2340_actions.csv"
elif dataset == '4099_actions':
	path = "./datasets/trainingSet_78_users_4099_actions.csv"
else:
	print 'The dataset does not exist!'

# running timeSVD++
if model == 'timeSVDpp':
	#loading the data
	ld = LoadData(dataset)
	userItems, itemsAsinList, itemImageFeatures, itemAsinToInt, test_per_user, val_per_user, min_timestamp, max_timestamp, nr_days, posEvents, timestamps = ld.run_load_data(path)

	nUsers = len(userItems)
	nItems = len(itemsAsinList)
	nBins = nEpochs

	#running the model
	timesvdpp = timeSVDpp(nIterations, nFactors, nUsers, nItems, nBins, userItems, itemsAsinList, itemAsinToInt, min_timestamp, max_timestamp,  test_per_user, val_per_user, timestamps)

# running TVBPR
elif model == 'TVBPR':
	# loading the data
	ld = LoadData(dataset)
	userItems, itemsAsinList, itemImageFeatures, itemAsinToInt, test_per_user, val_per_user, min_timestamp, max_timestamp, nr_days, posEvents, timestamps = ld.run_load_data(path)

	nUsers = len(userItems)
	nItems = len(itemsAsinList)
	nBins = nEpochs
	nImageFeatures = 4096

	# running the model
	tvbpr = TVBPR(nIterations, nFactors, nVisualFactors, nUsers, nItems, nImageFeatures, nEpochs, userItems, itemsAsinList, posEvents, max_timestamp, itemImageFeatures, itemAsinToInt, min_timestamp, test_per_user, val_per_user, nr_days)

# running timeSVD_VC
elif model == "timeSVD_VC":
	# loading the data
	ld = LoadData(dataset)
	userItems, itemsAsinList, itemImageFeatures, itemAsinToInt, test_per_user, val_per_user, min_timestamp, max_timestamp, nr_days, posEvents, timestamps = ld.run_load_data(path)

	nUsers = len(userItems)
	nItems = len(itemsAsinList)
	nBins = nEpochs
	nImageFeatures = 4096

	# running the model
	timesvd_vc = timeSVD_VC(nIterations, nFactors, nVisualFactors, nUsers, nItems, nImageFeatures, nEpochs, userItems, itemsAsinList, posEvents, max_timestamp, itemImageFeatures, itemAsinToInt, min_timestamp, test_per_user, val_per_user, nr_days)

else:
	print 'Choose the right model!'