text:
import pdb

import numpy as np
import math
import time

import chainer
import chainer.functions as F
import chainer.links as L
from chainer import cuda

from util import gaussian_logp0
from util import bernoulli_logp

class VAE(chainer.Chain):
    def __init__(self, dim_in, dim_hidden, dim_latent, num_layers, num_trans, temperature, num_zsamples=1):
        super(VAE, self).__init__()

        # initialise first encoder and decoder hidden layer separately because
        # the input and output dims differ from the other hidden layers
        self.qlin0 = L.Linear(dim_in, dim_hidden)
        self.plin0 = L.Linear(dim_latent, dim_hidden)
        self._children.append('qlin0')
        self._children.append('plin0')

        for i in range(num_layers-1):
            # encoder
            layer_name = 'qlin' + str(i+1)
            setattr(self, layer_name, L.Linear(2*dim_hidden, dim_hidden))
            self._children.append(layer_name)

            # decoder
            layer_name = 'plin' + str(i+1)
            setattr(self, layer_name, L.Linear(2*dim_hidden, dim_hidden))
            self._children.append(layer_name)

        # initialise the encoder and decoder output layer separately because
        # the input and output dims differ from the other hidden layers
        self.qlin_mu = L.Linear(2*dim_hidden, dim_latent)
        self.qlin_ln_var = L.Linear(2*dim_hidden, dim_latent)
        self.plin_ber_prob = L.Linear(2*dim_hidden, dim_in)
        self._children.append('qlin_mu')
        self._children.append('qlin_ln_var')
        self._children.append('plin_ber_prob')

        # flow
        for i in range(num_trans):
            layer_name = 'flow_w_' + str(i) # weights
            setattr(self, layer_name, L.Scale(axis=1, W_shape=(dim_latent), bias_term=False))
            self._children.append(layer_name)

            layer_name = 'flow_b_' + str(i) # bias
            setattr(self, layer_name, L.Bias(axis=0, shape=(1)))
            self._children.append(layer_name)

            layer_name = 'flow_u_' + str(i) # scaling factor u
            setattr(self, layer_name, L.Scale(axis=1, W_shape=(dim_latent), bias_term=False))
            self._children.append(layer_name)

        self.num_layers = num_layers
        self.num_trans = num_trans
        self.temperature = temperature
        self.num_zsamples = num_zsamples
        self.epochs_seen = 0

    def encode(self, x):
       h = F.crelu(self.qlin0(x))

       for i in range(self.num_layers-1):
           layer_name = 'qlin' + str(i+1)
           h = F.crelu(self[layer_name](h))

       self.qmu = self.qlin_mu(h)
       self.qln_var = self.qlin_ln_var(h)

       return self.qmu, self.qln_var

    def decode(self, z):
        h = F.crelu(self.plin0(z))

        for i in range(self.num_layers-1):
            layer_name = 'plin' + str(i+1)
            h = F.crelu(self[layer_name](h))

        self.p_ber_prob_logit = self.plin_ber_prob(h)

        return self.p_ber_prob_logit

    def planar_flows(self,z):
        self.z_trans = []
        self.z_trans.append(z)
        self.phi = []

        for i in range(self.num_trans):
            flow_w_name = 'flow_w_' + str(i)
            flow_b_name = 'flow_b_' + str(i)
            flow_u_name = 'flow_u_' + str(i)

            h = self[flow_w_name](z)
            h = F.sum(h,axis=(1))
            h = self[flow_b_name](h)
            h = F.tanh(h)
            h_tanh = h

            dim_latent = z.shape[1]
            h = F.transpose(F.tile(h, (dim_latent,1)))
            h = self[flow_u_name](h)

            z += h

            self.z_trans.append(z)

            # Calculate and store the phi term
            h_tanh_derivative = 1-(h_tanh*h_tanh)
            h_tanh_derivative = F.transpose(F.tile(h_tanh_derivative, (dim_latent,1)))

            phi = self[flow_w_name](h_tanh_derivative) # Equation (11)
            self.phi.append(phi)

        return z

    def __call__(self, x):
        # Compute q(z|x)
        encoding_time = time.time()
        qmu, qln_var = self.encode(x)
        encoding_time = float(time.time() - encoding_time)

        decoding_time_average = 0.

        self.kl = 0
        self.logp = 0

        current_temperature = min(self.temperature['value'],1.0)
        self.temperature['value'] += self.temperature['increment']

        for j in xrange(self.num_zsamples):
            # Sample z ~ q(z_0|x)
            z_0 = F.gaussian(self.qmu, self.qln_var)

            # Perform planar flow mappings, Equation (10)
            decoding_time = time.time()
            z_K = self.planar_flows(z_0)

            # Obtain parameters for p(x|z_K)
            p_ber_prob_logit =  self.decode(z_K)
            decoding_time = time.time() - decoding_time
            decoding_time_average += decoding_time

            # Compute log q(z_0)
            q_prior_log = current_temperature*gaussian_logp0(z_0)

            # Compute log p(x|z_K)
            decoder_log = bernoulli_logp(x, p_ber_prob_logit)
            # Compute log p(z_K)
            p_prior_log = current_temperature*gaussian_logp0(z_K)

            # Compute log p(x,z_K) which is log p(x|z_K) + log p(z_K)
            joint_log = decoder_log + p_prior_log

            # Compute second term of log q(z_K)
            q_K_log = 0
            for i in range(self.num_trans):
                flow_u_name = 'flow_u_' + str(i)
                lodget_jacobian = F.sum(self[flow_u_name](self.phi[i]), axis=1)
                q_K_log += F.log(1 + lodget_jacobian)
            q_K_log *= current_temperature

            # For recording purposes only
            self.logp += decoder_log
            self.kl += -(q_prior_log - p_prior_log - q_K_log)

        decoding_time_average /= self.num_zsamples
        # pdb.set_trace()
        self.obj_batch = ((q_prior_log -joint_log) - q_K_log)
        self.obj_batch /= self.num_zsamples
        batch_size = self.obj_batch.shape[0]

        self.obj = F.sum(self.obj_batch)/batch_size

        self.kl /= self.num_zsamples
        self.logp /= self.num_zsamples

        self.timing_info = np.array([encoding_time,decoding_time])

        return self.obj

max_stars_repo_name: ashwindcruz/dgm

max_issues_repo_name: ashwindcruz/dgm

max_forks_repo_name: ashwindcruz/dgm


text:
import netCDF4
import numpy
import argparse
import sys
import time
import functools
from scipy.integrate import odeint
from ugrid_reader import UgridReader
import re

parser = argparse.ArgumentParser(description='Generate upstream grid')
parser.add_argument('-u', dest='velocity_x', default='sin(4*pi*(2*x-y)/180.)',
                    help='Specify the contravariant velocity component u (deg/time) along longitudes as a function of x (deg. east) and y (deg. north)')
parser.add_argument('-v', dest='velocity_y', default='cos(4*pi*(x-2.76*y)/180.)*cos(pi*y/180.)',
                    help='Specify the contravariant velocity component v (deg/time) along latitudes as a function of x (deg. east) and y (deg. north)')
parser.add_argument('-i', dest='input_grid_file', default='',
                    help='Specify the netcdf file containing the grid geometry/topology and grid name as FILE_NAME:GRID_NAME')
parser.add_argument('-R', dest='regularization', action='store_false',
                    help='Turn off grid regularization (recommended for uniform lat-lon)')
parser.add_argument('-t', dest='time', default=1.0, type=float,
                    help='Specify time step')
parser.add_argument('-n', dest='num_steps', default=1, type=int,
                    help='Specify number of steps')
parser.add_argument('-o', dest='output_vtk_files', default='',
                    help='Specify the output VTK files for each time step')
args = parser.parse_args()

if len(args.input_grid_file) == 0:
    print('ERROR: must provide grid file (-i)')
    sys.exit(1)

if len(args.output_vtk_files) == 0:
    print('ERROR: must provide output VTK file (-o)')
    sys.exit(2)

ug = UgridReader(args.input_grid_file, regularization=args.regularization)
numCells = ug.getNumberOfCells()
print('number of cells = {}'.format(numCells))
numPoints = numCells * 4

vxy = numpy.zeros((numPoints*2,), numpy.float64)
xy =  numpy.zeros((numPoints*2,), numpy.float64)

# velocity field
from numpy import sin, cos, pi
def tendency(xy, t):
    x = xy[:numPoints]
    y = xy[numPoints:]
    vxy[:numPoints] = eval(args.velocity_x)
    vxy[numPoints:] = eval(args.velocity_y)
    return vxy

# advect grid forward
bname = re.sub(r'.vtk', '', args.output_vtk_files)
for i in range(args.num_steps):
    ug.saveToVtkFile(bname + "{:05d}".format(i) + '.vtk')
    x, y = ug.getLonLat()
    xy[:numPoints] = x.flat
    xy[numPoints:] = y.flat
    xyNew = odeint(tendency, xy, [0.0, args.time])
    ug.setLonLat(xyNew[1, :numPoints], xyNew[1, numPoints:])

ug.saveToVtkFile(bname + "{:05d}".format(i) + '.vtk')

ext: py

max_stars_repo_name: bjlittle/mint

max_issues_repo_name: bjlittle/mint

max_forks_repo_name: bjlittle/mint

reason: import numpy,from numpy,from scipy


text:
"""Numpy-related utilities."""
from __future__ import absolute_import
from __future__ import division
from __future__ import generator_stop
from __future__ import print_function

import numpy as np
from numpy.lib.arraysetops import _unique1d
from numpy.lib.arraysetops import _unpack_tuple

def rand_replace_elem(x, ratio, fill=0, inplace=True):
    """Random replace array with specific value at certain ratio.

    Parameters
    ----------
    x : numpy.ndarray
        2D array.
    ratio : float
        Filled value's ratio.
        It must be between 0 and 1.
    fill : x.dtype.type, default 0
        Filled value.
    inplace : bool, default True
        Whether to create a copy of `x` (False) or to replace values
        in-place (True).

    Returns
    -------
    new_x : numpy.ndarray
        When inplace is False, it would return a array copied from
        x and replace elements.
    """
    shape = x.shape
    nb_true = round(ratio * x.size)
    mask = np.zeros(shape, dtype=bool)
    for i in range(shape[0]):
        mask[i, :] = rand_bool_array(nb_true, (1, shape[1]))
    if inplace:
        x[mask] = fill
        return
    else:
        new_x = x.copy()
        new_x[mask] = fill
        return new_x

def rand_bool_array(num_true, out_shape):
    """Generate random bool array.

    Parameters
    ----------
    num_true : int
        Number of True.
    out_shape : tuple
        Random bool array's shape.

    Returns
    -------
    arr : numpy.array
        Random bool array.
    """
    num_element = 1
    for i in out_shape:
        num_element = num_element * i
    arr = np.zeros(num_element, dtype=bool)
    num_true = int(num_true)
    arr[:num_true] = True
    np.random.shuffle(arr)
    arr = arr.reshape(out_shape)

    return arr

def crop_zeros(array, remain=0, return_bound=False):
    """
    Crop the edge zero of the input array.

    Parameters
    ----------
    array : numpy.ndarray
        2D numpy array.
    remain : int
        The number of edges of all zeros which you want to remain.
    return_bound : str or bool
        Select the mode to manipulate the drawing.
        True: return array and bound.
        'only_bound': return bound.
        Others: return array.

    Returns
    -------
    out : np.ndarray, optional
        Cropped array.
    left_bound : int, optional
        The edge of the left cropping.
    right_bound : int, optional
        The edge of the right cropping.
    upper_bound : int, optional
        The edge of the upper cropping.
    lower_bound : int, optional
        The edge of the lower cropping.

    References
    ----------
    https://stackoverflow.com/questions/48987774/how-to-crop-a-numpy-2d-array-to-non-zero-values
    """
    row = array.any(1)
    if row.any():
        row_size, col_size = array.shape
        col = array.any(0)

        left_bound = np.max([col.argmax() - remain, 0])
        right_bound = np.min([col_size - col[::-1].argmax() + remain, col_size - 1])  # col[::-1] is reverse of col
        upper_bound = np.max([row.argmax() - remain, 0])
        lower_bound = np.min([row_size - row[::-1].argmax() + remain, row_size - 1])  # row[::-1] is reverse of row
        out = array[upper_bound:lower_bound, left_bound:right_bound]
    else:
        left_bound = None
        right_bound = None
        upper_bound = None
        lower_bound = None
        out = np.empty((0, 0), dtype=bool)

    if isinstance(return_bound, bool) and return_bound:
        return out, (left_bound, right_bound, upper_bound, lower_bound)
    elif return_bound == 'only_bound':
        return left_bound, right_bound, upper_bound, lower_bound
    else:
        return out

def unique(ar, return_index=False, return_inverse=False,
           return_counts=False, axis=None, stable=True):
    """
    Find the unique elements of an array.
    The only difference from the numpy version is that this version will return a stable unique array (after sorting).
    For more details, see numpy's unique function.

    References
    ----------
    https://github.com/numpy/numpy/blob/v1.17.0/numpy/lib/arraysetops.py#L151-L294
    """
    ar = np.asanyarray(ar)

    if axis is None:
        ret = _unique1d(ar, return_index, return_inverse, return_counts)
        return _unpack_tuple(ret)

    # axis was specified and not None
    try:
        ar = np.swapaxes(ar, axis, 0)
    except np.AxisError:
        # this removes the "axis1" or "axis2" prefix from the error message
        raise np.AxisError(axis, ar.ndim)

    # Must reshape to a contiguous 2D array for this to work...
    orig_shape, orig_dtype = ar.shape, ar.dtype
    ar = ar.reshape(orig_shape[0], -1)
    ar = np.ascontiguousarray(ar)

    # asvoid can make sorting stable, but This method might avoid the warnings mentioned in
    # https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    if stable:
        dtype = np.dtype((np.void, ar.dtype.itemsize * ar.shape[1]))
    else:
        dtype = [('f{i}'.format(i=i), ar.dtype) for i in range(ar.shape[1])]

    try:
        consolidated = ar.view(dtype)
    except TypeError:
        # There's no good way to do this for object arrays, etc...
        msg = 'The axis argument to unique is not supported for dtype {dt}'
        raise TypeError(msg.format(dt=ar.dtype))

    def reshape_uniq(uniq):
        uniq = uniq.view(orig_dtype)
        uniq = uniq.reshape(-1, *orig_shape[1:])
        uniq = np.swapaxes(uniq, 0, axis)
        return uniq

    output = _unique1d(consolidated, return_index,
                       return_inverse, return_counts)
    output = (reshape_uniq(output[0]),) + output[1:]
    return _unpack_tuple(output)

def _new_view(arr, asvoid=False):
    """
    View the array as a dtype that contains the entire column per row.

    Parameters
    ----------
    arr : numpy.ndarray, Iterable, int, float

    Returns
    -------
    new_arr : numpy.ndarray
        New view of the input arr.

    References
    ----------
    https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    https://stackoverflow.com/questions/8317022/get-intersecting-rows-across-two-2d-numpy-arrays
    https://github.com/simpeg/simpeg/blob/46c6619c3368dd0000c3b2fe27bdbd042fe8c17a/SimPEG/Utils/matutils.py#L76
    """
    new_arr = np.ascontiguousarray(arr)  # Return a contiguous array (ndim >= 1) in memory (C order)
    num_cols = arr.shape[-1]  # Number of columns (if arr is a 2D array)
    if asvoid:
        dtype = np.dtype((np.void, arr.dtype.itemsize * num_cols))
        return new_arr.view(dtype)
    else:
        # This method might avoid the warnings mentioned in
        # https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
        dtype = {'names': ['f{}'.format(i) for i in range(num_cols)],
                 'formats': [arr[0, i].dtype for i in range(num_cols)]}
        return new_arr.view(dtype)

def in2d(arr1, arr2, assume_unique=False, invert=False):
    """
    Test whether each row of a 2-D array is also present in a second array.

    Returns a boolean array the same length as number of row of arr1 that is True
    where an row of arr1 is in arr2 and False otherwise.

    Parameters
    ----------
    arr1 : array_like
        Input 2D array.
    arr2 : array_like
        The 'rows' against which to test each row of arr1.
    assume_unique : bool, optional
        If True, the 'rows' of input arrays are both assumed to be unique, which can speed up the calculation.
        Default is False.
    invert : bool, optional
        If True, the 'rows' in the returned array are inverted
        (that is, False where an element of arr1 is in arr2 and True otherwise).
        Default is False.

    Returns
    -------
    arr3 : numpy.ndarray
        The rows arr1[in2d, :] are in arr2.

    References
    ----------
    https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    https://stackoverflow.com/questions/8317022/get-intersecting-rows-across-two-2d-numpy-arrays
    """
    # view the array as a dtype that contains the entire column per row.
    tmp1, tmp2 = map(_new_view, (arr1, arr2))
    return np.in1d(tmp1, tmp2, assume_unique=assume_unique, invert=invert)

def intersect2d(arr1, arr2, assume_unique=False, return_indices=False):
    """
    Find the intersection of two arrays.

    Return the sorted, unique 'rows' that are in both of the input arrays.

    Parameters
    ----------
    arr1, arr2 : array_like
        Input 2D arrays.
    assume_unique : bool
        If True, the 'rows' of input arrays are both assumed to be unique, which can speed up the calculation.
        Default is False.
    return_indices : bool
        If True, the indices which correspond to the intersection of 'rows' of the two arrays are returned.
        The first instance of a value is used if there are multiple. Default is False.

    Returns
    -------
    arr3 : numpy.ndarray
        Sorted 2D array of common and unique 'row'.
    comm1 : numpy.ndarray
        The indices of the first occurrences of the common values in arr1. Only provided if return_indices is True.
    comm2 : numpy.ndarray
        The indices of the first occurrences of the common values in arr2. Only provided if return_indices is True.

    References
    ----------
    https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    https://stackoverflow.com/questions/8317022/get-intersecting-rows-across-two-2d-numpy-arrays
    """
    # view the array as a dtype that contains the entire column per row.
    tmp1, tmp2 = map(_new_view, (arr1, arr2))
    if return_indices:
        arr3, comm1, comm2 = np.intersect1d(tmp1, tmp2,
                                            assume_unique=assume_unique, return_indices=return_indices)
        # reshape the structured array with original view.
        arr3 = arr3.view(arr1.dtype).reshape(-1, arr1.shape[-1])
        return arr3, comm1, comm2
    else:
        arr3 = np.intersect1d(tmp1, tmp2, assume_unique=assume_unique)
        # reshape the structured array with original view.
        arr3 = arr3.view(arr1.dtype).reshape(-1, arr1.shape[-1])
        return arr3

def setdiff2d(arr1, arr2, assume_unique=False):
    """
    Find the set difference of two arrays.

    Return the sorted, unique 'rows' in arr1 that are not in arr2.

    Parameters
    ----------
    arr1 : array_like
        Input 2D array.
    arr2 : array_like
        Input comparison 2D array.
    assume_unique : bool
        If True, the 'rows' of input arrays are both assumed to be unique, which can speed up the calculation.
        Default is False.

    Returns
    -------
    arr3 : numpy.ndarray
        Sorted 2D array of 'rows' in arr1 that are not in arr2.

    References
    ----------
    https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    https://stackoverflow.com/questions/8317022/get-intersecting-rows-across-two-2d-numpy-arrays
    """
    # view the array as a dtype that contains the entire column per row.
    tmp1, tmp2 = map(_new_view, (arr1, arr2))
    arr3 = np.setdiff1d(tmp1, tmp2, assume_unique=assume_unique)
    # reshape the structured array with original view.
    arr3 = arr3.view(arr1.dtype).reshape(-1, arr1.shape[-1])
    return arr3

def setxor2d(arr1, arr2, assume_unique=False):
    """
    Find the set exclusive-or of rows of two arrays.

    Return the sorted, unique 'rows' that are in only one (not both) of the input arrays.

    Parameters
    ----------
    arr1, arr2 : array_like
        Input 2D arrays.
    assume_unique : bool
        If True, the 'rows' of input arrays are both assumed to be unique, which can speed up the calculation.
        Default is False.
    Returns
    -------
    arr3 : numpy.ndarray
        Sorted 2D array of unique 'rows' that are in only one of the input arrays.

    References
    ----------
    https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    https://stackoverflow.com/questions/8317022/get-intersecting-rows-across-two-2d-numpy-arrays
    """
    # view the array as a dtype that contains the entire column per row.
    tmp1, tmp2 = map(_new_view, (arr1, arr2))
    arr3 = np.setxor1d(tmp1, tmp2, assume_unique=assume_unique)
    # reshape the structured array with original view.
    arr3 = arr3.view(arr1.dtype).reshape(-1, arr1.shape[-1])
    return arr3

def union2d(arr1, arr2):
    """
    Return the unique, sorted array of 'rows' that are in either of the two input arrays.

    Parameters
    ----------
    arr1, arr2 : array_like
        Input 2D arrays.

    Returns
    -------
    arr3 : numpy.ndarray
        Unique, sorted union of the input arrays.

    References
    ----------
    https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    https://stackoverflow.com/questions/8317022/get-intersecting-rows-across-two-2d-numpy-arrays
    """
    # view the array as a dtype that contains the entire column per row.
    tmp1, tmp2 = map(_new_view, (arr1, arr2))
    arr3 = np.union1d(tmp1, tmp2)
    # reshape the structured array with original view.
    arr3 = arr3.view(arr1.dtype).reshape(-1, arr1.shape[-1])
    return arr3

def set_ops(arr1, arr2, relation, dim=2, **kwargs):
    """
    Set operation.
    The operation contains 'in', 'intersection', 'difference ', 'exclusive-or' and 'union'.

    Parameters
    ----------
    arr1, arr2 : array_like
        Input array.
        For more details, see 'in1d', 'intersect1d', 'isin',
        'setdiff1d', 'setxor1d', 'union1d' and the corresponding 2d version.
    relation : str
        Set relationship.
    dim : int, optional
        1D or 2D set operation.
    kwargs : bool, optional
        For more details, see 'in1d', 'intersect1d', 'isin',
        'setdiff1d', 'setxor1d', 'union1d' and the corresponding 2d version.

    Returns
    -------
    For more details, see 'in1d', 'intersect1d', 'isin',
    'setdiff1d', 'setxor1d', 'union1d' and the corresponding 2d version.

    References
    ----------
    https://stackoverflow.com/questions/22699756/python-version-of-ismember-with-rows-and-index
    https://stackoverflow.com/questions/8317022/get-intersecting-rows-across-two-2d-numpy-arrays

    Examples
    --------
    >>> import numpy as np
    >>> from erinn.python.utils.np_utils import set_ops
    >>> a = np.array([[5, 1, 2, 3, -1.6, 1000, 0],
    ...               [5, 1, 2, 4, -3.2, 1000, 0],
    ...               [5, 1, 2, 6, -3.6, 1000, 0],
    ...               [5, 1, 2, 7, -2.5, 1000, 0],
    ...               [5, 1, 2, 8, -2.1, 1000, 0],
    ...               [5, 1, 2, 10, -1.5, 1000, 0],
    ...               [5, 1, 2, 11, -1.8, 1000, 0],
    ...               [5, 1, 2, 12, -1.8, 1000, 0],
    ...               [5, 1, 2, 14, -1.8, 1000, 0],
    ...               [5, 1, 2, 15, -1.6, 1000, 0]])
    >>> b = a[::2, :].copy()
    >>> b = np.vstack((b, a[-1, :], a[0, :]))
    >>> print(set_ops(a, b, 'in', dim=2))
    [ True False  True False  True False  True False  True  True]
    """
    # default kwargs
    assume_unique = False
    invert = False
    return_indices = False
    # parse kwargs
    for k, v in kwargs.items():
        if k == 'assume_unique':
            assume_unique = v
        if k == 'invert':
            invert = v
        if k == 'return_indices':
            return_indices = v

    if dim == 1:
        if relation.lower() == 'in':
            return np.in1d(arr1, arr2, assume_unique=assume_unique, invert=invert)
        elif relation.lower() == 'intersect':
            return np.intersect1d(arr1, arr2, assume_unique=assume_unique, return_indices=return_indices)
        elif relation.lower() == 'isin':
            np.isin(arr1, arr2, assume_unique=assume_unique, invert=invert)
        elif relation.lower() == 'diff':
            return np.setdiff1d(arr1, arr2, assume_unique=assume_unique)
        elif relation.lower() == 'xor':
            return np.setxor1d(arr1, arr2, assume_unique=assume_unique)
        elif relation.lower() == 'union':
            return np.union1d(arr1, arr2)
        else:
            raise ValueError("The positional argument 'relation' is wrong."
                             + " Please input 'in', 'intersect', 'isin', 'diff', 'xor' or 'union'.")
    elif dim == 2:
        if relation.lower() == 'in':
            return in2d(arr1, arr2, assume_unique=assume_unique, invert=invert)
        elif relation.lower() == 'intersect':
            return intersect2d(arr1, arr2, assume_unique=assume_unique, return_indices=return_indices)
        elif relation.lower() == 'diff':
            return setdiff2d(arr1, arr2, assume_unique=assume_unique)
        elif relation.lower() == 'xor':
            return setxor2d(arr1, arr2, assume_unique=assume_unique)
        elif relation.lower() == 'union':
            return union2d(arr1, arr2)
        else:
            raise ValueError("The positional argument 'relation' is wrong."
                             + " Please input 'in', 'intersect', 'diff', 'xor' or 'union'.")
    else:
        raise ValueError("The keyword argument 'dim' is wrong. Please input 1 or 2.")

reason: import numpy,from numpy


text:
from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import numpy as np
from gym import spaces
from gym_brt.envs.qube_base_env import QubeBaseEnv

"""
    Description:
        A pendulum is attached to an un-actuated joint to a horizontal arm,
        which is actuated by a rotary motor. The pendulum begins
        downwards and the goal is flip the pendulum up and then to keep it from
        falling by applying a voltage on the motor which causes a torque on the
        horizontal arm.

    Source:
        This is modified for the Quanser Qube Servo2-USB from the Cart Pole
        problem described by Barto, Sutton, and Anderson, and implemented in
        OpenAI Gym: https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py
        This description is also modified from the description by the OpenAI
        team.

    Observation:
        Type: Box(4)
        Num Observation                   Min         Max
        0   Rotary arm angle (theta)     -90 deg      90 deg
        1   Pendulum angle (alpha)       -180 deg     180 deg
        2   Cart Velocity                -Inf         Inf
        3   Pole Velocity                -Inf         Inf
        Note: the velocities are limited by the physical system.

    Actions:
        Type: Real number (1-D Continuous) (voltage applied to motor)

    Reward:
        r(s_t, a_t) = 1 - (0.8 * abs(alpha) + 0.2 * abs(theta)) / pi

    Starting State:
        Theta = 0 + noise, alpha = pi + noise

    Episode Termination:
        When theta is greater than ±90° or after 2048 steps
"""

class QubeSwingupEnv(QubeBaseEnv):
    def _reward(self):
        reward = 1 - (
            (0.8 * np.abs(self._alpha) + 0.2 * np.abs(self._target_angle - self._theta))
            / np.pi
        )
        return max(reward, 0)  # Clip for the follow env case

    def _isdone(self):
        done = False
        done |= self._episode_steps >= self._max_episode_steps
        done |= abs(self._theta) > (90 * np.pi / 180)
        return done

    def reset(self):
        super(QubeSwingupEnv, self).reset()
        state = self._reset_down()
        return state

class QubeSwingupSparseEnv(QubeSwingupEnv):
    def _reward(self):
        within_range = True
        within_range &= np.abs(self._alpha) < (1 * np.pi / 180)
        within_range &= np.abs(self._theta) < (1 * np.pi / 180)
        return 1 if within_range else 0

class QubeSwingupFollowEnv(QubeSwingupEnv):
    def __init__(self, **kwargs):
        super(QubeSwingupFollowEnv, self).__init__(**kwargs)
        obs_max = np.asarray(
            [np.pi / 2, np.pi, np.inf, np.inf, 80 * (np.pi / 180)], dtype=np.float64
        )
        self.observation_space = spaces.Box(-obs_max, obs_max)

    def _get_state(self):
        state = np.array(
            [
                self._theta,
                self._alpha,
                self._theta_dot,
                self._alpha_dot,
                self._target_angle,
            ],
            dtype=np.float64,
        )
        return state

    def _next_target_angle(self):
        # Update the target angle twice a second on average at random intervals
        if np.random.randint(1, self._frequency / 2) == 1:
            max_angle = 80 * (np.pi / 180)  # 80 degrees
            angle = np.random.uniform(-max_angle, max_angle)
        else:
            angle = self._target_angle
        return angle

class QubeSwingupFollowSparseEnv(QubeSwingupFollowEnv):
    def _reward(self):
        within_range = True
        within_range &= np.abs(self._alpha) < (1 * np.pi / 180)
        within_range &= np.abs(self._theta) < (1 * np.pi / 180)
        return 1 if within_range else 0

max_stars_repo_name: bjoseru/quanser-openai-driver

max_issues_repo_name: bjoseru/quanser-openai-driver

max_forks_repo_name: bjoseru/quanser-openai-driver

reason: import numpy


text:
import pandas as pd
import numpy as np
import json
from bs4 import BeautifulSoup
import requests
import matplotlib.pyplot as plt

# save data
import pickle

def save(data,fileName):
    with open(fileName+'.dat', 'wb') as f:
        pickle.dump(data, f)
def load(fileName):
    with open(fileName+'.dat', 'rb') as f:
        new_data = pickle.load(f)
        return new_data

pivoted_portfolio = pd.read_csv("crypto_prices_sub.csv")
# get covariance & returns of the coin - daily & for the period
daily_returns = pivoted_portfolio.pct_change()
# daily_returns_sum = daily_returns.sum()
# daily_returns_cumsum = np.exp(np.log1p(daily_returns).cumsum())-1
# daily_cor = daily_returns.corr()
# daily_cor_sub = daily_cor[daily_cor < 0.5]
# print(daily_cor_sub)
# figure = ff.create_annotated_heatmap(
#     z=corrs.values,
#     x=list(corrs.columns),
#     y=list(corrs.index),
#     annotation_text=corrs.round(2).values,
#     showscale=True)
# plt.matshow(daily_cor)
# plt.show()
# print(daily_cor.info())
# print(daily_cor.loc['priceUsd','bitcoin'].sort_values(ascending=True))
# daily_returns_cumsum.to_csv('file_name.csv', index=False)
# daily_cor.to_csv('bitcoincorrelations.csv', index=False)
# pivoted_portfolio.to_csv('crypto_prices.csv', index=False)
# print(daily_returns_cumsum.iloc[-1].sort_values(ascending=False))

# period_returns = daily_returns.mean()*729

# daily_covariance = daily_returns.cov()
# period_covariance = daily_covariance*729

p_returns, p_volatility, p_sharpe_ratio, p_coin_weights=([] for i in range(4))

# portfolio combinations to probe
number_of_cryptoassets = len(daily_returns.columns)
number_crypto_portfolios = 10000

# for each portoflio, get returns, risk and weights
for a_crypto_portfolio in range(number_crypto_portfolios):
    weights = np.random.random(number_of_cryptoassets)
    weights /= np.sum(weights)

    # print(weights)
    cumsum = []
    cumsum.append( 100.0)

    for index, row in daily_returns.iterrows():
        # print(index)
        if index > 0:
            returnsPort = np.dot(weights, row)
            # print(returnsPort)
            portValue = cumsum[index-1] + (cumsum[index-1] * returnsPort)
            # print(cumsum[index-1],portValue)
            cumsum.append(portValue)

    # returns = np.dot(weights, period_returns)*100
    returns = 100 * (cumsum[-1] - cumsum[0])/cumsum[0]
    # print(returns)
    # print(cumsum)
    cumsum = pd.Series(cumsum)
    cumsumReturns = cumsum.pct_change()
    # print(cumsumReturns)
    # volatility = np.sqrt(np.dot(weights.T, np.dot(period_covariance, weights)))*100
    # daily_covariance = cumsumReturns.cov()
    # period_covariance = daily_covariance * len(cumsumReturns)
    # volatility = np.sqrt(np.dot(weights.T, np.dot(period_covariance, weights)))*100
    volatility = 100 * cumsumReturns.std()

    p_sharpe_ratio.append(returns/volatility)
    p_returns.append(returns)
    p_volatility.append(volatility)
    p_coin_weights.append(weights)

# a dictionary for Returns and Risk values of each portfolio
portfolio = {'volatility': p_volatility,
             'sharpe_ratio': p_sharpe_ratio, 'returns': p_returns}

# extend original dictionary to accomodate each ticker and weight in the portfolio
for counter,symbol in enumerate(daily_returns.columns):
    portfolio[symbol+'-%'] = [Weight[counter] for Weight in p_coin_weights]

# make a nice dataframe of the extended dictionary
df = pd.DataFrame(portfolio)

order_cols = ['returns', 'volatility', 'sharpe_ratio']+[coin+'-%' for coin in daily_returns.columns]
df = df[order_cols]

sharpe_portfolio = df.loc[df['sharpe_ratio'] == df['sharpe_ratio'].max()]
min_variance_port = df.loc[df['volatility'] == df['volatility'].min()]
max_returns_port = df.loc[df['returns'] == df['returns'].max()]

plt.style.use('seaborn-dark')
df.plot.scatter(x='volatility', y='returns', c='sharpe_ratio', cmap='PRGn_r',
                edgecolors='black', figsize=(10, 7), grid=True)

plt.xlabel('Volatility (Std. Deviation)')
plt.ylabel('Expected Returns')
plt.title('Efficient Frontier')
plt.show()

print('****Portfolio llocations*****')
print("Best Sharpe")
print(sharpe_portfolio.T)
print("Min Volatility")
print(min_variance_port.T)
print("Max Returns")
print(max_returns_port.T)


text:
from chainer import cuda
from chainer import functions
from chainer import gradient_check
import numpy
import pytest

from chainer_chemistry.config import MAX_ATOMIC_NUM
from chainer_chemistry.links.readout.general_readout import GeneralReadout
from chainer_chemistry.utils.permutation import permute_node

atom_size = 5
hidden_dim = 7
batch_size = 2

@pytest.fixture
def readouts():
    modes = ['sum', 'max', 'summax']
    return (GeneralReadout(mode=mode) for mode in modes)

@pytest.fixture
def data():
    numpy.random.seed(0)
    atom_data = numpy.random.uniform(
        0, high=MAX_ATOMIC_NUM, size=(batch_size, atom_size, hidden_dim)
    ).astype('f')
    y_grad = numpy.random.uniform(-1, 1, (batch_size, hidden_dim)).astype('f')
    return atom_data, y_grad

def check_forward(readout, atom_data):
    y_actual = cuda.to_cpu(readout(atom_data).data)
    if readout.mode == ('sum' and 'max'):
        assert y_actual.shape == (batch_size, hidden_dim)
    elif readout.mode == 'summax':
        assert y_actual.shape == (batch_size, hidden_dim * 2)

def test_forward_cpu(readouts, data):
    atom_data = data[0]
    for readout in readouts:
        check_forward(readout, atom_data)

@pytest.mark.gpu
def test_forward_gpu(readouts, data):
    atom_data = cuda.to_gpu(data[0])
    for readout in readouts:
        readout.to_gpu()
        check_forward(readout, atom_data)

def test_forward_cpu_assert_raises(data):
    atom_data = data[0]
    readout = GeneralReadout(mode='invalid')
    with pytest.raises(ValueError):
        cuda.to_cpu(readout(atom_data).data)

def test_backward_cpu(readouts, data):
    atom_data, y_grad = data
    for readout in readouts:
        if readout.mode == 'summax':
            y_grad = functions.concat((y_grad, y_grad), axis=1).data
        gradient_check.check_backward(
            readout, atom_data, y_grad, atol=1e-2, rtol=1e-2)

@pytest.mark.gpu
def test_backward_gpu(readouts, data):
    atom_data, y_grad = map(cuda.to_gpu, data)
    for readout in readouts:
        readout.to_gpu()
        if readout.mode == 'summax':
            y_grad = functions.concat((y_grad, y_grad), axis=1).data
        # TODO(nakago): check why tolerance is so high.
        gradient_check.check_backward(
            readout, atom_data, y_grad, atol=1e-1, rtol=1e-1)

def test_forward_cpu_graph_invariant(readouts, data):
    atom_data = data[0]
    permutation_index = numpy.random.permutation(atom_size)
    permute_atom_data = permute_node(atom_data, permutation_index, axis=1)
    for readout in readouts:
        y_actual = cuda.to_cpu(readout(atom_data).data)
        permute_y_actual = cuda.to_cpu(readout(permute_atom_data).data)
        numpy.testing.assert_allclose(
            y_actual, permute_y_actual, rtol=1e-5, atol=1e-5)

if __name__ == '__main__':
    pytest.main([__file__, '-v', '-s'])

max_stars_repo_name: pfnet/chainerchem

max_issues_repo_name: pfnet/chainerchem

max_forks_repo_name: pfnet/chainerchem


text:
import numpy as np
from arch import arch_model
import pandas as pd

def GARCH_HS(s, winsize, day=4, miu=False):
    s = s.reset_index(drop=True)  # 对Series s 重设索引

    garch = arch_model(y=s, mean='Constant', lags=0, vol='GARCH', p=1, o=0, q=1, dist='normal')

    garchmodel = garch.fit(disp=0)
    params = garchmodel.params
    sigma = sigma_2 = np.zeros(winsize + 1)  # +1 是为了放未来波动率的预测值

    # Set volatility at the 1st time point of the 2rd window
    if day != 0:
        sigma[0] = (s[winsize - day:winsize]).std()
    else:
        sigma[0] = 0.0

    ret = s[winsize - 1:]
    ret = ret.reset_index(drop=True)

    # Set gamma (γ)
    if miu:
        gamma = ret - params[0]
    else:
        gamma = ret

    # gamma = ret

    gamma_2 = gamma ** 2

    for i in range(winsize):
        sigma_2[i + 1] = params[1] + params[2] * gamma_2[i] + params[3] * sigma_2[i]

    sigma = np.sqrt(sigma_2)

    wight = 1 / sigma[:-1]
    wight_ret = ret * wight * sigma[-1]
    VaR_p5 = np.percentile(wight_ret, 5)
    VaR_p50 = np.percentile(wight_ret, 50)

    return VaR_p5, VaR_p50


text:
#!/usr/bin/env python3
# coding: utf8

# Author: Lenz Furrer, 2019

'''
CNN for classifying mentions as concepts.
'''

import logging

import numpy as np
from keras.models import Model
from keras.layers import Dense
from keras.callbacks import EarlyStopping

from ..preprocessing.samples import Sampler, _deduplicated
from ..rank import cnn
from ..datasets.load import load_data
from .callback import EarlyStoppingClassificationAccuracy as AccEarlyStopping

def run_training(conf, dumpfn, **evalparams):
    '''
    Train a model and evaluate/predict.
    '''
    resources = Sampler(conf)
    labelset = getlabels(conf, resources)
    logging.info('compiling model architecture...')
    model = _create_model(conf, resources, len(labelset))

    # Pretraining.
    logging.info('pretrain on terminology entries...')
    pre_data = pre_samples(conf, resources, labelset)
    earlystopping = EarlyStopping('acc', patience=conf.stop.patience)
    _fit_model(conf, model, pre_data, callbacks=[earlystopping])

    # Actual training.
    logging.info('train on annotated corpus data...')
    tr_data, val_data = (f(conf, resources, labelset)
                         for f in (tr_samples, val_samples))
    earlystopping = AccEarlyStopping(conf, val_data, dumpfn, evalparams)
    _fit_model(conf, model, tr_data, callbacks=[earlystopping])
    logging.info('done.')

def _create_model(conf, resources, nlabels):
    embs = {e: cnn.embedding_layer(conf[e], resources.emb[e].emb_matrix)
            for e in conf.rank.embeddings}
    emb_info = [(size, embs[e]) for e, size in _input_sizes(conf)]
    inp, sem = cnn.semantic_layers(conf, emb_info)
    hidden = Dense(300, activation=conf.rank.activation)(sem)
    out = Dense(nlabels, activation='softmax')(hidden)
    model = Model(inputs=inp, outputs=out)
    model.compile(optimizer=conf.rank.optimizer,
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])
    return model

def _fit_model(conf, model, data, **kwargs):
    model.fit(data.x, data.y, sample_weight=data.weights,
              epochs=conf.rank.epochs, batch_size=conf.rank.batch_size,
              **kwargs)

def getlabels(conf, rsrc):
    '''
    An ordered list of prediction labels.

    These strings correspond to the last layer of the network.
    '''
    ids = rsrc.terminology.ids(rsrc.terminology.iter_names())
    ids.discard(conf.general.nil_symbol)
    labels = [conf.general.nil_symbol, *sorted(ids)]
    return labels

def pre_samples(conf, rsrc, labels, skip_nil=False):
    '''A DataSet of pretraining samples.'''
    logging.info('collecting names from terminology...')
    samples = list(_pretraining_samples(conf, rsrc, labels, skip_nil))
    x, y = vectorize(conf, rsrc, samples, len(labels))
    return DataSet(labels=labels, x=x, y=y)

def _pretraining_samples(conf, rsrc, labels, skip_nil=False):
    for nid, cid in enumerate(labels):
        if skip_nil and cid == conf.general.nil_symbol:
            continue
        # Train the names and definitions ("context") separately.
        for name in rsrc.terminology.names([cid]):
            yield (nid, name, '')
        for def_ in rsrc.terminology.definitions(id_=cid):
            if def_:  # skip empty definitions
                yield (nid, '', def_)

def tr_samples(conf, rsrc, labelset):
    '''A DataSet of corpus samples for training.'''
    return _corpus_data(conf, rsrc, labelset, conf.general.training_subset)

def val_samples(conf, rsrc, labelset):
    '''A DataSet of corpus samples for validation.'''
    return _corpus_data(conf, rsrc, labelset, conf.general.prediction_subset)

def _corpus_data(conf, rsrc, labels, subset):
    logging.info('collecting corpus samples from %s set...', subset)
    data = DataSet(labels=labels, mentions=[], refids=[], occs=[])
    samples = []
    cid2nid = {cid: nid for nid, cid in enumerate(labels)}
    corpus = load_data(conf, subset, rsrc.terminology)
    for (text, refs, context), occs in _deduplicated(corpus).items():
        nid = _label_index(refs, cid2nid)
        samples.append((nid, text, context))
        data.mentions.append(text)
        data.refids.append(refs)
        data.occs.append(occs)
    data.x, data.y = vectorize(conf, rsrc, samples, len(labels))
    return data

def _label_index(refs, cid2nid):
    '''
    Get the index of this label.

    Usually, there is only one label, but composite mentions
    get multiple labels.  Therefore, a list of indices is
    returned always.  Because of numpy0s advanced slice
    assignment, we can still write `y[nids] = 1`.
    '''
    # If there are multiple alternative reference IDs, get the
    # first one (alphabetically lowest).
    # In evaluation, any of the alternatives is regarded as correct.
    id_ = sorted(refs)[0]
    try:
        # Model composite mentions by setting multiple nodes to 1.
        nids = [cid2nid[i] for i in id_.split('|')]
    except KeyError:
        # Don't model multiple-concept annotations (using "+").
        nids = [0]  # NIL
    return nids

def vectorize(conf, rsrc, samples, nlabels):
    '''Produce input/output matrices for Keras.'''
    logging.info('vectorizing %d samples...', len(samples))
    xdata = [np.zeros((len(samples), size), dtype=int)
             for _, size in _input_sizes(conf)]
    ydata = np.zeros((len(samples), nlabels), dtype=int)
    vectorizers = rsrc.vectorizers
    def _vectors(text, context):
        for t, vs in ((text, vectorizers['mention']),
                      (context, vectorizers['context'])):
            for v in vs:  # vs == [] if the corresponding size == 0
                yield v.vectorize(t)

    for i, (label, text, context) in enumerate(samples):
        ydata[i][label] = 1
        for x, vec in zip(xdata, _vectors(text, context)):
            x[i] = vec

    return xdata, ydata

class DataSet:
    '''
    Container for original and vectorized input/output data.
    '''
    def __init__(self, **kwargs):
        # Original data.
        self.labels = None    # label set: sorted list of strings
        self.mentions = None  # list of strings
        self.refids = None    # list of RefID
        self.occs = None      # list of lists of triples <doc, start, end>
        # Vectorized data.
        self.x = None         # input sequences: list of 2D arrays
        self.y = None         # 1-hot encoded labels: 2D array
        self.scores = None    # hook for predictions
        # Consume constructor info.
        self.__dict__.update(kwargs)

    @property
    def weights(self):
        '''Repetition counts.'''
        if self.occs is None:
            return None
        else:
            return np.array(list(map(len, self.occs)))

    def zip(self, scores=False):
        '''
        Iterate over non-vectorized sample tuples.

        If scores is False, iterate over triples
            <mention, ref, occs>.
        If scores is True, iterate over quadruples
            <mention, ref, occs, score>.
        '''
        scores = [self.scores] if scores else []
        return zip(self.mentions, self.refids, self.occs, *scores)

def _input_sizes(conf):
    for size_name in ('sample_size', 'context_size'):
        for e in conf.rank.embeddings:
            size = conf[e][size_name]
            if size:
                yield e, size

max_stars_repo_name: lfurrer/tzlink

max_issues_repo_name: lfurrer/tzlink

max_forks_repo_name: lfurrer/tzlink


text:
# Copyright 1999-2018 Alibaba Group Holding Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pandas as pd
import numpy as np

from mars.tests.core import TestBase
from mars.dataframe.datasource.series import from_pandas as from_pandas_series
from mars.dataframe.datasource.dataframe import from_pandas as from_pandas_df

class Test(TestBase):
    def testSeriesSum(self):
        data = pd.Series(np.random.rand(20), index=[str(i) for i in range(20)], name='a')
        sum_df1 = from_pandas_series(data).sum()
        self.assertEqual(data.sum(), sum_df1.execute())

        sum_df2 = from_pandas_series(data, chunk_size=6).sum()
        self.assertAlmostEqual(data.sum(), sum_df2.execute())

        sum_df3 = from_pandas_series(data, chunk_size=3).sum()
        self.assertAlmostEqual(data.sum(), sum_df3.execute())

        sum_df4 = from_pandas_series(data, chunk_size=4).sum(axis='index')
        self.assertAlmostEqual(data.sum(axis='index'), sum_df4.execute())

        data = pd.Series(np.random.rand(20), name='a')
        data[0] = 0.1  # make sure not all elements are NAN
        data[data > 0.5] = np.nan
        sum_df1 = from_pandas_series(data, chunk_size=3).sum()
        self.assertAlmostEqual(data.sum(), sum_df1.execute())

        sum_df2 = from_pandas_series(data, chunk_size=3).sum(skipna=False)
        self.assertTrue(np.isnan(sum_df2.execute()))

        sum_df3 = from_pandas_series(data, chunk_size=3).sum(skipna=False, min_count=2)
        self.assertTrue(np.isnan(sum_df3.execute()))

        sum_df4 = from_pandas_series(data, chunk_size=3).sum(min_count=1)
        self.assertAlmostEqual(data.sum(min_count=1), sum_df4.execute())

        sum_df5 = from_pandas_series(data, chunk_size=3).sum(min_count=21)
        self.assertTrue(np.isnan(sum_df5.execute()))

    def testDataFrameSum(self):
        data = pd.DataFrame(np.random.rand(20, 10))
        sum_df1 = from_pandas_df(data).sum()
        pd.testing.assert_series_equal(data.sum(), sum_df1.execute())

        sum_df2 = from_pandas_df(data, chunk_size=3).sum()
        pd.testing.assert_series_equal(data.sum(), sum_df2.execute())

        sum_df3 = from_pandas_df(data, chunk_size=6).sum(axis='index', numeric_only=True)
        pd.testing.assert_series_equal(data.sum(axis='index', numeric_only=True), sum_df3.execute())

        sum_df4 = from_pandas_df(data, chunk_size=3).sum(axis=1)
        pd.testing.assert_series_equal(data.sum(axis=1), sum_df4.execute())

        # test null
        np_data = np.random.rand(20, 10)
        np_data[np_data > 0.6] = np.nan
        data = pd.DataFrame(np_data)

        sum_df1 = from_pandas_df(data, chunk_size=3).sum()
        pd.testing.assert_series_equal(data.sum(), sum_df1.execute())

        sum_df2 = from_pandas_df(data, chunk_size=3).sum(skipna=False)
        pd.testing.assert_series_equal(data.sum(skipna=False), sum_df2.execute())

        sum_df3 = from_pandas_df(data, chunk_size=3).sum(min_count=15)
        pd.testing.assert_series_equal(data.sum(min_count=15), sum_df3.execute())

        sum_df4 = from_pandas_df(data, chunk_size=3).sum(min_count=3)
        pd.testing.assert_series_equal(data.sum(min_count=3), sum_df4.execute())

        sum_df5 = from_pandas_df(data, chunk_size=3).sum(axis=1, min_count=3)
        pd.testing.assert_series_equal(data.sum(axis=1, min_count=3), sum_df5.execute())

        sum_df5 = from_pandas_df(data, chunk_size=3).sum(axis=1, min_count=8)
        pd.testing.assert_series_equal(data.sum(axis=1, min_count=8), sum_df5.execute())

        # test numeric_only
        data = pd.DataFrame(np.random.rand(10, 10), index=np.random.randint(-100, 100, size=(10,)),
                            columns=[np.random.bytes(10) for _ in range(10)])
        sum_df1 = from_pandas_df(data, chunk_size=2).sum()
        pd.testing.assert_series_equal(data.sum(), sum_df1.execute())

        sum_df2 = from_pandas_df(data, chunk_size=6).sum(axis='index', numeric_only=True)
        pd.testing.assert_series_equal(data.sum(axis='index', numeric_only=True), sum_df2.execute())

        sum_df3 = from_pandas_df(data, chunk_size=3).sum(axis='columns')
        pd.testing.assert_series_equal(data.sum(axis='columns'), sum_df3.execute())

        data_dict = dict((str(i), np.random.rand(10)) for i in range(10))
        data_dict['string'] = [str(i) for i in range(10)]
        data_dict['bool'] = np.random.choice([True, False], (10,))
        data = pd.DataFrame(data_dict)
        sum_df = from_pandas_df(data, chunk_size=3).sum(axis='index', numeric_only=True)
        pd.testing.assert_series_equal(data.sum(axis='index', numeric_only=True), sum_df.execute())

max_stars_repo_name: sighingnow/mars

max_issues_repo_name: sighingnow/mars

max_forks_repo_name: sighingnow/mars


text:
#!/usr/bin/env python
"""
Generate code for orientation transforms using symbolic algebra.

To make it easier to generate correct transforms for oriented shapes, we
use the sympy symbolic alegbra package to do the matrix multiplication.
The transforms are displayed both using an ascii math notation, and as
C or python code which can be pasted directly into the kernel driver.

If ever we decide to change conventions, we simply need to adjust the
order and parameters to the rotation matrices.  For display we want to
use forward transforms for the mesh describing the shape, first applying
jitter, then adjusting the view.  For calculation we know the effective q
so we instead need to first unwind the view, using the inverse rotation,
then undo the jitter to get the q to calculate for the shape in its
canonical orientation.

Set *OUTPUT* to the type of code you want to see: ccode, python, math
or any combination.
"""

from __future__ import print_function

import codecs
import sys
import re

import sympy as sp
from sympy import pi, sqrt, sin, cos, Matrix, Eq

# Select output
OUTPUT = ""
OUTPUT = OUTPUT + "ccode"
#OUTPUT = OUTPUT + "python "
OUTPUT = OUTPUT + "math "
REUSE_SINCOS = True
QC_ONLY = True # show only what is needed for dqc in the symmetric case

# include unicode symbols in output, even if piping to a pager
if sys.version_info[0] < 3:
    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
sp.init_printing(use_unicode=True)

def subs(s):
    """
    Transform sympy generated code to follow sasmodels naming conventions.
    """
    if REUSE_SINCOS:
        s = re.sub(r'(phi|psi|theta)\^\+', r'\1', s)  # jitter rep:  V^+ => V
    s = re.sub(r'([a-z]*)\^\+', r'd\1', s)  # jitter rep:  V^+ => dV
    s = re.sub(r'(cos|sin)\(([a-z]*)\)', r'\1_\2', s)  # cos(V) => cos_V
    s = re.sub(r'pow\(([a-z]*), 2\)', r'\1*\1', s)  # pow(V, 2) => V*V
    return s

def comment(s):
    r"""
    Add a comment to the generated code.  Use '\n' to separate lines.
    """
    if 'ccode' in OUTPUT:
        for line in s.split("\n"):
            print("// " + line if line else "")
    if 'python' in OUTPUT:
        for line in s.split("\n"):
            print("    ## " + line if line else "")

def vprint(var, vec, comment=None, post=None):
    """
    Generate assignment statements.

    *var* could be a single sympy symbol or a 1xN vector of symbols.

    *vec* could be a single sympy expression or a 1xN vector of expressions
    such as results from a matrix-vector multiplication.

    *comment* if present is added to the start of the block as documentation.
    """
    #for v, row in zip(var, vec): sp.pprint(Eq(v, row))
    desc = sp.pretty(Eq(var, vec), wrap_line=False)
    if not isinstance(var, Matrix):
        var, vec = [var], [vec]
    if 'ccode' in OUTPUT:
        if 'math' in OUTPUT:
            print("\n// " + comment if comment else "")
            print("/*")
            for line in desc.split("\n"):
                print(" * "+line)
            print(" *\n */")
        else:
            print("\n    // " + comment if comment else "")
        if post:
            print("    // " + post)
        for v, row in zip(var, vec):
            print(subs("    const double " + sp.ccode(row, assign_to=v)))
    if 'python' in OUTPUT:
        if comment:
            print("\n    ## " + comment)
        if 'math' in OUTPUT:
            for line in desc.split("\n"):
                print("    # " + line)
        if post:
            print("    ## " + post)
        for v, row in zip(var, vec):
            print(subs("    " + sp.ccode(row, assign_to=v)[:-1]))

    if OUTPUT == 'math ':
        print("\n// " + comment if comment else "")
        if post: print("// " + post)
        print(desc)

def mprint(var, mat, comment=None, post=None):
    """
    Generate assignment statements for matrix elements.
    """
    n = sp.prod(var.shape)
    vprint(var.reshape(n, 1), mat.reshape(n, 1), comment=comment, post=post)

# From wikipedia:
#    https://en.wikipedia.org/wiki/Rotation_matrix#Basic_rotations
def Rx(a):
    """Rotate y and z about x"""
    R = [[1, 0, 0],
         [0, +cos(a), -sin(a)],
         [0, +sin(a), +cos(a)]]
    return Matrix(R)

def Ry(a):
    """Rotate x and z about y"""
    R = [[+cos(a), 0, +sin(a)],
         [0, 1, 0],
         [-sin(a), 0, +cos(a)]]
    return Matrix(R)

def Rz(a):
    """Rotate x and y about z"""
    R = [[+cos(a), -sin(a), 0],
         [+sin(a), +cos(a), 0],
         [0, 0, 1]]
    return Matrix(R)

## ===============  Describe the transforms ====================

# Define symbols used.  Note that if you change the symbols for the jitter
# angles, you will need to update the subs() function accordingly.
dphi, dpsi, dtheta = sp.var("phi^+ psi^+ theta^+")
phi, psi, theta = sp.var("phi psi theta")
#dphi, dpsi, dtheta = sp.var("beta^+ gamma^+ alpha^+")
#phi, psi, theta = sp.var("beta gamma alpha")
x, y, z = sp.var("x y z")
q = sp.var("q")
qx, qy, qz = sp.var("qx qy qz")
dqx, dqy, dqz = sp.var("qx^+ qy^+ qz^+")
qa, qb, qc = sp.var("qa qb qc")
dqa, dqb, dqc = sp.var("qa^+ qb^+ qc^+")
qab = sp.var("qab")

# 3x3 matrix M
J = Matrix([sp.var("J(1:4)(1:4)")]).reshape(3,3)
V = Matrix([sp.var("V(1:4)(1:4)")]).reshape(3,3)
R = Matrix([sp.var("R(1:4)(1:4)")]).reshape(3,3)

# various vectors
xyz = Matrix([[x], [y], [z]])
x_hat = Matrix([[x], [0], [0]])
y_hat = Matrix([[0], [y], [0]])
z_hat = Matrix([[0], [0], [z]])
q_xy = Matrix([[qx], [qy], [0]])
q_abc = Matrix([[qa], [qb], [qc]])
q_xyz = Matrix([[qx], [qy], [qz]])
dq_abc = Matrix([[dqa], [dqb], [dqc]])
dq_xyz = Matrix([[dqx], [dqy], [dqz]])

def print_steps(jitter, jitter_inv, view, view_inv, qc_only):
    """
    Show the forward/reverse transform code for view and jitter.
    """
    if 0:  # forward calculations
        vprint(q_xyz, jitter*q_abc, "apply jitter")
        #vprint(xyz, jitter*z_hat, "r")
        #mprint(J, jitter, "forward jitter")
        vprint(dq_xyz, view*q_xyz, "apply view after jitter")
        #mprint(V, view, "forward view")

        #vprint(dq_xyz, view*jitter*q_abc, "combine view and jitter")
        mprint(R, view*jitter, "forward matrix")

    if 1:  # reverse calculations
        pre_view = "set angles from view" if REUSE_SINCOS else None
        pre_jitter = "set angles from jitter" if REUSE_SINCOS else None
        index = slice(2,3) if qc_only else slice(None,None)

        comment("\n**** direct ****")
        vprint(q_abc, view_inv*q_xy, "reverse view", post=pre_view)
        vprint(dq_abc[index,:], (jitter_inv*q_abc)[index,:],
               "reverse jitter after view", post=pre_jitter)

        comment("\n\n**** precalc ****")
        #vprint(q_abc, jitter_inv*view_inv*q_xy, "combine jitter and view reverse")
        mprint(V[:,:2], view_inv[:,:2], "reverse view matrix", post=pre_view)
        mprint(J[index,:], jitter_inv[index,:], "reverse jitter matrix", post=pre_jitter)
        mprint(R[index,:2], (J*V)[index,:2], "reverse matrix")
        comment("\n**** per point ****")
        mprint(q_abc[index,:], (R*q_xy)[index,:], "applied reverse matrix")
        #mprint(q_abc, J*V*q_xy, "applied reverse matrix")
        #mprint(R[index,:2], jitter_inv*view_inv, "reverse matrix direct")

        #vprint(q_abc, M*q_xy, "matrix application")

if 1:
    comment("==== asymmetric ====")
    print_steps(
        jitter=Rx(dphi)*Ry(dtheta)*Rz(dpsi),
        jitter_inv=Rz(-dpsi)*Ry(-dtheta)*Rx(-dphi),
        view=Rz(phi)*Ry(theta)*Rz(psi),
        view_inv=Rz(-psi)*Ry(-theta)*Rz(-phi),
        qc_only=False,
    )

if 1:
    comment("\n\n==== symmetric ====")
    print_steps(
        jitter=Rx(dphi)*Ry(dtheta),
        jitter_inv=Ry(-dtheta)*Rx(-dphi),
        view=Rz(phi)*Ry(theta),
        view_inv=Ry(-theta)*Rz(-phi),
        qc_only=QC_ONLY,
    )

    comment("\n**** qab from qc ****")
    # The indirect calculation of qab is better than directly c
    # alculating qab^2 = qa^2 + qb^2 since qc can be computed
    # as qc = M31*qx + M32*qy, thus requiring only two elements
    # of the rotation matrix.
    #vprint(qab, sqrt(qa**2 + qb**2), "Direct calculation of qab")
    vprint(dqa, sqrt((qx**2+qy**2) - dqc**2),
        "Indirect calculation of qab, from qab^2 = |q|^2 - qc^2")

if 0:
    comment("==== asymmetric (3.x) ====")
    view_inv = Rz(-psi)*Rx(theta)*Ry(-(pi/2 - phi))
    vprint(q_abc, view_inv*q_xy, "reverse view")
    print("""  existing code
    cos_alpha = cos_theta*cos_phi*qxhat + sin_theta*qyhat;
    cos_mu = (-sin_theta*cos_psi*cos_phi - sin_psi*sin_phi)*qxhat + cos_theta*cos_psi*qyhat;
    cos_nu = (-cos_phi*sin_psi*sin_theta + sin_phi*cos_psi)*qxhat + sin_psi*cos_theta*qyhat;
    """)

max_stars_repo_name: pkienzle/sasmodels

max_issues_repo_name: pkienzle/sasmodels

max_forks_repo_name: pkienzle/sasmodels

reason: import sympy,from sympy


text:
# This code is written at BigVision LLC. It is based on the OpenCV project. It is subject to the license terms in the LICENSE file found in this distribution and at http://opencv.org/license.html

# Usage example:  python3 object_detection_yolo.py --video=run.mp4
#                 python3 object_detection_yolo.py --image=bird.jpg

import cv2 as cv
import argparse
import sys
import numpy as np
import os.path

# Initialize the parameters
confThreshold = 0.5  #Confidence threshold
nmsThreshold = 0.4   #Non-maximum suppression threshold
inpWidth = 416       #Width of network's input image
inpHeight = 416      #Height of network's input image

parser = argparse.ArgumentParser(description='Object Detection using YOLO in OPENCV')
parser.add_argument('--image', help='Path to image file.')
parser.add_argument('--video', help='Path to video file.')
args = parser.parse_args()

# Load names of classes
classesFile = "coco.names";
classes = None
with open(classesFile, 'rt') as f:
    classes = f.read().rstrip('\n').split('\n')

# Give the configuration and weight files for the model and load the network using them.
modelConfiguration = "yolov3.cfg";
modelWeights = "yolov3.weights";

net = cv.dnn.readNetFromDarknet(modelConfiguration, modelWeights)
net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV)
net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU)

# Get the names of the output layers
def getOutputsNames(net):
    # Get the names of all the layers in the network
    layersNames = net.getLayerNames()
    # Get the names of the output layers, i.e. the layers with unconnected outputs
    return [layersNames[i[0] - 1] for i in net.getUnconnectedOutLayers()]

# Draw the predicted bounding box
def drawPred(classId, conf, left, top, right, bottom):
    # Draw a bounding box.
    cv.rectangle(frame, (left, top), (right, bottom), (255, 178, 50), 3)

    label = '%.2f' % conf

    # Get the label for the class name and its confidence
    if classes:
        assert(classId < len(classes))
        label = '%s:%s' % (classes[classId], label)

    #Display the label at the top of the bounding box
    labelSize, baseLine = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, 0.5, 1)
    top = max(top, labelSize[1])
    cv.rectangle(frame, (left, top - round(1.5*labelSize[1])), (left + round(1.5*labelSize[0]), top + baseLine), (255, 255, 255), cv.FILLED)
    cv.putText(frame, label, (left, top), cv.FONT_HERSHEY_SIMPLEX, 0.75, (0,0,0), 1)

# Remove the bounding boxes with low confidence using non-maxima suppression
def postprocess(frame, outs):
    frameHeight = frame.shape[0]
    frameWidth = frame.shape[1]

    classIds = []
    confidences = []
    boxes = []
    # Scan through all the bounding boxes output from the network and keep only the
    # ones with high confidence scores. Assign the box's class label as the class with the highest score.
    classIds = []
    confidences = []
    boxes = []
    for out in outs:
        for detection in out:
            scores = detection[5:]
            classId = np.argmax(scores)
            confidence = scores[classId]
            if confidence > confThreshold:
                center_x = int(detection[0] * frameWidth)
                center_y = int(detection[1] * frameHeight)
                width = int(detection[2] * frameWidth)
                height = int(detection[3] * frameHeight)
                left = int(center_x - width / 2)
                top = int(center_y - height / 2)
                classIds.append(classId)
                confidences.append(float(confidence))
                boxes.append([left, top, width, height])

    # Perform non maximum suppression to eliminate redundant overlapping boxes with
    # lower confidences.
    indices = cv.dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold)
    for i in indices:
        i = i[0]
        box = boxes[i]
        left = box[0]
        top = box[1]
        width = box[2]
        height = box[3]
        drawPred(classIds[i], confidences[i], left, top, left + width, top + height)

# Process inputs
winName = 'Deep learning object detection in OpenCV'
cv.namedWindow(winName, cv.WINDOW_NORMAL)

outputFile = "yolo_out_py.avi"
if (args.image):
    # Open the image file
    if not os.path.isfile(args.image):
        print("Input image file ", args.image, " doesn't exist")
        sys.exit(1)
    cap = cv.VideoCapture(args.image)
    outputFile = args.image[:-4]+'_yolo_out_py.jpg'
elif (args.video):
    # Open the video file
    if not os.path.isfile(args.video):
        print("Input video file ", args.video, " doesn't exist")
        sys.exit(1)
    cap = cv.VideoCapture(args.video)
    outputFile = args.video[:-4]+'_yolo_out_py.avi'
else:
    # Webcam input
    cap = cv.VideoCapture(0)

# Get the video writer initialized to save the output video
if (not args.image):
    vid_writer = cv.VideoWriter(outputFile, cv.VideoWriter_fourcc('M','J','P','G'), 30, (round(cap.get(cv.CAP_PROP_FRAME_WIDTH)),round(cap.get(cv.CAP_PROP_FRAME_HEIGHT))))

while cv.waitKey(1) < 0:

    # get frame from the video
    hasFrame, frame = cap.read()

    # Stop the program if reached end of video
    if not hasFrame:
        print("Done processing !!!")
        print("Output file is stored as ", outputFile)
        cv.waitKey(3000)
        break

    # Create a 4D blob from a frame.
    blob = cv.dnn.blobFromImage(frame, 1/255, (inpWidth, inpHeight), [0,0,0], 1, crop=False)

    # Sets the input to the network
    net.setInput(blob)

    # Runs the forward pass to get output of the output layers
    outs = net.forward(getOutputsNames(net))

    # Remove the bounding boxes with low confidence
    postprocess(frame, outs)

    # Put efficiency information. The function getPerfProfile returns the overall time for inference(t) and the timings for each of the layers(in layersTimes)
    t, _ = net.getPerfProfile()
    label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency())
    cv.putText(frame, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255))

    # Write the frame with the detection boxes
    if (args.image):
        cv.imwrite(outputFile, frame.astype(np.uint8));
    else:
        vid_writer.write(frame.astype(np.uint8))

    cv.imshow(winName, frame)

max_stars_repo_name: xiaohaijin/OpenCV

max_issues_repo_name: xiaohaijin/OpenCV

max_forks_repo_name: xiaohaijin/OpenCV


text:
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable as Var
import sys

path = os.getcwd()
print('path: ', path)
sys.path.insert(0, path)
import learning.treelstm.Constants as Constants
import torch.nn as nn
import numpy as np
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
import torch
from transformers import BertTokenizer, BertModel
from torch import device as device_
# sys.path.insert(0,'/cluster/home/xlig/qg')

# module for childsumtreelstm
class ChildSumTreeLSTM(nn.Module):
    def __init__(self, in_dim, mem_dim):
        super(ChildSumTreeLSTM, self).__init__()
        self.in_dim = in_dim
        self.mem_dim = mem_dim
        self.ioux = nn.Linear(self.in_dim, 3 * self.mem_dim)
        self.iouh = nn.Linear(self.mem_dim, 3 * self.mem_dim)
        self.fx = nn.Linear(self.in_dim, self.mem_dim)
        self.fh = nn.Linear(self.mem_dim, self.mem_dim)

    def node_forward(self, inputs, child_c, child_h):
        child_h_sum = torch.sum(child_h, dim=0, keepdim=True)
        #print(child_h_sum.shape)
        iou = self.ioux(inputs) + self.iouh(child_h_sum)
        #print(iou.shape)
        i, o, u = torch.split(iou, iou.size(1) // 3, dim=1)
        #print(i.shape)
        #print(o.shape)
        #print(u.shape)
        i, o, u = F.sigmoid(i), F.sigmoid(o), F.tanh(u)
        #print(i.shape)
        #print(o.shape)
        #print(u.shape)

        f = F.sigmoid(
            self.fh(child_h) +
            self.fx(inputs).repeat(len(child_h), 1)
        )
        #print(f.shape)

        fc = torch.mul(f, child_c)
        #print(fc.shape)

        c = torch.mul(i, u) + torch.sum(fc, dim=0, keepdim=True)
        #print(c.shape)
        h = torch.mul(o, F.tanh(c))
        #print(h.shape)
        return c, h

    def forward(self, tree, inputs):
        _ = [self.forward(tree.children[idx], inputs) for idx in range(tree.num_children)]

        if tree.num_children == 0:
            child_c = Var(inputs[0].data.new(1, self.mem_dim).fill_(0.))
            child_h = Var(inputs[0].data.new(1, self.mem_dim).fill_(0.))
        else:
            child_c, child_h = zip(*map(lambda x: x.state, tree.children))
            child_c, child_h = torch.cat(child_c, dim=0), torch.cat(child_h, dim=0)

        tree.state = self.node_forward(inputs[tree.idx], child_c, child_h)
        return tree.state

# module for distance-angle similarity
class DASimilarity(nn.Module):
    def __init__(self, mem_dim, hidden_dim, num_classes):
        super(DASimilarity, self).__init__()
        self.mem_dim = mem_dim
        self.hidden_dim = hidden_dim
        self.num_classes = num_classes
        self.wh = nn.Linear(2 * self.mem_dim, self.hidden_dim)
        self.wp = nn.Linear(self.hidden_dim, self.num_classes)

    def forward(self, lvec, rvec):
        mult_dist = torch.mul(lvec, rvec)
        abs_dist = torch.abs(torch.add(lvec, -rvec))
        vec_dist = torch.cat((mult_dist, abs_dist), 1)

        vec_dist = F.dropout(vec_dist, p=0.2, training=self.training)
        out = F.sigmoid(self.wh(vec_dist))
        out = F.log_softmax(self.wp(out))
        return out

# module for cosine similarity
class CosSimilarity(nn.Module):
    def __init__(self, mem_dim):
        super(CosSimilarity, self).__init__()
        self.cos = nn.CosineSimilarity(dim=mem_dim)

    def forward(self, lvec, rvec):
        out = self.cos(lvec, rvec)
        out = torch.autograd.Variable(torch.FloatTensor([[1 - out.data[0], out.data[0]]]), requires_grad=True)
        if torch.cuda.is_available():
            out = out.cuda()
        return F.log_softmax(out)

class RNNLstm(nn.Module):

    def __init__(self, dimension=1024):
        super(RNNLstm, self).__init__()

        self.dimension = dimension
        self.lstm = nn.LSTM(input_size=768,
                            hidden_size=dimension,
                            num_layers=1,
                            batch_first=True,
                            bidirectional=True)
        self.drop = nn.Dropout(p=0.5)

        self.fc = nn.Linear(2*dimension, 150)

    def forward(self, text_emb, text_len):

        print(text_emb.shape)
        print(text_len.type())
        packed_input = pack_padded_sequence(text_emb, text_len, batch_first=True)
        torch.set_default_tensor_type(torch.cuda.FloatTensor)
        self.lstm.flatten_parameters()
        packed_output, _ = self.lstm(packed_input)
        output, _ = pad_packed_sequence(packed_output, batch_first=True)
        print(output.shape)

        out_forward = output[range(len(output)), text_len - 1, :self.dimension]
        print(out_forward.shape)
        out_reverse = output[:, 0, self.dimension:]
        print(out_reverse.shape)
        out_reduced = torch.cat((out_forward, out_reverse), 1)
        print(out_reduced.shape)
        text_fea = self.drop(out_reduced)
        print(text_fea.shape)

        text_fea = self.fc(text_fea)
        print(text_fea.shape)
        text_fea = torch.squeeze(text_fea, 1)
        print(text_fea.shape)
        text_out = torch.sigmoid(text_fea)
        print(text_out.shape)

        return text_out

def getBertmodel():
    model = BertModel.from_pretrained("bert-base-cased")
    model.eval()
    model.cuda()
    return model

def readLines(filename):
    list = []
    count = 0;
    with open(filename, encoding="utf8") as f:

        while True:
            count += 1

            # Get next line from file
            line = f.readline()
            if(line !=""):
                list.append(line)
            # if line is empty
            # end of file is reached
            if not line:
                break
            # print("Line{}: {}".format(count, line.strip()))

        f.close()
    return list

def getBerttoken():
    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
    '''
    model = getBertmodel()
    bert_vocab = readLines("D:\\downloads\\QA\\listvocab.txt")

    for i in bert_vocab:
        print(i.rstrip())
        tokenizer.add_tokens([i.rstrip()])

    print(len(tokenizer))  # 28996

    model.resize_token_embeddings(len(tokenizer))
    # The new vector is added at the end of the embedding matrix
    print(tokenizer.get_vocab())
    '''
    return tokenizer

def getSize(linputs, tokenizer):
    marked_text = "[CLS] " + linputs + " [SEP]"
    tokenized_text = tokenizer.tokenize(marked_text)
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    segments_ids = [1] * len(tokenized_text)

    print(segments_ids)

    tokens_tensor = torch.tensor([indexed_tokens])
    segments_tensors = torch.tensor([segments_ids])
    return tokens_tensor,segments_tensors

def getEmbedding(model,tokens_tensor, segments_tensors):

    device = model.device
    cp = device.type
    if(device.type == 'cpu'):
        tokens_tensor = torch.as_tensor(tokens_tensor, dtype=torch.int64, device='cpu')
        segments_tensors = torch.as_tensor(segments_tensors, dtype=torch.int64, device='cpu')

    with torch.no_grad():
        encoded_layers, _ = model(tokens_tensor, segments_tensors)

    token_vecs = encoded_layers[0]

    # Calculate the average of all 22 token vectors.
    sentence_embedding = torch.mean(token_vecs, dim=0)

    list = sentence_embedding.squeeze().tolist()

    torch.set_default_tensor_type('torch.cuda.FloatTensor')
    a = torch.Tensor(list)

    print(a.shape)
    sentence_embedding = a[np.newaxis, :]

    print(sentence_embedding.shape)

    return token_vecs

class SimilarityTreeLSTM(nn.Module):
    def __init__(self, vocab_size, in_dim, mem_dim, similarity, sparsity):
        super(SimilarityTreeLSTM, self).__init__()
        self.emb = nn.Embedding(vocab_size, in_dim, padding_idx=Constants.PAD, sparse=sparsity)
        self.childsumtreelstm = ChildSumTreeLSTM(in_dim, mem_dim)
        self.rnnlstm = RNNLstm()
        self.similarity = similarity

    def forward(self, linputs, rinputs, bertclass):
        #size = len(linputs)
        #size2 = len(rinputs)

        linputs_tokens_tensor, linputs_segments_tensors = getSize(linputs, bertclass.bert_token)
        rinputs_tokens_tensor, rinputs_segments_tensors = getSize(rinputs, bertclass.bert_token)

        linputs_tokens_tensor = linputs_tokens_tensor.cuda()
        rinputs_tokens_tensor = rinputs_tokens_tensor.cuda()
        linputs_segments_tensors = linputs_segments_tensors.cuda()
        rinputs_segments_tensors = rinputs_segments_tensors.cuda()

        #linputs = getEmbedding(self.model_BERT,linputs_tokens_tensor, linputs_segments_tensors)
        #rinputs = getEmbedding(self.model_BERT, rinputs_tokens_tensor, rinputs_segments_tensors)

        linputs,rinputs = bertclass.getSentenceEmbedding(bertclass,linputs_tokens_tensor,linputs_segments_tensors,
                                rinputs_tokens_tensor, rinputs_segments_tensors)

        #tokens_tensor = torch.as_tensor(tokens_tensor, dtype=torch.int64, device='cpu')
        linputs = linputs.cuda()
        rinputs = rinputs.cuda()
        size = linputs_tokens_tensor.shape[1]
        size2 = rinputs_tokens_tensor.shape[1]
        torch.set_default_tensor_type('torch.cuda.FloatTensor')

        arrsize = [size]
        arrsize2 = [size2]

        V = torch.tensor(arrsize)
        V2 = torch.tensor(arrsize2)

        print(linputs.shape)
        print(rinputs.shape)

        list = linputs.squeeze().tolist()
        list2 = rinputs.squeeze().tolist()
        torch.set_default_tensor_type('torch.cuda.FloatTensor')
        a = torch.Tensor(list)
        a2 = torch.Tensor(list2)

        print(a.shape)
        new_a = a[np.newaxis, :]
        new_a2 = a2[np.newaxis, :]
        print(new_a.shape)

        print(linputs.type())
        print(new_a.type())
        print(V.type())

        #lstate, lhidden = self.childsumtreelstm(ltree, linputs)
        #rstate, rhidden = self.childsumtreelstm(rtree, rinputs)

        V = torch.as_tensor(V, dtype=torch.int64, device='cpu')
        V2 = torch.as_tensor(V2, dtype=torch.int64, device='cpu')
        torch.set_default_tensor_type(torch.FloatTensor)

        outputLSTM = self.rnnlstm(new_a.contiguous(), V.contiguous())
        torch.set_default_tensor_type(torch.FloatTensor)
        outputLSTM2 = self.rnnlstm(new_a2, V2)
        #print(lstate.shape)
       # print(lhidden.shape)
        #print(rstate.shape)
        #print(rstate.shape)
        #print(outputLSTM.shape)

        #output = self.similarity(lstate, rstate)
       # output2 = self.similarity(outputLSTM, rstate)
        output3 = self.similarity(outputLSTM, outputLSTM2)
        return output3


text:
import os
import glob
import nibabel as nib
import numpy as np
from core import utils

def load_image(name, dtype=np.float32):
    img = nib.load(name)
    return np.expand_dims(np.asarray(img.get_fdata(), dtype), 0), img.affine, img.header

if __name__ == '__main__':
    data_path = '../../../../../../dataset/training_ct_40_ffd_aug_commonspace2/lecc_images/*lecc.nii.gz'

    image_names = glob.glob(data_path)
    image_suffix = 'lecc.nii.gz'

    os.chdir(os.path.dirname(data_path))
    print(os.getcwd())

    save_path = '.'
    save_suffix = 'lecc_exp.nii.gz'
    original_size = (112, 96, 112)

    for name in image_names:
        image_name = os.path.basename(name)
        image, affine, header = load_image(image_name)

        image_exp = np.exp(image)

        image_pad = utils.pad_to_shape_image(image_exp, original_size, mode='np', method='edge')

        image_nii = nib.Nifti1Image(image_pad.squeeze(0), affine=affine, header=header)
        nib.save(image_nii, os.path.join(save_path, image_name.replace(image_suffix, save_suffix)))


text:
import numpy as np
# from qpsolvers import solve_qp
class NaiveController(object):
    def __init__(self, spec, model):
        self.name = 'PD'
        self.spec = spec
        self.model = model
        self.kp = spec["kp"]
        self.kv = spec["kv"]
    def model_inverse(self, est_data, est_params, cartesian_goal):
        """This function represents the inverse function in the model
        """
        state_goal = cartesian_goal
        return state_goal

    def control(self, dt, est_data, goal, est_params):
        # it seems that cartesian goal is simply the goal itself i.e. position and velocity.
        # if we are calling it cartesian, we should omit velocity.
        cartesian_goal = np.vstack(goal)
        cartesian_state = np.vstack([est_data["cartesian_sensor_est"]["pos"], est_data["cartesian_sensor_est"]["vel"]])
        state_goal = self.model_inverse(est_data, est_params, cartesian_goal)

        e          = (cartesian_goal - cartesian_state)
        e          = np.ravel(e)
        n = len(e)
        u          = self.kp*e[:n//2] + self.kv * e[n//2:n]
        return u

class NaiveJointController(object):
    def __init__(self, spec, model):
        self.name = 'NaiveJoint'
        self.spec = spec
        self.model = model
        self._kp = spec["kp"]
    def control(self, dt, est_data, goal, est_params):
        # simply transfer the reference joint position to the output
        cartesian_goal = np.vstack(goal)
        n = len(cartesian_goal)
        u = self._kp * cartesian_goal[:n//2]
        return u


text:
import numpy as np
import tinychain as tc
import unittest

from testutils import ClientTest

ENDPOINT = "/transact/hypothetical"
SIZE = 10

# in this case, we know the `size` of the Tensor at compile time,
# so it's safe to use a regular Python for loop, and we don't need an Op decorator like `@tc.post_op`
def example1(tensor, size):
    # Tensor.write is a PUT Op that we're using for its side-effects
    writes = [tensor[i].write(tensor[i] * 10) for i in range(size) if i % 2 == 0]
    # so we have to use `After` explicitly to make sure the writes are executed before the `tensor` is returned
    return tc.After(writes, tensor)

# in this case, we don't know the size of the Tensor at compile time,
# so we have to call `Stream.range` at runtime to achieve the same result
#
# since this function `example2` is only called at compile time though,
# it still doesn't need a decorator like `@tc.post_op`
def example2(tensor):
    # the `@tc.closure` decorator captures referenced states from the outer context
    # in this case "tensor"
    @tc.closure(tensor)
    @tc.get_op  # since the `step` Op will be called at runtime, it needs a decorator
    def step(i: tc.U64):
        return tensor[i].write(tensor[i] * 10)

    # this creates a new `Stream` with a range of numbers, and executes `step` for each `i` in the `Stream`
    return tc.Stream.range((0, tensor.size, 2)).for_each(step)

# `example3` needs its own Op context at runtime, so we define it as a POST Op using the `@tc.post_op` decorator
@tc.post_op
def example3(x: tc.tensor.Dense):  # without this type annotation, TinyChain won't know what type of `x` to expect
    @tc.closure(x)
    @tc.post_op
    def cond(i: tc.UInt):
        return i < x.size

    @tc.closure(x)
    @tc.post_op
    def step(i: tc.UInt):
        write = x[i].write(x[i] * 10)
        return tc.After(write, tc.Map(i=i + 2))  # return the new state of the loop

    return tc.While(cond, step, tc.Map(i=0))

class ExampleTests(ClientTest):
    @staticmethod
    def context():
        cxt = tc.Context()
        cxt.tensor = tc.tensor.Dense.arange([SIZE], 1, SIZE + 1)
        return cxt

    def testExample1(self):
        cxt = self.context()
        cxt.result = example1(cxt.tensor, 10)

        self.execute(cxt)

    def testExample2(self):
        cxt = self.context()
        # since we're using `example2` for its side effects, we have to use `After` explicitly
        cxt.result = tc.After(example2(cxt.tensor), cxt.tensor)

        self.execute(cxt)

    def testExample3(self):
        cxt = self.context()
        # since `example3` needs to be called at runtime, we have to explicitly include it in the test context
        cxt.loop = example3
        cxt.result = tc.After(cxt.loop(x=cxt.tensor), cxt.tensor)

        self.execute(cxt)

    def execute(self, cxt):
        expected = np.array([i if i % 2 == 0 else i * 10 for i in range(1, SIZE + 1)])
        actual = self.host.post(ENDPOINT, cxt)
        self.assertEqual(actual, expect_dense(expected))

class TensorTests(ClientTest):
    def testWhere(self):
        size = 5
        x = np.random.random(size).astype(np.bool)
        a = np.random.random(size)
        b = np.random.random(size)
        expected = np.where(x, a, b)

        cxt = tc.Context()
        cxt.x = load_dense(x, tc.Bool)
        cxt.a = load_dense(a)
        cxt.b = load_dense(b)
        cxt.result = tc.tensor.where(cxt.x, cxt.a, cxt.b)

        actual = self.host.post(ENDPOINT, cxt)
        self.assertTrue(all_close(actual, expected))

    def testRandomUniform(self):
        minval = -1
        maxval = 3

        cxt = tc.Context()
        cxt.x = tc.tensor.Dense.random_uniform([5, 1], minval, maxval)
        cxt.result = (cxt.x > -1).all().logical_and((cxt.x < 3).all()).logical_and(cxt.x.mean() > 0)

        self.assertTrue(self.host.post(ENDPOINT, cxt))

    def testTruncatedNormal(self):
        tolerance = 0.5

        cxt = tc.Context()
        cxt.x = tc.tensor.Dense.truncated_normal([10, 20])
        cxt.result = cxt.x.mean(), cxt.x.std()

        mean, std = self.host.post(ENDPOINT, cxt)
        self.assertTrue(abs(mean) < tolerance)
        self.assertTrue(abs(std - 1) < tolerance)

# Example of a matrix transpose implemented using a nested loop.
# This is not intended to be performant. Use `Tensor.transpose` to transpose any `Tensor`.
@tc.post_op
def transpose(cxt, a: tc.tensor.Dense) -> tc.tensor.Dense:
    m, n = a.shape.unpack(2)

    # this creates one `Tensor` in this `Op` context, to write to
    cxt.transposed = tc.tensor.Dense.zeros([n, m])

    # this is a tensor creation `Op` itself, i.e. each usage of `transposed` would create a new tensor
    # transposed = tc.tensor.Dense.zeros([n, m])

    @tc.closure(a, cxt.transposed)
    @tc.get_op
    def row_step(x: tc.U64):

        @tc.closure(a, x, cxt.transposed)
        @tc.get_op
        def step(y: tc.U64):
            return cxt.transposed[y, x].write(a[x, y])

        return tc.Stream.range((0, n)).for_each(step)

    rows = tc.Stream.range((0, m)).for_each(row_step)

    return tc.If(
        a.ndim == 2,
        tc.After(rows, cxt.transposed),
        tc.error.BadRequest("this test only supports a 2D Tensor"))

class NestedLoopTests(ClientTest):
    def testTranspose(self):
        cxt = tc.Context()
        cxt.transpose = transpose
        cxt.a = tc.tensor.Dense.arange([3, 4], 0, 12)
        cxt.a_t = cxt.transpose(a=cxt.a)
        cxt.test = (cxt.a.transpose() == cxt.a_t).all()
        self.assertTrue(self.host.post(ENDPOINT, cxt))

def all_close(actual, expected):
    return np.allclose(actual[tc.uri(tc.tensor.Dense)][1], expected.flatten())

def expect_dense(x, dtype=tc.I64):
    return {tc.uri(tc.tensor.Dense): [[list(x.shape), tc.uri(dtype)], x.flatten().tolist()]}

def load_dense(x, dtype=tc.F32):
    return tc.tensor.Dense.load(x.shape, dtype, x.flatten().tolist())

if __name__ == "__main__":
    unittest.main()


text:
import sqlflow_models
from tests.base import BaseTestCases

import tensorflow as tf
import numpy as np
import unittest
import random

def build_karate_club_graph():
    # All 78 edges are stored in two numpy arrays. One for source endpoints
    # while the other for destination endpoints.
    # Credit to: https://docs.dgl.ai/tutorials/basics/1_first.html
    src = np.array([1, 2, 2, 3, 3, 3, 4, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 9, 10, 10,
        10, 11, 12, 12, 13, 13, 13, 13, 16, 16, 17, 17, 19, 19, 21, 21,
        25, 25, 27, 27, 27, 28, 29, 29, 30, 30, 31, 31, 31, 31, 32, 32,
        32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33,
        33, 33, 33, 33, 33, 33, 33, 33, 33, 33])
    dst = np.array([0, 0, 1, 0, 1, 2, 0, 0, 0, 4, 5, 0, 1, 2, 3, 0, 2, 2, 0, 4,
        5, 0, 0, 3, 0, 1, 2, 3, 5, 6, 0, 1, 0, 1, 0, 1, 23, 24, 2, 23,
        24, 2, 23, 26, 1, 8, 0, 24, 25, 28, 2, 8, 14, 15, 18, 20, 22, 23,
        29, 30, 31, 8, 9, 13, 14, 15, 18, 19, 20, 22, 23, 26, 27, 28, 29, 30,
        31, 32])
    u = np.concatenate([src, dst])
    v = np.concatenate([dst, src])
    u = np.expand_dims(u, axis=1)
    v = np.expand_dims(v, axis=1)
    return np.concatenate([u,v], 1)

def acc(y, label):
    '''Function to calculate the accuracy.'''
    ll = tf.equal(tf.argmax(label, -1), tf.argmax(y, -1))
    accuarcy = tf.reduce_mean(tf.cast(ll, dtype=tf.float32))
    return accuarcy

def evaluate(x, y, model):
    '''Function to evaluate the performance of model.'''
    metric = dict()
    y_pred = model.predict(x)
    metric['acc'] = np.round(acc(y, y_pred), 5)
    return metric

class TestGCN(BaseTestCases.BaseTest):
    def setUp(self):
        feature = [[0,1,2]+random.sample(range(3, 20), 8),
                   [0,1,2]+random.sample(range(18, 40),8),
                   [0,1,2]+random.sample(range(38, 60),8),
                   [0,1,2]+random.sample(range(58, 80),8)]
        label = ['Shotokan', 'Gōjū-ryū', 'Wadō-ryū', 'Shitō-ryū']
        nodes = np.array(list(range(34)))
        edges = build_karate_club_graph()
        features, labels = list(), list()
        for i in range(34):
            idx = random.randint(0,3)
            features.append(np.eye(81)[feature[idx]].sum(0))
            labels.append(label[idx])
        self.inputs = [dict() for i in range(len(edges)*2)]
        self.labels = list()
        for i in range(len(edges)):
            self.inputs[i]['id'] = tf.convert_to_tensor(edges[i][0])
            self.inputs[i]['features'] = tf.convert_to_tensor(features[edges[i][0]])
            self.inputs[i]['from_node_id'] = tf.convert_to_tensor(edges[i][0])
            self.inputs[i]['to_node_id'] = tf.convert_to_tensor(edges[i][1])
            self.labels.append(tf.convert_to_tensor([labels[edges[i][0]]]))
        for i in range(len(edges)):
            self.inputs[i+len(edges)]['id'] = tf.convert_to_tensor(edges[i][1])
            self.inputs[i+len(edges)]['features'] = tf.convert_to_tensor(features[edges[i][1]])
            self.inputs[i+len(edges)]['from_node_id'] = tf.convert_to_tensor(edges[i][0])
            self.inputs[i+len(edges)]['to_node_id'] = tf.convert_to_tensor(edges[i][1])
            self.labels.append(tf.convert_to_tensor([labels[edges[i][1]]]))
        self.model = sqlflow_models.GCN(nhid=16, nclass=4, epochs=20, train_ratio=0.2, eval_ratio=0.15)
        self.model_class = sqlflow_models.GCN

    def test_train_and_predict(self):
        self.setUp()
        self.model.compile(optimizer=optimizer(),
                           loss='categorical_crossentropy')
        self.model.sqlflow_train_loop(zip(self.inputs, self.labels))
        metric = evaluate([self.model.features, self.model.adjacency], self.model.labels, self.model)
        assert (metric['acc'] > 0)

def optimizer():
    return tf.keras.optimizers.Adam(lr=0.01)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: hebafer/models

max_issues_repo_name: xieliaing/models

max_forks_repo_name: xieliaing/models