text:
import numpy

import cupy
from cupy_backends.cuda.api import runtime
from cupy_backends.cuda.libs import cublas
from cupy_backends.cuda.libs import cusolver
from cupy.cuda import device
from cupy import _util

_available_cuda_version = {
    'gesvdj': (9000, None),
    'gesvda': (10010, None),
    'potrfBatched': (9010, None),
    'potrsBatched': (9010, None),
    'syevj': (9000, None),
}

@_util.memoize()
def check_availability(name):
    if name not in _available_cuda_version:
        msg = 'No available version information specified for {}'.format(name)
        raise ValueError(msg)
    version_added, version_removed = _available_cuda_version[name]
    cuda_version = runtime.runtimeGetVersion()
    if version_added is not None and cuda_version < version_added:
        return False
    if version_removed is not None and cuda_version >= version_removed:
        return False
    return True

def gesvdj(a, full_matrices=True, compute_uv=True, overwrite_a=False):
    """Singular value decomposition using cusolverDn<t>gesvdj().

    Factorizes the matrix ``a`` into two unitary matrices ``u`` and ``v`` and
    a singular values vector ``s`` such that ``a == u @ diag(s) @ v*``.

    Args:
        a (cupy.ndarray): The input matrix with dimension ``(M, N)``.
        full_matrices (bool): If True, it returns u and v with dimensions
            ``(M, M)`` and ``(N, N)``. Otherwise, the dimensions of u and v
            are respectively ``(M, K)`` and ``(K, N)``, where
            ``K = min(M, N)``.
        compute_uv (bool): If ``False``, it only returns singular values.
        overwrite_a (bool): If ``True``, matrix ``a`` might be overwritten.

    Returns:
        tuple of :class:`cupy.ndarray`:
            A tuple of ``(u, s, v)``.
    """
    if not check_availability('gesvdj'):
        raise RuntimeError('gesvdj is not available.')

    if a.ndim == 3:
        return _gesvdj_batched(a, full_matrices, compute_uv, overwrite_a)

    assert a.ndim == 2

    if a.dtype == 'f':
        helper = cusolver.sgesvdj_bufferSize
        solver = cusolver.sgesvdj
        s_dtype = 'f'
    elif a.dtype == 'd':
        helper = cusolver.dgesvdj_bufferSize
        solver = cusolver.dgesvdj
        s_dtype = 'd'
    elif a.dtype == 'F':
        helper = cusolver.cgesvdj_bufferSize
        solver = cusolver.cgesvdj
        s_dtype = 'f'
    elif a.dtype == 'D':
        helper = cusolver.zgesvdj_bufferSize
        solver = cusolver.zgesvdj
        s_dtype = 'd'
    else:
        raise TypeError

    handle = device.get_cusolver_handle()
    m, n = a.shape
    a = cupy.array(a, order='F', copy=not overwrite_a)
    lda = m
    mn = min(m, n)
    s = cupy.empty(mn, dtype=s_dtype)
    ldu = m
    ldv = n
    if compute_uv:
        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
    else:
        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR
        full_matrices = False
    if full_matrices:
        econ = 0
        u = cupy.empty((ldu, m), dtype=a.dtype, order='F')
        v = cupy.empty((ldv, n), dtype=a.dtype, order='F')
    else:
        econ = 1
        u = cupy.empty((ldu, mn), dtype=a.dtype, order='F')
        v = cupy.empty((ldv, mn), dtype=a.dtype, order='F')
    params = cusolver.createGesvdjInfo()
    lwork = helper(handle, jobz, econ, m, n, a.data.ptr, lda, s.data.ptr,
                   u.data.ptr, ldu, v.data.ptr, ldv, params)
    work = cupy.empty(lwork, dtype=a.dtype)
    info = cupy.empty(1, dtype=numpy.int32)
    solver(handle, jobz, econ, m, n, a.data.ptr, lda, s.data.ptr,
           u.data.ptr, ldu, v.data.ptr, ldv, work.data.ptr, lwork,
           info.data.ptr, params)
    cupy.linalg.util._check_cusolver_dev_info_if_synchronization_allowed(
        gesvdj, info)

    cusolver.destroyGesvdjInfo(params)
    if compute_uv:
        return u, s, v
    else:
        return s

def _gesvdj_batched(a, full_matrices, compute_uv, overwrite_a):
    if a.dtype == 'f':
        helper = cusolver.sgesvdjBatched_bufferSize
        solver = cusolver.sgesvdjBatched
        s_dtype = 'f'
    elif a.dtype == 'd':
        helper = cusolver.dgesvdjBatched_bufferSize
        solver = cusolver.dgesvdjBatched
        s_dtype = 'd'
    elif a.dtype == 'F':
        helper = cusolver.cgesvdjBatched_bufferSize
        solver = cusolver.cgesvdjBatched
        s_dtype = 'f'
    elif a.dtype == 'D':
        helper = cusolver.zgesvdjBatched_bufferSize
        solver = cusolver.zgesvdjBatched
        s_dtype = 'd'
    else:
        raise TypeError

    handle = device.get_cusolver_handle()
    batch_size, m, n = a.shape
    a = cupy.array(a.swapaxes(-2, -1), order='C', copy=not overwrite_a)
    lda = m
    mn = min(m, n)
    s = cupy.empty((batch_size, mn), dtype=s_dtype)
    ldu = m
    ldv = n
    if compute_uv:
        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
    else:
        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR
        # if not batched, `full_matrices = False` could speedup.

    u = cupy.empty((batch_size, m, ldu), dtype=a.dtype).swapaxes(-2, -1)
    v = cupy.empty((batch_size, n, ldv), dtype=a.dtype).swapaxes(-2, -1)
    params = cusolver.createGesvdjInfo()
    lwork = helper(handle, jobz, m, n, a.data.ptr, lda, s.data.ptr,
                   u.data.ptr, ldu, v.data.ptr, ldv, params, batch_size)
    work = cupy.empty(lwork, dtype=a.dtype)
    info = cupy.empty(1, dtype=numpy.int32)
    solver(handle, jobz, m, n, a.data.ptr, lda, s.data.ptr,
           u.data.ptr, ldu, v.data.ptr, ldv, work.data.ptr, lwork,
           info.data.ptr, params, batch_size)
    cupy.linalg.util._check_cusolver_dev_info_if_synchronization_allowed(
        gesvdj, info)

    cusolver.destroyGesvdjInfo(params)
    if not full_matrices:
        u = u[..., :mn]
        v = v[..., :mn]
    if compute_uv:
        return u, s, v
    else:
        return s

def gesvda(a, compute_uv=True):
    """Singular value decomposition using cusolverDn<t>gesvdaStridedBatched().

    Factorizes the matrix ``a`` into two unitary matrices ``u`` and ``v`` and
    a singular values vector ``s`` such that ``a == u @ diag(s) @ v*``.

    Args:
        a (cupy.ndarray): The input matrix with dimension ``(.., M, N)``.
        compute_uv (bool): If ``False``, it only returns singular values.

    Returns:
        tuple of :class:`cupy.ndarray`:
            A tuple of ``(u, s, v)``.
    """
    if not check_availability('gesvda'):
        raise RuntimeError('gesvda is not available.')

    assert a.ndim >= 2
    a_ndim = a.ndim
    a_shape = a.shape
    m, n = a_shape[-2:]
    assert m >= n

    if a.dtype == 'f':
        helper = cusolver.sgesvdaStridedBatched_bufferSize
        solver = cusolver.sgesvdaStridedBatched
        s_dtype = 'f'
    elif a.dtype == 'd':
        helper = cusolver.dgesvdaStridedBatched_bufferSize
        solver = cusolver.dgesvdaStridedBatched
        s_dtype = 'd'
    elif a.dtype == 'F':
        helper = cusolver.cgesvdaStridedBatched_bufferSize
        solver = cusolver.cgesvdaStridedBatched
        s_dtype = 'f'
    elif a.dtype == 'D':
        helper = cusolver.zgesvdaStridedBatched_bufferSize
        solver = cusolver.zgesvdaStridedBatched
        s_dtype = 'd'
    else:
        raise TypeError

    handle = device.get_cusolver_handle()
    if compute_uv:
        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
    else:
        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR
    rank = min(m, n)
    if a_ndim == 2:
        batch_size = 1
    else:
        batch_size = numpy.array(a_shape[:-2]).prod().item()
    a = a.reshape((batch_size, m, n))
    a = cupy.ascontiguousarray(a.transpose(0, 2, 1))
    lda = m
    stride_a = lda * n
    s = cupy.empty((batch_size, rank), dtype=s_dtype)
    stride_s = rank
    ldu = m
    ldv = n
    u = cupy.empty((batch_size, rank, ldu), dtype=a.dtype, order='C')
    v = cupy.empty((batch_size, rank, ldv), dtype=a.dtype, order='C')
    stride_u = rank * ldu
    stride_v = rank * ldv
    lwork = helper(handle, jobz, rank, m, n, a.data.ptr, lda, stride_a,
                   s.data.ptr, stride_s, u.data.ptr, ldu, stride_u,
                   v.data.ptr, ldv, stride_v, batch_size)
    work = cupy.empty((lwork,), dtype=a.dtype)
    info = cupy.empty((batch_size,), dtype=numpy.int32)
    r_norm = numpy.empty((batch_size,), dtype=numpy.float64)
    solver(handle, jobz, rank, m, n, a.data.ptr, lda, stride_a, s.data.ptr,
           stride_s, u.data.ptr, ldu, stride_u, v.data.ptr, ldv, stride_v,
           work.data.ptr, lwork, info.data.ptr, r_norm.ctypes.data, batch_size)

    s = s.reshape(a_shape[:-2] + (s.shape[-1],))
    if not compute_uv:
        return s

    u = u.transpose(0, 2, 1)
    v = v.transpose(0, 2, 1)
    u = u.reshape(a_shape[:-2] + (u.shape[-2:]))
    v = v.reshape(a_shape[:-2] + (v.shape[-2:]))
    return u, s, v

def syevj(a, UPLO='L', with_eigen_vector=True):
    """Eigenvalue decomposition of symmetric matrix using cusolverDn<t>syevj().

    Computes eigenvalues ``w`` and (optionally) eigenvectors ``v`` of a complex
    Hermitian or a real symmetric matrix.

    Args:
        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
            of symmetric 2-D square matrices ``(..., M, M)``.
        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
            part of ``a`` is used. ``'L'`` uses the lower triangular part of
            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
        with_eigen_vector (bool): Indicates whether or not eigenvectors
            are computed.

    Returns:
        tuple of :class:`~cupy.ndarray`:
            Returns a tuple ``(w, v)``. ``w`` contains eigenvalues and
            ``v`` contains eigenvectors. ``v[:, i]`` is an eigenvector
            corresponding to an eigenvalue ``w[i]``. For batch input,
            ``v[k, :, i]`` is an eigenvector corresponding to an eigenvalue
            ``w[k, i]`` of ``a[k]``.
    """
    if not check_availability('syevj'):
        raise RuntimeError('syevj is not available.')

    if UPLO not in ('L', 'U'):
        raise ValueError('UPLO argument must be \'L\' or \'U\'')

    if a.ndim > 2:
        return _syevj_batched(a, UPLO, with_eigen_vector)

    assert a.ndim == 2

    if a.dtype == 'f' or a.dtype == 'e':
        dtype = 'f'
        inp_w_dtype = 'f'
        inp_v_dtype = 'f'
        ret_w_dtype = a.dtype
        ret_v_dtype = a.dtype
    elif a.dtype == 'd':
        dtype = 'd'
        inp_w_dtype = 'd'
        inp_v_dtype = 'd'
        ret_w_dtype = 'd'
        ret_v_dtype = 'd'
    elif a.dtype == 'F':
        dtype = 'F'
        inp_w_dtype = 'f'
        inp_v_dtype = 'F'
        ret_w_dtype = 'f'
        ret_v_dtype = 'F'
    elif a.dtype == 'D':
        dtype = 'D'
        inp_w_dtype = 'd'
        inp_v_dtype = 'D'
        ret_w_dtype = 'd'
        ret_v_dtype = 'D'
    else:
        # NumPy uses float64 when an input is not floating point number.
        dtype = 'd'
        inp_w_dtype = 'd'
        inp_v_dtype = 'd'
        ret_w_dtype = 'd'
        ret_v_dtype = 'd'

    # Note that cuSolver assumes fortran array
    v = a.astype(inp_v_dtype, order='F', copy=True)

    m, lda = a.shape
    w = cupy.empty(m, inp_w_dtype)
    dev_info = cupy.empty((), numpy.int32)
    handle = device.Device().cusolver_handle

    if with_eigen_vector:
        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
    else:
        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR

    if UPLO == 'L':
        uplo = cublas.CUBLAS_FILL_MODE_LOWER
    else:  # UPLO == 'U'
        uplo = cublas.CUBLAS_FILL_MODE_UPPER

    if dtype == 'f':
        buffer_size = cusolver.ssyevj_bufferSize
        syevj = cusolver.ssyevj
    elif dtype == 'd':
        buffer_size = cusolver.dsyevj_bufferSize
        syevj = cusolver.dsyevj
    elif dtype == 'F':
        buffer_size = cusolver.cheevj_bufferSize
        syevj = cusolver.cheevj
    elif dtype == 'D':
        buffer_size = cusolver.zheevj_bufferSize
        syevj = cusolver.zheevj
    else:
        raise RuntimeError('Only float and double and cuComplex and '
                           + 'cuDoubleComplex are supported')

    params = cusolver.createSyevjInfo()
    work_size = buffer_size(
        handle, jobz, uplo, m, v.data.ptr, lda, w.data.ptr, params)
    work = cupy.empty(work_size, inp_v_dtype)
    syevj(
        handle, jobz, uplo, m, v.data.ptr, lda,
        w.data.ptr, work.data.ptr, work_size, dev_info.data.ptr, params)
    cupy.linalg.util._check_cusolver_dev_info_if_synchronization_allowed(
        syevj, dev_info)

    cusolver.destroySyevjInfo(params)

    w = w.astype(ret_w_dtype, copy=False)
    if not with_eigen_vector:
        return w
    v = v.astype(ret_v_dtype, copy=False)
    return w, v

def _syevj_batched(a, UPLO, with_eigen_vector):
    if a.dtype == 'f' or a.dtype == 'e':
        dtype = 'f'
        inp_w_dtype = 'f'
        inp_v_dtype = 'f'
        ret_w_dtype = a.dtype
        ret_v_dtype = a.dtype
    elif a.dtype == 'd':
        dtype = 'd'
        inp_w_dtype = 'd'
        inp_v_dtype = 'd'
        ret_w_dtype = 'd'
        ret_v_dtype = 'd'
    elif a.dtype == 'F':
        dtype = 'F'
        inp_w_dtype = 'f'
        inp_v_dtype = 'F'
        ret_w_dtype = 'f'
        ret_v_dtype = 'F'
    elif a.dtype == 'D':
        dtype = 'D'
        inp_w_dtype = 'd'
        inp_v_dtype = 'D'
        ret_w_dtype = 'd'
        ret_v_dtype = 'D'
    else:
        # NumPy uses float64 when an input is not floating point number.
        dtype = 'd'
        inp_w_dtype = 'd'
        inp_v_dtype = 'd'
        ret_w_dtype = 'd'
        ret_v_dtype = 'd'

    *batch_shape, m, lda = a.shape
    batch_size = numpy.prod(batch_shape)
    a = a.reshape(batch_size, m, lda)
    v = cupy.array(a.swapaxes(-2, -1), order='C', copy=True, dtype=inp_v_dtype)

    w = cupy.empty((batch_size, m), inp_w_dtype).swapaxes(-2, 1)
    dev_info = cupy.empty((), numpy.int32)
    handle = device.Device().cusolver_handle

    if with_eigen_vector:
        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
    else:
        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR

    if UPLO == 'L':
        uplo = cublas.CUBLAS_FILL_MODE_LOWER
    else:  # UPLO == 'U'
        uplo = cublas.CUBLAS_FILL_MODE_UPPER

    if dtype == 'f':
        buffer_size = cusolver.ssyevjBatched_bufferSize
        syevjBatched = cusolver.ssyevjBatched
    elif dtype == 'd':
        buffer_size = cusolver.dsyevjBatched_bufferSize
        syevjBatched = cusolver.dsyevjBatched
    elif dtype == 'F':
        buffer_size = cusolver.cheevjBatched_bufferSize
        syevjBatched = cusolver.cheevjBatched
    elif dtype == 'D':
        buffer_size = cusolver.zheevjBatched_bufferSize
        syevjBatched = cusolver.zheevjBatched
    else:
        raise RuntimeError('Only float and double and cuComplex and '
                           + 'cuDoubleComplex are supported')

    params = cusolver.createSyevjInfo()
    work_size = buffer_size(
        handle, jobz, uplo, m, v.data.ptr, lda, w.data.ptr, params, batch_size)
    work = cupy.empty(work_size, inp_v_dtype)
    syevjBatched(
        handle, jobz, uplo, m, v.data.ptr, lda,
        w.data.ptr, work.data.ptr, work_size, dev_info.data.ptr, params,
        batch_size)
    cupy.linalg.util._check_cusolver_dev_info_if_synchronization_allowed(
        syevjBatched, dev_info)

    cusolver.destroySyevjInfo(params)

    w = w.astype(ret_w_dtype, copy=False)
    w = w.swapaxes(-2, -1).reshape(*batch_shape, m)
    if not with_eigen_vector:
        return w
    v = v.astype(ret_v_dtype, copy=False)
    v = v.swapaxes(-2, -1).reshape(*batch_shape, m, m)
    return w, v

def gesv(a, b):
    """Solve a linear matrix equation using cusolverDn<t>getr[fs]().

    Computes the solution to a system of linear equation ``ax = b``.

    Args:
        a (cupy.ndarray): The matrix with dimension ``(M, M)``.
        b (cupy.ndarray): The matrix with dimension ``(M)`` or ``(M, K)``.

    Returns:
        cupy.ndarray:
            The matrix with dimension ``(M)`` or ``(M, K)``.
    """
    if a.ndim != 2:
        raise ValueError('a.ndim must be 2 (actual: {})'.format(a.ndim))
    if b.ndim not in (1, 2):
        raise ValueError('b.ndim must be 1 or 2 (actual: {})'.format(b.ndim))
    if a.shape[0] != a.shape[1]:
        raise ValueError('a must be a square matrix.')
    if a.shape[0] != b.shape[0]:
        raise ValueError('shape mismatch (a: {}, b: {}).'.
                         format(a.shape, b.shape))

    dtype = numpy.promote_types(a.dtype.char, 'f')
    if dtype == 'f':
        t = 's'
    elif dtype == 'd':
        t = 'd'
    elif dtype == 'F':
        t = 'c'
    elif dtype == 'D':
        t = 'z'
    else:
        raise ValueError('unsupported dtype (actual:{})'.format(a.dtype))
    helper = getattr(cusolver, t + 'getrf_bufferSize')
    getrf = getattr(cusolver, t + 'getrf')
    getrs = getattr(cusolver, t + 'getrs')

    n = b.shape[0]
    nrhs = b.shape[1] if b.ndim == 2 else 1
    a_data_ptr = a.data.ptr
    b_data_ptr = b.data.ptr
    a = cupy.asfortranarray(a, dtype=dtype)
    b = cupy.asfortranarray(b, dtype=dtype)
    if a.data.ptr == a_data_ptr:
        a = a.copy()
    if b.data.ptr == b_data_ptr:
        b = b.copy()

    handle = device.get_cusolver_handle()
    dipiv = cupy.empty(n, dtype=numpy.int32)
    dinfo = cupy.empty(1, dtype=numpy.int32)
    lwork = helper(handle, n, n, a.data.ptr, n)
    dwork = cupy.empty(lwork, dtype=a.dtype)
    # LU factrization (A = L * U)
    getrf(handle, n, n, a.data.ptr, n, dwork.data.ptr, dipiv.data.ptr,
          dinfo.data.ptr)
    cupy.linalg.util._check_cusolver_dev_info_if_synchronization_allowed(
        getrf, dinfo)
    # Solves Ax = b
    getrs(handle, cublas.CUBLAS_OP_N, n, nrhs, a.data.ptr, n,
          dipiv.data.ptr, b.data.ptr, n, dinfo.data.ptr)
    cupy.linalg.util._check_cusolver_dev_info_if_synchronization_allowed(
        getrs, dinfo)
    return b

max_stars_repo_name: garanews/cupy

max_issues_repo_name: trivialfis/cupy

max_forks_repo_name: trivialfis/cupy

reason: import numpy,import cupy,from cupy


text:
import numpy as np

def LPFbead(qrange, sigmabead):
    '''
    Compute the spherical form factor given a range of q values.

    Parameters:
    ----------
    qrange: numpy.array
        array of values in q-space to compute form factor for.
    sigmabead: float
        diameter of the sphere.

    Return:
    ----------
    Fqb: numpy.array
        array of values of the spherical form factors (F(q)) computed at q-points listed in qrange.

    '''
    import numpy as np
    R=np.true_divide(sigmabead,2)
    QR=np.multiply(qrange,R)
    Fqb=np.multiply(np.true_divide(np.sin(QR)-np.multiply(QR,np.cos(QR)),np.power(QR,3)),3)

    return Fqb

ext: py

max_stars_repo_name: arthijayaraman-lab/crease-ga

max_issues_repo_name: arthijayaraman-lab/crease-ga

reason: import numpy


text:
# Copyright 2019 DeepMind Technologies Limited and Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tools for latent optimisation."""
import collections
import os

from absl import logging
import numpy as np
import tensorflow.compat.v1 as tf
import tensorflow_probability as tfp

from cs_gan import nets

tfd = tfp.distributions

class ModelOutputs(
    collections.namedtuple('AdversarialModelOutputs',
                           ['optimization_components', 'debug_ops'])):
  """All the information produced by the adversarial module.

  Fields:

    * `optimization_components`: A dictionary. Each entry in this dictionary
      corresponds to a module to train using their own optimizer. The keys are
      names of the components, and the values are `common.OptimizationComponent`
      instances. The keys of this dict can be made keys of the configuration
      used by the main train loop, to define the configuration of the
      optimization details for each module.
    * `debug_ops`: A dictionary, from string to a scalar `tf.Tensor`. Quantities
      used for tracking training.
  """

class OptimizationComponent(
    collections.namedtuple('OptimizationComponent', ['loss', 'vars'])):
  """Information needed by the optimizer to train modules.

  Usage:
      `optimizer.minimize(
          opt_compoment.loss, var_list=opt_component.vars)`

  Fields:

    * `loss`: A `tf.Tensor` the loss of the module.
    * `vars`: A list of variables, the ones which will be used to minimize the
      loss.
  """

def cross_entropy_loss(logits, expected):
  """The cross entropy classification loss between logits and expected values.

  The loss proposed by the original GAN paper: https://arxiv.org/abs/1406.2661.

  Args:
    logits: a `tf.Tensor`, the model produced logits.
    expected: a `tf.Tensor`, the expected output.

  Returns:
    A scalar `tf.Tensor`, the average loss obtained on the given inputs.

  Raises:
    ValueError: if the logits do not have shape [batch_size, 2].
  """

  num_logits = logits.get_shape()[1]
  if num_logits != 2:
    raise ValueError(('Invalid number of logits for cross_entropy_loss! '
                      'cross_entropy_loss supports only 2 output logits!'))
  return tf.reduce_mean(
      tf.nn.sparse_softmax_cross_entropy_with_logits(
          logits=logits, labels=expected))

def optimise_and_sample(init_z, module, data, is_training):
  """Optimising generator latent variables and sample."""

  if module.num_z_iters is None or module.num_z_iters == 0:
    z_final = init_z
  else:
    init_loop_vars = (0, _project_z(init_z, module.z_project_method))
    loop_cond = lambda i, _: i < module.num_z_iters
    def loop_body(i, z):
      loop_samples = module.generator(z, is_training)
      gen_loss = module.gen_loss_fn(data, loop_samples)
      z_grad = tf.gradients(gen_loss, z)[0]
      z -= module.z_step_size * z_grad
      z = _project_z(z, module.z_project_method)
      return i + 1, z

    # Use the following static loop for debugging
    # z = init_z
    # for _ in xrange(num_z_iters):
    #   _, z = loop_body(0, z)
    # z_final = z

    _, z_final = tf.while_loop(loop_cond,
                               loop_body,
                               init_loop_vars)

  return module.generator(z_final, is_training), z_final

def get_optimisation_cost(initial_z, optimised_z):
  optimisation_cost = tf.reduce_mean(
      tf.reduce_sum((optimised_z - initial_z)**2, -1))
  return optimisation_cost

def _project_z(z, project_method='clip'):
  """To be used for projected gradient descent over z."""
  if project_method == 'norm':
    z_p = tf.nn.l2_normalize(z, axis=-1)
  elif project_method == 'clip':
    z_p = tf.clip_by_value(z, -1, 1)
  else:
    raise ValueError('Unknown project_method: {}'.format(project_method))
  return z_p

class DataProcessor(object):

  def preprocess(self, x):
    return x * 2 - 1

  def postprocess(self, x):
    return (x + 1) / 2.

def _get_np_data(data_processor, dataset, split='train'):
  """Get the dataset as numpy arrays."""
  index = 0 if split == 'train' else 1
  if dataset == 'mnist':
    # Construct the dataset.
    x, _ = tf.keras.datasets.mnist.load_data()[index]
    # Note: tf dataset is binary so we convert it to float.
    x = x.astype(np.float32)
    x = x / 255.
    x = x.reshape((-1, 28, 28, 1))

  if dataset == 'cifar':
    x, _ = tf.keras.datasets.cifar10.load_data()[index]
    x = x.astype(np.float32)
    x = x / 255.

  if data_processor:
    # Normalize data if a processor is given.
    x = data_processor.preprocess(x)
  return x

def make_output_dir(output_dir):
  logging.info('Creating output dir %s', output_dir)
  if not tf.gfile.IsDirectory(output_dir):
    tf.gfile.MakeDirs(output_dir)

def get_ckpt_dir(output_dir):
  ckpt_dir = os.path.join(output_dir, 'ckpt')
  if not tf.gfile.IsDirectory(ckpt_dir):
    tf.gfile.MakeDirs(ckpt_dir)
  return ckpt_dir

def get_real_data_for_eval(num_eval_samples, dataset, split='valid'):
  data = _get_np_data(data_processor=None, dataset=dataset, split=split)
  data = data[:num_eval_samples]
  return tf.constant(data)

def get_summaries(ops):
  summaries = []
  for name, op in ops.items():
    # Ensure to log the value ops before writing them in the summary.
    # We do this instead of a hook to ensure IS/FID are never computed twice.
    print_op = tf.print(name, [op], output_stream=tf.logging.info)
    with tf.control_dependencies([print_op]):
      summary = tf.summary.scalar(name, op)
      summaries.append(summary)
  return summaries

def get_train_dataset(data_processor, dataset, batch_size):
  """Creates the training data tensors."""
  x_train = _get_np_data(data_processor, dataset, split='train')
  # Create the TF dataset.
  dataset = tf.data.Dataset.from_tensor_slices(x_train)

  # Shuffle and repeat the dataset for training.
  # This is required because we want to do multiple passes through the entire
  # dataset when training.
  dataset = dataset.shuffle(100000).repeat()

  # Batch the data and return the data batch.
  one_shot_iterator = dataset.batch(batch_size).make_one_shot_iterator()
  data_batch = one_shot_iterator.get_next()
  return data_batch

def get_generator(dataset):
  if dataset == 'mnist':
    return nets.MLPGeneratorNet()
  if dataset == 'cifar':
    return nets.ConvGenNet()

def get_metric_net(dataset, num_outputs=2, use_sn=True):
  if dataset == 'mnist':
    return nets.MLPMetricNet(num_outputs)
  if dataset == 'cifar':
    return nets.ConvMetricNet(num_outputs, use_sn)

def make_prior(num_latents):
  # Zero mean, unit variance prior.
  prior_mean = tf.zeros(shape=(num_latents), dtype=tf.float32)
  prior_scale = tf.ones(shape=(num_latents), dtype=tf.float32)

  return tfd.Normal(loc=prior_mean, scale=prior_scale)

max_stars_repo_name: Study-Repos-Forks/deepmind-research

max_issues_repo_name: Study-Repos-Forks/deepmind-research

max_forks_repo_name: Study-Repos-Forks/deepmind-research


text:
import cv2
import numpy as np
import torch
from torch.utils.data import Dataset
import string
import random

class TestDataset(Dataset):
    def __init__(self,
                 epoch_len = 10000,
                 seq_len = 8,
                 transform=None,
                 abc=string.digits):
        super().__init__()
        self.abc = abc
        self.epoch_len = epoch_len
        self.seq_len = seq_len
        self.transform = transform

    def __len__(self):
        return self.epoch_len

    def get_abc(self):
        return self.abc

    def set_mode(self):
        return

    def generate_string(self):
        return ''.join(random.choice(self.abc) for _ in range(self.seq_len))

    def get_sample(self):
        h, w = 64, int(self.seq_len * 64 * 2.5)
        pw = int(w / self.seq_len)
        seq = []
        img = np.zeros((h, w), dtype=np.uint8)
        text = self.generate_string()
        for i in range(len(text)):
            c = text[i]
            seq.append(self.abc.find(c) + 1)
            hs, ws = 32, 32
            symb = np.zeros((hs, ws), dtype=np.uint8)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(symb, str(c), (3, 30), font, 1.2, (255), 2, cv2.LINE_AA)
            # Rotation
            angle = 60
            ang_rot = np.random.uniform(angle) - angle/2
            transform = cv2.getRotationMatrix2D((ws/2, hs/2), ang_rot, 1)
            symb = cv2.warpAffine(symb, transform, (ws, hs), borderValue = 0)
            # Scale
            scale = np.random.uniform(0.7, 1.0)
            transform = np.float32([[scale, 0, 0],[0, scale, 0]])
            symb = cv2.warpAffine(symb, transform, (ws, hs), borderValue = 0)
            y = np.random.randint(hs, h)
            x = np.random.randint(i * pw, (i + 1) * pw - ws)
            img[y-hs:y, x:x+ws] = symb
        nw = int(w * 32 / h)
        img = cv2.resize(img, (nw, 32))
        img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
        return img, seq

    def __getitem__(self, idx):
        img, seq = self.get_sample()
        sample = {"img": img, "seq": seq, "seq_len": len(seq), "aug": self.mode == "train"}
        if self.transform:
            sample = self.transform(sample)
        return sample


text:
"""Functionality relating to statistics, timeseries, etc."""
from eofs.xarray import Eof
import numpy as np
import ruptures as rpt
from sklearn import linear_model
import sklearn.metrics
import scipy.stats
import xarray as xr

from .names import LAT_STR, LON_STR, YEAR_STR
from .nb_utils import cosdeg

# TODO: make this a decorator.
def _infer_dim_if_1d(arr, dim):
    """Helper function to get dim name of 1D arrays."""
    if dim is None:
        if arr.ndim == 1:
            dim = arr.dims[0]
        else:
            raise ValueError("Dimension must be specified if array isn't 1D.")
    return dim

# Trends: computing trends, detrending, etc.
def trend(arr, dim=None, order=1, return_coeffs=False):
    """Compute linear or higher-order polynomial fit.

    If return_coeffs is True, then coeffs.degree(sel=0) is the y-intercept,
    coeffs.degree(sel=1) is the slope, etc.

    """
    dim = _infer_dim_if_1d(arr, dim)
    coeffs = arr.polyfit(dim, order)["polyfit_coefficients"]
    if return_coeffs:
        return coeffs
    return xr.polyval(arr[dim], coeffs)

def detrend(arr, dim=None, order=1):
    """Subtract off the linear or higher order polynomial fit."""
    dim = _infer_dim_if_1d(arr, dim)
    return (arr - trend(arr, dim, order) + arr.mean(dim)).transpose(*arr.dims)

# Common timeseries transforms: anomalies, standardized anomalies, etc.
def anomaly(arr, dim=None):
    """Deviation at each point from the mean along a dimension."""
    dim = _infer_dim_if_1d(arr, dim)
    return arr - arr.mean(dim)

def standardize(arr, dim=None):
    """Anomaly normalized by the standard deviation along a dimension."""
    dim = _infer_dim_if_1d(arr, dim)
    return anomaly(arr, dim) / arr.std(dim)

def dt_std_anom(arr, dim=None, order=1):
    """Detrended standardized anomaly timeseries."""
    dim = _infer_dim_if_1d(arr, dim)
    return detrend(standardize(arr, dim), dim=dim, order=order)

# Filtering (time or otherwise)
def run_mean(arr, n=10, dim="time", center=True, **kwargs):
    """Simple running average along a dimension."""
    return arr.rolling(**{dim: n}, center=center, **kwargs).mean().dropna(dim)

def run_mean_anom(arr, n=10, dim="time", center=True, **kwargs):
    """Anomaly at each point from a running average along a dimension."""
    return arr - run_mean(arr, n, dim, center=center, **kwargs)

def xwelch(arr, **kwargs):
    """Wrapper for scipy.signal.welch for xr.DataArrays"""
    freqs, psd = scipy.signal.welch(arr, **kwargs)
    return xr.DataArray(psd, dims=["frequency"], coords={"frequency": freqs},
                        name="psd")

def welch(arr, dim="time", **welch_kwargs):
    """Use xr.apply_ufunc to broadcast scipy.signal.welch.

    For example, over latitude and longitude.

    """
    def _welch(x):
        """Wrapper around scipy.signal.welch to use in apply_ufunc."""
        freqs, densities = scipy.signal.welch(x, **welch_kwargs)
        return np.array([freqs, densities])

    arr = xr.apply_ufunc(
        _welch,
        arr,
        input_core_dims=[[dim]],
        output_core_dims=[["parameter", "frequency"]],
        vectorize=True,
        dask="parallelized",
        output_dtypes=["float64"],
        dask_gufunc_kwargs=dict(output_sizes={"parameter": 2}),
    )
    freqs = arr.isel(parameter=0, lat=0, lon=0).rename("frequency").reset_coords(drop=True)
    psd = arr.isel(parameter=1)
    ds = psd.to_dataset(name="psd")
    ds.coords["frequency"] = freqs
    return ds["psd"]

def butterworth(arr, n, windows, filttype="bandpass", dim="time"):
    """Apply the Butterworth filter to an xr.DataArray."""
    if filttype == "bandpass" and windows[0] > windows[1]:
        raise ValueError("windows need to be in (high, low) "
                         "frequency order; got (low, high)")
    b, a = scipy.signal.butter(n, windows, filttype)
    axis_num = arr.get_axis_num(dim)
    return scipy.signal.filtfilt(b, a, arr, axis=axis_num)

# Correlations and linear regression.
def corr_detrended(arr1, arr2, dim=None, order=1):
    """Correlation coefficient of two arrays after they are detrended."""
    corr = xr.corr(detrend(arr1, dim, order), detrend(arr2, dim, order), dim)
    if corr.shape == tuple():
        return float(corr)
    return corr

def autocorr(arr, lag=None, dim="time", do_detrend=False):
    """Autocorrelation on xarray.DataArrays computed for specified lag(s).

    Adapted from https://stackoverflow.com/a/21863139/1706640

    """
    if do_detrend:
        arr = detrend(arr, dim=dim)

    if np.isscalar(lag):
        if lag == 0:
            return 1.
        return np.corrcoef(np.array([arr[:-lag], arr[lag:]]))[0, 1]
    else:
        if lag is None:
            lag = range(len(arr) - 2)
        values = [autocorr(arr, l) for l in lag]
        return xr.DataArray(values, dims=["lag"], coords={"lag": lag},
                            name="autocorrelation")

def lag_corr(arr1, arr2, lag=None, dim="time", do_align=True,
             do_detrend=False):
    """Lag correlation on xarray.DataArrays computed for specified lag(s).

    Lags can be negative, zero, or positive.  Positive lag corresponds to arr1
    leading arr2, negative to arr2 leading arr1.  Lag is the desired time
    index, not the time itself.

    Adapted from https://stackoverflow.com/a/21863139/1706640

    """
    if do_align:
        arr1, arr2 = xr.align(arr1, arr2)

    if do_detrend:
        arr1 = detrend(arr1, dim=dim)
        arr2 = detrend(arr2, dim=dim)

    if np.isscalar(lag):
        if lag == 0:
            arrs = [arr1.values, arr2.values]
        elif lag > 0:
            arrs = [arr1.values[lag:], arr2.values[:-lag]]
        elif lag < 0:
            arrs = [arr1.values[:lag], arr2.values[-lag:]]
        return np.corrcoef(np.array(arrs))[0, 1]
    else:
        if lag is None:
            lag = range(min(len(arr1), len(arr2)) - 2)
        values = [lag_corr(arr1, arr2, l) for l in lag]
        return xr.DataArray(values, dims=["lag"], coords={"lag": lag},
                            name="lagged-correlation")

def lin_regress(arr1, arr2, dim):
    """Use xr.apply_ufunc to broadcast scipy.stats.linregress.

    For example, over latitude and longitude.

    Adapted from
    https://github.com/pydata/xarray/issues/1815#issuecomment-614216243.

    """
    def _linregress(x, y):
        """Wrapper around scipy.stats.linregress to use in apply_ufunc."""
        slope, intercept, r_val, p_val, std_err = scipy.stats.linregress(x, y)
        return np.array([slope, intercept, r_val, p_val, std_err])

    arr1_trunc, arr2_trunc = xr.align(arr1, arr2)

    arr = xr.apply_ufunc(
        _linregress,
        arr1_trunc,
        arr2_trunc,
        input_core_dims=[[dim], [dim]],
        output_core_dims=[["parameter"]],
        vectorize=True,
        dask="parallelized",
        output_dtypes=['float64'],
        dask_gufunc_kwargs=dict(output_sizes={"parameter": 5}),
    )
    arr.coords["parameter"] = xr.DataArray(
        ["slope", "intercept", "r_value", "p_value", "std_err"],
        dims=["parameter"],
    )
    return arr

def multi_regress(target, predictors):
    """Multiple linear regression with xarray."""
    clf_X = np.array([arr.values for arr in predictors]).transpose()
    clf = linear_model.LinearRegression()
    clf.fit(clf_X, target.values)
    return clf, xr.ones_like(target) * clf.predict(clf_X)

def rmse(arr1, arr2, dim):
    """Root mean square error using xr.apply_ufunc to broadcast.

    Adapted from
    https://github.com/pydata/xarray/issues/1815#issuecomment-614216243.

    """
    def _rmse(x, y):
        """Wrapper around scipy.stats.linregress to use in apply_ufunc."""
        return sklearn.metrics.mean_squared_error(x, y, squared=False)

    return xr.apply_ufunc(_rmse, arr1, arr2, input_core_dims=[[dim], [dim]],
                          vectorize=True, dask="parallelized")

# Breakpoint detection
def detect_breakpoint(arr, dim, rpt_class=rpt.Binseg, model="l2", n_bkps=1):
    """Use xr.apply_ufunc to broadcast breakpoint detections using ruptures"""

    rpt_instance = rpt_class(model=model)

    def _detect_bp(arr):
        """Wrapper to use in apply_ufunc."""
        return rpt_instance.fit(arr).predict(n_bkps=n_bkps)[0]

    inds_bp = xr.apply_ufunc(
        _detect_bp,
        arr,
        input_core_dims=[[dim]],
        vectorize=True,
        dask="parallelized",
    )
    return arr[dim][inds_bp]

# Empirical orthogonal functions (EOFs)
def eof_solver(arr, lat_str=LAT_STR, time_str=YEAR_STR):
    """Generate an EOF solver for gridded lat-lon data."""
    weights = np.sqrt(cosdeg(arr[lat_str])).values[..., np.newaxis]
    # The eofs package requires that the time dimension be named "time".
    return Eof(arr.rename({time_str: "time"}).transpose("time", ...),
               weights=weights)

max_stars_repo_name: spencerahill/puffins

max_issues_repo_name: spencerahill/puffins

max_forks_repo_name: spencerahill/puffins

reason: import numpy,import scipy


text:
"""Methods that use the gradient in a very direct way."""
import numpy as np

from optimus.types import DirectionMethod
from optimus.types import Function

class Gradient(DirectionMethod):
    """Just using the direction is the gradient itself."""

    def __call__(self, parameters: np.ndarray, objective_function: Function):
        gradient = objective_function.gradient(parameters)
        return gradient

max_stars_repo_name: IanTayler/tao-exercises

max_issues_repo_name: IanTayler/tao-exercises

max_forks_repo_name: IanTayler/tao-exercises


text:
import argparse
import os
import random
import subprocess

import numpy as np
import torch

import rlkit.util.hyperparameter as hyp
from rlkit.launchers.launcher_util import run_experiment

def experiment(variant):
    from a2c_ppo_acktr.main import experiment

    experiment(variant)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--exp_prefix", type=str, default="test")
    parser.add_argument("--num_seeds", type=int, default=1)
    parser.add_argument("--mode", type=str, default="local")
    parser.add_argument("--env", type=str, default="")
    parser.add_argument("--debug", action="store_true", default=False)
    args = parser.parse_args()
    if args.debug:
        exp_prefix = "test" + args.exp_prefix
    else:
        exp_prefix = args.exp_prefix
    variant = dict(
        algorithm_kwargs=dict(
            entropy_coef=0.01,
            value_loss_coef=0.5,
            lr=3e-4,
            num_mini_batch=64,
            ppo_epoch=10,
            clip_param=0.2,
            eps=1e-5,
            max_grad_norm=0.5,
        ),
        rollout_kwargs=dict(
            use_gae=True,
            gamma=0.8,
            gae_lambda=0.95,
            use_proper_time_limits=True,
        ),
        env_kwargs=dict(
            dense=False,
            image_obs=True,
            action_scale=1.4,
            use_workspace_limits=True,
            control_mode="primitives",
            imwidth=84,
            imheight=84,
            usage_kwargs=dict(
                use_dm_backend=True,
                use_raw_action_wrappers=False,
                use_image_obs=True,
                max_path_length=5,
                unflatten_images=True,
            ),
            image_kwargs=dict(),
        ),
        actor_kwargs=dict(recurrent=False, hidden_size=512, hidden_activation="relu"),
        num_processes=12,
        num_env_steps=int(5e5),
        num_steps=2048 // 12,
        log_interval=1,
        eval_interval=1,
        use_raw_actions=False,
        env_suite="kitchen",
        discrete_continuous_dist=True,
        use_linear_lr_decay=False,
    )

    search_space = {
        "env_name": [args.env],
        "discrete_continuous_dist": [False],
    }
    sweeper = hyp.DeterministicHyperparameterSweeper(
        search_space,
        default_parameters=variant,
    )
    for exp_id, variant in enumerate(sweeper.iterate_hyperparameters()):
        for _ in range(args.num_seeds):
            seed = random.randint(0, 100000)
            variant["seed"] = seed
            variant["exp_id"] = exp_id
            run_experiment(
                experiment,
                exp_prefix=args.exp_prefix,
                mode=args.mode,
                variant=variant,
                use_gpu=True,
                snapshot_mode="none",
                python_cmd=subprocess.check_output("which python", shell=True).decode(
                    "utf-8"
                )[:-1],
                seed=seed,
                exp_id=exp_id,
            )

max_stars_repo_name: mihdalal/raps

max_issues_repo_name: mihdalal/raps

max_forks_repo_name: mihdalal/raps


text:
from __future__ import print_function

import numpy as np
import pandas as pd
import xarray as xr

from xarray.testing import assert_equal

from gcpy.util import *

def test_convert_lon():
    lons_greater_zero = np.arange(0, 360)
    ones = np.ones_like(lons_greater_zero)
    lons_greater_zero = xr.DataArray(ones, dims=['lon', ],
                                     coords={'lon': lons_greater_zero})

    lons_less_zero = np.arange(-180, 180)
    lons_less_zero = xr.DataArray(ones, dims=['lon', ],
                                  coords={'lon': lons_less_zero})

    # Test atlantic
    assert_equal(convert_lon(lons_greater_zero, 'lon', 'atlantic'),
                 lons_less_zero)

    # Test pacific
    assert_equal(convert_lon(lons_less_zero, 'lon', 'pacific'),
                 lons_greater_zero)

def test_maybe_as_array():
    """ Check that we pass through expected values correctly """

    # Float -> array
    arr1 = 0.
    assert maybe_as_array(arr1) == np.asarray(arr1)

    # List of floats -> array
    arr2 = [0., 1., 2., 3.]
    res2 = maybe_as_array(arr2)
    assert isinstance(res2, np.ndarray)
    np.testing.assert_array_equal(res2, np.asarray(arr2))

    # List of list of floats -> array
    arr3 = [[0., 0.,], [1., 1.], [2., 2.]]
    res3 = maybe_as_array(arr3)
    assert isinstance(res3, np.ndarray)
    np.testing.assert_array_equal(res3, np.asarray(arr3))

    # pandas Series
    arr4 = pd.Series(arr2)
    res4 = maybe_as_array(arr4)
    assert isinstance(res4, pd.Series)

    # xarray DataArray
    arr5 = xr.DataArray(arr2)
    res5 = maybe_as_array(arr5)
    assert isinstance(res5, xr.DataArray)

max_stars_repo_name: LiamBindle/gcpy

max_stars_repo_licenses: NCSA Apache-2.0 MIT

max_issues_repo_name: LiamBindle/gcpy

max_issues_repo_licenses: NCSA Apache-2.0 MIT

max_forks_repo_name: LiamBindle/gcpy

max_forks_repo_licenses: NCSA Apache-2.0 MIT


text:
from metagraph import translator
from metagraph.plugins import has_networkx, has_scipy

if has_networkx and has_scipy:
    import networkx as nx
    import numpy as np
    from .types import NetworkXGraph
    from ..scipy.types import ScipyGraph

    @translator
    def graph_from_scipy(x: ScipyGraph, **props) -> NetworkXGraph:
        from ..python.types import dtype_casting

        aprops = ScipyGraph.Type.compute_abstract_properties(
            x, {"is_directed", "edge_type", "edge_dtype", "node_type", "node_dtype"}
        )

        nx_graph = nx.from_scipy_sparse_matrix(
            x.value,
            create_using=nx.DiGraph if aprops["is_directed"] else nx.Graph,
            edge_attribute="weight",
        )

        if aprops["edge_type"] == "set":
            # Remove weight attribute
            for _, _, attr in nx_graph.edges(data=True):
                del attr["weight"]
        else:
            caster = dtype_casting[aprops["edge_dtype"]]
            for _, _, attr in nx_graph.edges(data=True):
                attr["weight"] = caster(attr["weight"])

        is_sequential_node_list = (x.node_list == np.arange(len(x.node_list))).all()
        if not is_sequential_node_list:
            pos2id = dict(enumerate(x.node_list))
            nx.relabel_nodes(nx_graph, pos2id, False)

        if x.node_vals is not None:
            caster = dtype_casting[aprops["node_dtype"]]
            node_weights = {
                idx: caster(val) for idx, val in zip(x.node_list, x.node_vals)
            }
            nx.set_node_attributes(nx_graph, node_weights, name="weight")

        return NetworkXGraph(nx_graph, aprops=aprops)

max_stars_repo_name: paul-tqh-nguyen/metagraph

max_issues_repo_name: paul-tqh-nguyen/metagraph

max_forks_repo_name: paul-tqh-nguyen/metagraph

reason: import numpy,import networkx


text:
import itertools
import os.path as osp
import sys
from random import triangular
from numpy.lib.function_base import append
import pandas as pd
import numpy as np
import torch
import torch.nn.functional as F
from torch.utils.data import Dataset,DataLoader
import torch.nn as nn
from torch_geometric.datasets import Planetoid
import torch_geometric.transforms as T
from torch_geometric.nn import ARMAConv,AGNNConv
from torch_geometric.nn import GINConv,GATConv,GCNConv
from torch_geometric.nn import SAGEConv,SplineConv
from torch_geometric.nn import global_mean_pool
from model.aug_GNN import NeuralTensorNetwork

class StrucFeaGNN(nn.Module):
    def __init__(self, concat_fea = True, concat_fea_num = 2, embed_method = 'GIN', input_dim = 1024, output_dim = 7, depth = 2, cat_method = 'SimpleConcat', required_batch = True, embed_dim = 8):
        super(StrucFeaGNN, self).__init__()
        self.concat_fea_num = concat_fea_num
        self.embed_method = embed_method
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.depth = depth
        self.concat_fea = concat_fea
        self.cat_method = cat_method
        self.required_batch = required_batch
        self.embed_dim = embed_dim
        in_dim = 0
        if self.concat_fea:
            in_dim = self.input_dim - self.concat_fea_num + self.embed_dim * 2
        else:
            in_dim = self.input_dim - self.concat_fea_num

        self.pre_mlp = nn.ModuleList()
        self.bilinear = nn.ModuleList()
        self.ntn = nn.ModuleList()
        if self.concat_fea_num == 3:
            self.e_mlp = nn.Linear(self.embed_dim * 3, self.embed_dim * 2)
        elif self.concat_fea_num == 2:
            self.e_mlp = nn.Linear(self.embed_dim * 2, self.embed_dim * 2)
        for i in range(self.concat_fea_num):
            self.pre_mlp.append(nn.Linear(1, self.embed_dim))
        for i in range(self.concat_fea_num-1):
            if i == self.concat_fea_num-2:
                self.bilinear.append(nn.Bilinear(self.embed_dim, self.embed_dim, self.embed_dim * 2))
                self.ntn.append(NeuralTensorNetwork(self.embed_dim, self.embed_dim * 2))
            else:
                self.bilinear.append(nn.Bilinear(self.embed_dim, self.embed_dim, self.embed_dim))
                self.ntn.append(NeuralTensorNetwork(self.embed_dim, self.embed_dim))

        if self.embed_method != 'MLP':
            self.post_mlp1 = nn.Linear(self.embed_dim * 4, self.embed_dim * 2)
        else:
            self.post_mlp1 = nn.Linear(in_dim, self.embed_dim * 2)
        self.post_mlp2 = nn.Linear(self.embed_dim * 2, output_dim)

        #self.bilinear = nn.Bilinear(self.embed_dim * 2, self.embed_dim * 2,self.embed_dim * 4)
        #self.ntn = NeuralTensorNetwork(self.embed_dim * 2, self.embed_dim * 4)
        self.convs = nn.ModuleList()
        self.mlps = nn.ModuleList()
        self.batch_norms = nn.ModuleList()

        for i in range(self.depth):
            if i == 0:
                self.mlps.append(nn.Sequential(
                nn.Linear(in_dim, self.embed_dim * 4),
                nn.BatchNorm1d(self.embed_dim * 4),
                nn.ReLU(),
                nn.Linear(self.embed_dim * 4, self.embed_dim * 4),))
            else:
                self.mlps.append(nn.Sequential(
                nn.Linear(self.embed_dim * 4, self.embed_dim * 4),
                nn.BatchNorm1d(self.embed_dim * 4),
                nn.ReLU(),
                nn.Linear(self.embed_dim * 4, self.embed_dim * 4),))
            self.batch_norms.append(
                            nn.BatchNorm1d(self.embed_dim * 4,)
            )

        for i in range(self.depth):
            if embed_method == 'GIN':
                self.convs.append(GINConv(self.mlps[i]))
            elif embed_method == 'GCN':
                if i == 0:
                    self.convs.append(GCNConv(in_dim, self.embed_dim * 4, cached=True))
                else:
                    self.convs.append(GCNConv(self.embed_dim * 4, self.embed_dim * 4, cached=True))
            elif embed_method == 'GAT':
                if i == 0:
                    self.convs.append(GATConv(in_dim, self.embed_dim * 4, heads= 4, concat = False, dropout= 0.4))
                else:
                    self.convs.append(GATConv(self.embed_dim * 4, self.embed_dim * 4, heads= 4, concat = False, dropout= 0.4))
            elif embed_method == 'SAGE':
                if i == 0:
                    self.convs.append(SAGEConv(in_dim, self.embed_dim * 4, normalize=True))
                else:
                    self.convs.append(SAGEConv(self.embed_dim * 4,self.embed_dim * 4, normalize=True))
            elif embed_method == 'None' or embed_method == 'MLP' :
                pass
            else:
                print("please give four of the embedding methods: GIN, GAT, GCN, SAGE")
                sys.exit(0)

    def forward(self, data):

        ident_idx = self.input_dim - self.concat_fea_num

        ident_vec = data.x[:,:ident_idx]
        #print(ident_vec.shape)
        struc_vec = data.x[:,-self.concat_fea_num:]
       # print(struc_vec.shape)
        _x = []
        for i in range(self.concat_fea_num):
            _x.append(F.relu(self.pre_mlp[i](struc_vec[:, i:i+1]))) # two or three N * self.embed_dim

        for i in range(self.concat_fea_num-1):
            if self.cat_method == 'SimpleConcat':
                if i == 0:
                    x = torch.cat((_x[i], _x[i+1]), dim = 1)
                else:
                    x = torch.cat((x, _x[i + 1]), dim = 1)
                if  i == self.concat_fea_num-2:
                    x = self.e_mlp(x)

            elif self.cat_method == 'Bilinear':
                if i == 0:
                    x = self.bilinear[i](_x[i], _x[i+1])
                else:
                    x = self.bilinear[i](x, _x[i+1])
            elif self.cat_method == 'NTN':
                if i == 0:
                    x = self.ntn[i](_x[i], _x[i+1])
                else:
                    x = self.ntn[i](x, _x[i+1])

        init_x = ident_vec
        #print(init_x.shape)
        new_x = torch.cat((init_x, x), dim = 1) if self.concat_fea == True else init_x
        #print(new_x.shape)
        if self.embed_method != 'MLP':
            # two gnn layers
            graph_embed_0 = self.convs[0](new_x, data.edge_index)
            graph_embed_0 = F.relu(graph_embed_0)
            graph_embed_0 = F.dropout(graph_embed_0, training=self.training)
            #graph_embed_0+=new_x # skip connection
            graph_embed_1 = self.convs[1](graph_embed_0, data.edge_index)
            graph_embed_1 = F.relu(graph_embed_1)
            graph_embed_1 = F.dropout(graph_embed_1, p = 0.6, training=self.training)
            graph_embed_1 = graph_embed_1 + graph_embed_0# skip connection
            graph_embed_2 = self.convs[2](graph_embed_1, data.edge_index)
            #graph_embed_2 = F.relu(graph_embed_2)
            graph_embed_2 = F.dropout(graph_embed_2, p = 0.6, training=self.training)
            graph_embed_2 = graph_embed_2 + graph_embed_1 + graph_embed_0# skip connection
            # readout layer
            if self.required_batch:
                tmp = global_mean_pool(graph_embed_2, data.batch)
                tmp = F.dropout(tmp, p= 0.5, training=self.training)
            else:
                tmp = graph_embed_2
        else:
            if self.required_batch:
                tmp = global_mean_pool(new_x, data.batch)
                tmp = F.dropout(tmp, p= 0.5, training=self.training)
            else:
                tmp = new_x

        output = F.relu(self.post_mlp1(tmp))
        output = self.post_mlp2(output)

        return F.log_softmax(output, dim =1)

reason: import numpy,from numpy


text:
import numpy as np
import tensorflow as tf

from tensorstream.finance.returns import Return, LogarithmicReturn
from tensorstream.tests import TestCase

class ReturnSpec(TestCase):
  def setUp(self):
    self.sheets = self.read_ods(self.from_test_res('returns.ods', __file__))

  def test_return_single_dim(self):
    single_dim_ts = self.sheets['Sheet1']
    prices = tf.placeholder(tf.float32)
    return_ = Return(5)

    return_ts, _, _ = return_(prices)

    with tf.Session() as sess:
      output = sess.run(return_ts, {
        prices: single_dim_ts['Prices']
      })

    np.testing.assert_almost_equal(output,
      single_dim_ts['Return 5d'].values, decimal=4)

class LogarithmicReturnSpec(TestCase):
  def setUp(self):
    self.sheets = self.read_ods(self.from_test_res('logarithmic_returns.ods', __file__))

  def test_log_return_single_dim(self):
    single_dim_ts = self.sheets['Sheet1']
    prices = tf.placeholder(tf.float32)
    log_return = LogarithmicReturn(5)
    log_return_ts, _, _ = log_return(prices)

    with tf.Session() as sess:
      output = sess.run(log_return_ts, {
        prices: single_dim_ts['Prices']
      })

    np.testing.assert_almost_equal(output,
      single_dim_ts['Log Return 5d'].values, decimal=4)


text:
import datetime
import json
import logging
import random
import math
import itertools
import numpy as np

logging.basicConfig(level=logging.DEBUG, filename='/home/wechat/wechatbot/output.log')
logger = logging.getLogger(__name__)

'''这样记录日志
logger.info('This is a log info')
logger.debug('Debugging')
logger.warning('Warning exists')
'''

logger.info('helloworld')

name = '2020毕业热爱'
model = r'frog2.0启动！淦！已经\h点\m分了！你今天学习了吗？'

# 获取报时内容
def gettext():
    response_text = ''
    status = False
    for c in model:
        if not status:
            if c == chr(92):
                status = True
            else:
                response_text += c
        else:
            status = False
            if c == chr(92):
                response_text += c
            elif c == 'h':
                response_text += str(datetime.datetime.now().hour)
            elif c == 'm':
                response_text += str(datetime.datetime.now().minute)
            elif c == 's':
                response_text += str(datetime.datetime.now().second)
            else:
                pass
    return response_text

def judgePoint24(nums):
    if len(nums) == 1:
        return math.isclose(nums[0], 24)
    return any(judgePoint24([x] + rest)
                for a, b, *rest in itertools.permutations(nums)
                for x in {a+b, a-b, a*b, b and a/b})

def handle(data):
    if data['mentionSelf']:
        global model
        text = data['text'].replace('@青蛙 ', '')

        # 测试开始
        #return json.dumps({'type': 'text', 'content': eval(text)})
        # 测试结束

        if len(text) >= 6 and text[:4] == '修改模板':
            model = text[5:]
            return json.dumps({'type': 'text', 'content': '修改成功！现在的模板是：\n'+model})
        elif text == '报时':
            return json.dumps({'type': 'text', 'content': gettext()})
        elif len(text) >= 4  and text[:2] == '加法':
            add1 = text.split(' ')[1]
            add2 = text.split(' ')[2]
            try:
                answer = int(add1) + int(add2)
                return json.dumps({'type': 'text', 'content': '算好了哦!答案是: '+str(answer)})
            except:
                return json.dumps({'type': 'text', 'content': '淦！你给我的是啥?'})

        elif text == '24点':
            num_list = []
#             for i in range(4):
#                 num_list.append(str(random.randint(1,9)))
#                 res = ' '.join(num_list)
            array_data = np.loadtxt('24p_1_9.txt')
            len(list(array_data))
            id=np.random.randint(low=0, high=len(list(array_data)))
            choice = list(array_data)[id]
            choice = [int(c) for c in choice]
            num_str_list = [str(c) for c in choice]
            res = ' '.join(num_str_list)
            return json.dumps({'type':'text','content':'4 numbers:'+res})

        elif len(text) >= 4  and text[:2] == '乘法':
            add1 = text.split(' ')[1]
            add2 = text.split(' ')[2]
            try:
                answer = float(add1) * float(add2)
                return json.dumps({'type': 'text', 'content': '算好了哦!答案是: '+str(answer)})
            except:
                return json.dumps({'type': 'text', 'content': '淦！你给我的是啥?'})
        else:
            return json.dumps({'type': 'null'})
    else:
        return json.dumps({'type': 'null'})


text:
# -*- coding: utf-8 -*-
"""
Types for closures and inner functions.
"""
from __future__ import print_function, division, absolute_import

from numba.minivect import minitypes
from numba.typesystem.basetypes import NumbaType
from numba.exttypes.types.extensiontype import ExtensionType

class ClosureType(NumbaType, minitypes.ObjectType):
    """
    Type of closures and inner functions.
    """

    is_closure = True

    def __init__(self, signature, **kwds):
        super(ClosureType, self).__init__(**kwds)
        self.signature = signature
        self.closure = None

    def add_scope_arg(self, scope_type):
        self.signature.args = (scope_type,) + self.signature.args

    def __repr__(self):
        return "<closure(%s)>" % self.signature

class ClosureScopeType(ExtensionType):
    """
    Type of the enclosing scope for closures. This is always passed in as
    first argument to the function.
    """

    is_closure_scope = True
    is_final = True

    def __init__(self, py_class, parent_scope, **kwds):
        super(ClosureScopeType, self).__init__(py_class, **kwds)
        self.parent_scope = parent_scope
        self.unmangled_symtab = None

        if self.parent_scope is None:
            self.scope_prefix = ""
        else:
            self.scope_prefix = self.parent_scope.scope_prefix + "0"

    def __repr__(self):
        return "closure_scope(%s)" % self.attribute_table

max_stars_repo_name: shiquanwang/numba

max_issues_repo_name: shiquanwang/numba

max_forks_repo_name: shiquanwang/numba

reason: from numba


text:
import os, atexit
import logging
from collections import defaultdict
import numpy as np
import tensorflow as tf

from core.log import do_logging
from utility.display import pwc
from utility.graph import image_summary, video_summary
from utility.utils import isscalar

logger = logging.getLogger(__name__)

""" Recorder """
class Recorder:
    def __init__(self, recorder_dir=None, record_file='record.txt'):
        """
        Initialize a Recorder.

        Args:
            log_dir (string): A directory for saving results to. If
                `None/False`, Recorder only serves as a storage but
                doesn't write anything to the disk.

            record_file (string): Name for the tab-separated-value file
                containing metrics logged throughout a training run.
                Defaults to "log.txt".
        """
        record_file = record_file if record_file.endswith('record.txt') \
            else record_file + '/record.txt'
        self._recorder_dir = recorder_dir
        if self._recorder_dir:
            path = os.path.join(self._recorder_dir, record_file)
            # if os.path.exists(path) and os.stat(path).st_size != 0:
            #     i = 1
            #     name, suffix = path.rsplit('.', 1)
            #     while os.path.exists(name + f'{i}.' + suffix):
            #         i += 1
            #     pwc(f'Warning: Log file "{path}" already exists!',
            #         f'Data will be logged to "{name + f"{i}." + suffix}" instead.',
            #         color='magenta')
            #     path = name + f"{i}." + suffix
            if not os.path.isdir(self._recorder_dir):
                os.makedirs(self._recorder_dir)
            self._out_file = open(path, 'a')
            atexit.register(self._out_file.close)
            pwc(f'Record data to "{self._out_file.name}"', color='green')
        else:
            self._out_file = None
            pwc(f'Record directory is not specified, '
                'no data will be recorded to the disk',
                color='magenta')

        self._first_row=True
        self._headers = []
        self._current_row = {}
        self._store_dict = defaultdict(list)

    def __contains__(self, item):
        return item in self._store_dict and self._store_dict[item] != []

    def contains_stats(self, item):
        return item in self._store_dict and self._store_dict[item] != []

    def store(self, **kwargs):
        for k, v in kwargs.items():
            if isinstance(v, tf.Tensor):
                v = v.numpy()
            if v is None:
                return
            elif isinstance(v, (list, tuple)):
                self._store_dict[k] += list(v)
            else:
                self._store_dict[k].append(v)

    """ All get functions below will remove the corresponding items from the store """
    def get_raw_item(self, key):
        if key in self._store_dict:
            v = self._store_dict[key]
            del self._store_dict[key]
            return {key: v}
        return None

    def get_item(self, key, mean=True, std=False, min=False, max=False):
        stats = {}
        if key not in self._store_dict:
            return stats
        v = self._store_dict[key]
        if isscalar(v):
            stats[key] = v
            return
        if mean:
            stats[f'{key}'] = np.mean(v).astype(np.float32)
        if std:
            stats[f'{key}_std'] = np.std(v).astype(np.float32)
        if min:
            stats[f'{key}_min'] = np.min(v).astype(np.float32)
        if max:
            stats[f'{key}_max'] = np.max(v).astype(np.float32)
        del self._store_dict[key]
        return stats

    def get_raw_stats(self):
        stats = self._store_dict.copy()
        self._store_dict.clear()
        return stats

    def get_stats(self, mean=True, std=False, min=False, max=False):
        stats = {}
        for k in sorted(self._store_dict):
            v = self._store_dict[k]
            k_std, k_min, k_max = std, min, max
            if k.startswith('train/') or k.startswith('stats/'):
                k_std = k_min = k_max = True
            if isscalar(v):
                stats[k] = v
                continue
            if mean:
                stats[f'{k}'] = np.mean(v).astype(np.float32)
            if k_std:
                stats[f'{k}_std'] = np.std(v).astype(np.float32)
            if k_min:
                stats[f'{k}_min'] = np.min(v).astype(np.float32)
            if k_max:
                stats[f'{k}_max'] = np.max(v).astype(np.float32)
        self._store_dict.clear()
        return stats

    def get_count(self, name):
        return len(self._store_dict[name])

    def record_stats(self, stats, print_terminal_info=True):
        if not self._first_row and not set(stats).issubset(set(self._headers)):
            if self._first_row:
                do_logging(f'All previous records are erased because stats does not match the first row\n'
                    f'stats = {set(stats)}\nfirst row = {set(self._headers)}',
                    logger=logger, level='WARNING')
            self._out_file.seek(0)
            self._out_file.truncate()
            self._headers.clear()
            self._first_row = True
        [self.record_tabular(k, v) for k, v in stats.items()]
        self.dump_tabular(print_terminal_info=print_terminal_info)

    def _record_tabular(self, key, val):
        """
        Record a value of some diagnostic.

        Call this only once for each diagnostic quantity, each iteration.
        After using ``record_tabular`` to store values for each diagnostic,
        make sure to call ``dump_tabular`` to write them out to file and
        stdout (otherwise they will not get saved anywhere).
        """
        if self._first_row:
            self._headers.append(key)
        else:
            assert key in self._headers, \
                f"Trying to introduce a new key {key} " \
                "that you didn't include in the first iteration"
        assert key not in self._current_row, \
            f"You already set {key} this iteration. " \
            "Maybe you forgot to call dump_tabular()"
        self._current_row[key] = val

    def record_tabular(self, key, val=None, mean=True, std=False, min=False, max=False):
        """
        Record a value or possibly the mean/std/min/max values of a diagnostic.
        """
        if val is not None:
            self._record_tabular(key, val)
        else:
            v = np.asarray(self._store_dict[key])
            if mean:
                self._record_tabular(f'{key}_mean', np.mean(v))
            if std:
                self._record_tabular(f'{key}_std', np.std(v))
            if min:
                self._record_tabular(f'{key}_min', np.min(v))
            if max:
                self._record_tabular(f'{key}_max', np.max(v))
        self._store_dict[key] = []

    def dump_tabular(self, print_terminal_info=True):
        """
        Write all of the diagnostics from the current iteration.
        """
        vals = []
        key_lens = [len(key) for key in self._headers]
        max_key_len = max(15,max(key_lens))
        n_slashes = 22 + max_key_len
        if print_terminal_info:
            print("-"*n_slashes)
        for key in self._headers:
            val = self._current_row.get(key, "")
            # print(key, np.array(val).dtype)
            valstr = f"{val:8.3g}" if hasattr(val, "__float__") else val
            if print_terminal_info:
                print(f'| {key:>{max_key_len}s} | {valstr:>15s} |')
            vals.append(val)
        if print_terminal_info:
            print("-"*n_slashes)
        if self._out_file is not None:
            if self._first_row:
                self._out_file.write("\t".join(self._headers)+"\n")
            self._out_file.write("\t".join(map(str,vals))+"\n")
            self._out_file.flush()
        self._current_row.clear()
        self._store_dict.clear()
        self._first_row=False

""" Tensorboard Writer """
class TensorboardWriter:
    def __init__(self, root_dir, model_name, name):
        self._writer = create_tb_writer(root_dir, model_name)
        self.name = name
        tf.summary.experimental.set_step(0)

    def set_summary_step(self, step):
        """ Sets tensorboard step """
        set_summary_step(step)

    def scalar_summary(self, stats, prefix=None, step=None):
        """ Adds scalar summary to tensorboard """
        scalar_summary(self._writer, stats, prefix=prefix, step=step)

    def histogram_summary(self, stats, prefix=None, step=None):
        """ Adds histogram summary to tensorboard """
        histogram_summary(self._writer, stats, prefix=prefix, step=step)

    def graph_summary(self, sum_type, *args, step=None):
        """ Adds graph summary to tensorboard
        Args:
            sum_type str: either "video" or "image"
            args: Args passed to summary function defined in utility.graph,
                of which the first must be a str to specify the tag in Tensorboard
        """
        assert isinstance(args[0], str), f'args[0] is expected to be a name string, but got "{args[0]}"'
        args = list(args)
        args[0] = f'{self.name}/{args[0]}'
        graph_summary(self._writer, sum_type, args, step=step)

    def video_summary(self, video, step=None):
        video_summary(f'{self.name}/sim', video, step=step)

    def flush(self):
        self._writer.flush()

""" Recorder Ops """
def record_stats(recorder, stats, print_terminal_info=True):
    [recorder.record_tabular(k, v) for k, v in stats.items()]
    recorder.dump_tabular(print_terminal_info=print_terminal_info)

def store(recorder, **kwargs):
    recorder.store(**kwargs)

def get_raw_item(recorder, key):
    return recorder.get_raw_item(key)

def get_item(recorder, key, mean=True, std=False, min=False, max=False):
    return recorder.get_item(key, mean=mean, std=std, min=min, max=max)

def get_raw_stats(recorder):
    return recorder.get_raw_stats()

def get_stats(recorder, mean=True, std=False, min=False, max=False):
    return recorder.get_stats(mean=mean, std=std, min=min, max=max)

def contains_stats(recorder, key):
    return key in recorder

def create_recorder(root_dir, model_name):
    recorder_dir = root_dir and f'{root_dir}/{model_name}'
    # recorder save stats in f'{root_dir}/{model_name}/logs/record.txt'
    recorder = Recorder(recorder_dir)
    return recorder

""" Tensorboard Ops """
def set_summary_step(step):
    tf.summary.experimental.set_step(step)

def scalar_summary(writer, stats, prefix=None, step=None):
    if step is not None:
        tf.summary.experimental.set_step(step)
    prefix = prefix or 'stats'
    with writer.as_default():
        for k, v in stats.items():
            if isinstance(v, str):
                continue
            if '/' not in k:
                k = f'{prefix}/{k}'
            # print(k, np.array(v).dtype)
            tf.summary.scalar(k, tf.reduce_mean(v), step=step)

def histogram_summary(writer, stats, prefix=None, step=None):
    if step is not None:
        tf.summary.experimental.set_step(step)
    prefix = prefix or 'stats'
    with writer.as_default():
        for k, v in stats.items():
            if isinstance(v, (str, int, float)):
                continue
            tf.summary.histogram(f'{prefix}/{k}', v, step=step)

def graph_summary(writer, sum_type, args, step=None):
    """ This function should only be called inside a tf.function """
    fn = {'image': image_summary, 'video': video_summary}[sum_type]
    if step is None:
        step = tf.summary.experimental.get_step()
    def inner(*args):
        tf.summary.experimental.set_step(step)
        with writer.as_default():
            fn(*args)
    return tf.numpy_function(inner, args, [])

def create_tb_writer(root_dir, model_name):
    # writer for tensorboard summary
    # stats are saved in directory f'{root_dir}/{model_name}'
    writer = tf.summary.create_file_writer(
        f'{root_dir}/{model_name}', max_queue=1000, flush_millis=20000)
    writer.set_as_default()
    return writer

def create_tensorboard_writer(root_dir, model_name, name):
    return TensorboardWriter(root_dir, model_name, name)


text:
# *******************************************************************************
# Copyright (C) 2021 INAF
#
# This software is distributed under the terms of the BSD-3-Clause license
#
# Authors:
# Ambra Di Piano <ambra.dipiano@inaf.it>
# *******************************************************************************

import os
import sys
import argparse
import numpy as np
from time import time
from shutil import copy
from astropy.io import fits
from multiprocessing import Pool
from os.path import isdir, join, isfile
from RTAscience.cfg.Config import Config
from RTAscience.lib.RTAManageXml import ManageXml
from RTAscience.lib.RTACtoolsSimulation import RTACtoolsSimulation, make_obslist
from RTAscience.lib.RTACtoolsAnalysis import RTACtoolsAnalysis
from RTAscience.lib.RTAUtils import get_alert_pointing_gw, get_mergermap, get_pointing, str2bool

def main(args):
    cfg = Config(args.cfgfile)
    # GRB ---!
    if cfg.get('runid') == 'all':
        runids = [f.replace('.fits', '') for f in os.listdir(cfg.get('catalog')) if isfile(join(cfg.get('catalog'), f))]
    elif type(cfg.get('runid')) == str:
        runids = [cfg.get('runid')]
    else:
        runids = cfg.get('runid')
    runids = sorted(runids)

    # general ---!
    trials = cfg.get('trials')
    tmax = cfg.get('tobs')-cfg.get('onset')+cfg.get('delay')

    # paths ---!
    datapath = cfg.get('data')
    if not isdir(datapath):  # main data folder
        raise ValueError('Please specify a valid path')
    if not isdir(join(datapath, 'obs')):  # obs parent folder
        os.mkdir(join(datapath, 'obs'))
    # background model ---!
    bkg_model = cfg.get('bkg')

    # ------------------------------------------------------- loop runid --- !!!
    for runid in runids:
        print(f"{'-'*50} #\nProcessing runid: {runid}")
        # grb path ---!
        grbpath = join(datapath, 'obs', runid)  # folder that will host the phlist
        if not isdir(grbpath):
            os.mkdir(grbpath)
        modelpath = join(datapath, f'extracted_data/{runid}')  # bin model folder
        if not isdir(modelpath):
            raise ValueError(f'Folder {runid} not found in {modelpath}')
        tcsv = join(datapath, f'extracted_data/{runid}/time_slices.csv')  # times table
        if not isfile(tcsv):
            raise ValueError(f'Data from {runid} have not been correctly extracted.')
        mergerpath = os.path.expandvars(cfg.get('merger'))
        mergermap = get_mergermap(runid, mergerpath)
        if mergermap == None:
            print(f'Skip runid {runid}. ')
            continue

        # get alert pointing
        if type(cfg.get('offset')) == str and cfg.get('offset').lower() == 'gw':
            pointing = get_alert_pointing_gw(mergermap)
        else:
            pointing = list(get_pointing(f"{os.path.expandvars(cfg.get('catalog'))}/{runid}.fits"))
            if pointing[1] < 0:
                pointing[0] += 0.0
                pointing[1] += -cfg.get('offset')
            else:
                pointing[0] += 0.0
                pointing[1] += cfg.get('offset')

        # Dumping the Conf object to txt file
        dumpedConfig = os.path.join(grbpath, "config.yaml")
        if not os.path.isfile(dumpedConfig):
            copy(args.cfgfile, str(dumpedConfig))

        # ---------------------------------------------------- loop trials ---!!!
        if args.mp_enabled:
            with Pool(args.mp_threads) as p:
                times = p.map(simulateTrial, [ (i, cfg, pointing, tmax, datapath, runid, tcsv, grbpath, bkg_model) for i in range(trials)])
        else:
            for i in range(trials):
                times = simulateTrial((i, cfg, pointing, tmax, datapath, runid, tcsv, grbpath, bkg_model))
        # time ---!
        if args.print:
            if len(times) > 1:
                print(f"Trial elapsed time (mean): {np.array(times).mean()}")
            else:
                print(f"Trial elapsed time: {times[0]}")
        print('\n... done.\n')

def simulateTrial(trial_args):
    start_t = time()
    i=trial_args[0]
    cfg=trial_args[1]
    pointing=trial_args[2]
    tmax=trial_args[3]
    datapath=trial_args[4]
    runid=trial_args[5]
    tcsv=trial_args[6]
    grbpath=trial_args[7]
    bkg_model=trial_args[8]
    # initialise ---!
    count = cfg.get('start_count') + i + 1
    name = f'ebl{count:06d}'
    # setup ---!
    sim = RTACtoolsSimulation()
    if type(cfg.get('caldb')) == list:
        sim.caldb = cfg.get('caldb')[0]
    else:
        sim.caldb = cfg.get('caldb')
    if type(cfg.get('irf')) == list:
        sim.irf = cfg.get('irf')[0]
    else:
        sim.irf = cfg.get('irf')
    sim.fov = cfg.get('roi')
    sim.e = [cfg.get('emin'), cfg.get('emax')]
    sim.seed = count
    sim.set_ebl = cfg.get('set_ebl')
    sim.pointing = pointing
    if args.print:
        print(f'Pointing = {sim.pointing} s')
    sim.tmax = tmax

    # get time grid ---!
    sim.template = join(os.path.expandvars(cfg.get('catalog')).replace(cfg.get('data'), datapath), f'{runid}.fits')
    event_bins = []
    sim.table = tcsv
    tgrid, tbin_start, tbin_stop = sim.getTimeSlices(GTI=(cfg.get('delay'), tmax), return_bins=True)

    # -------------------------------------------------------- simulate ---!!!
    print(f'Simulate template seed={sim.seed}')
    for j in range(tbin_stop-tbin_start-1):
        sim.t = [tgrid[j]+cfg.get('onset'), tgrid[j + 1]+cfg.get('onset')]
        if args.print:
            print(f'GTI (bin) = {sim.t} s')
        sim.model = join(datapath, f'extracted_data/{runid}/{runid}_tbin{tbin_start+j:02d}.xml')
        event = join(grbpath, f'{name}_tbin{tbin_start+j:02d}.fits')
        event_bins.append(event)
        sim.output = event
        sim.run_simulation()
    # -------------------------------------------- shift time --- !!!
    if cfg.get('onset') != 0:
        if cfg.get('delay') != 0:
            raise ValueError('Bad configuration. Either "onset" or "delay" must be equal to 0.')
        # ------------------------------------ add background --- !!!
        print('Simulate bkg to append before the burst')
        bkg = os.path.join(grbpath, f'bkg{count:06d}.fits')
        event_bins.insert(0, bkg)
        sim.t = [0, cfg.get('onset')]
        if args.print:
            print(f"GTI (bkg) = {sim.t} s")
        sim.model = bkg_model
        sim.output = bkg
        sim.run_simulation()

    # ---------------------------------------- gather bins ---!!!
    if args.merge:
        print('Merge in photon-list')
        phlist = join(grbpath, f'{name}.fits')
        sim.input = event_bins
        sim.output = phlist
        sim.appendEventsSinglePhList(GTI=[cfg.get('delay'), cfg.get('delay')+cfg.get('tobs')])
        if args.print:
            h = fits.open(phlist)
            print('Check GTI and EVENTS time range:')
            print('************')
            print(h[2].data)
            print(h[1].data.field('TIME').min(), h[1].data.field('TIME').max())
            print('************')
            h.close()
    else:
        # observation list ---!
        obslist = join(grbpath, f'{name}.xml')
        if os.path.isfile(obslist):
            os.remove(obslist)
        make_obslist(obslist=obslist, items=event_bins, names=name)

    sim.input = phlist
    sim.sortObsEvents()
    del sim

    # selections ---!
    """for texp in cfg.get('exposure'):
        selphlist = phlist.replace(f'{name}', f'texp{texp}s_{name}')
        grb = RTACtoolsAnalysis()
        grb.caldb = cfg.get('caldb')
        grb.irf = cfg.get('irf')
        grb.roi = cfg.get('roi')
        grb.e = [cfg.get('emin'), cfg.get('emax')]
        grb.t = [cfg.get('delay'), cfg.get('delay')+texp]
        if args.print:
            print(f"Selection t = {grb.t} s")
        grb.input = phlist
        grb.output = selphlist
        if args.merge:
            grb.run_selection()
        else:
            prefix = join(grbpath, f'texp{texp}s_')
            grb.run_selection(prefix=prefix) """

    # remove files ---!
    if args.remove and args.merge:
        # remove bins ---!
        os.system('rm ' + join(grbpath, f'{name}*tbin*'))
        if cfg.get('onset') != 0:
            # remove bkg ---!
            os.system('rm ' + join(grbpath, f'{name.replace("ebl", "bkg")}*'))

    # time ---!
    elapsed_t = time()-start_t
    if args.print:
        print(f"Trial {count} took {elapsed_t} seconds")
    return (count, elapsed_t)

if __name__=='__main__':
    parser = argparse.ArgumentParser(description='ADD SCRIPT DESCRIPTION HERE')
    parser.add_argument('-f', '--cfgfile', type=str, required=True, help="Path to the yaml configuration file")
    parser.add_argument('--merge', type=str2bool, default=True, help='Merge in single phlist (true) or use observation library (false)')
    parser.add_argument('--remove', type=str2bool, default=True, help='Keep only outputs')
    parser.add_argument('--print', type=str2bool, default=False, help='Print out results')
    parser.add_argument('-mp', '--mp-enabled', type=str2bool, default=False, help='To parallelize trials loop')
    parser.add_argument('-mpt', '--mp-threads', type=int, default=4, help='The size of the threads pool')
    args = parser.parse_args()

    if args.remove and not args.merge:
        raise ValueError('Keyword "remove" cannot be True if keyword "merge" is False.')
    main(args)

max_stars_repo_name: ambra-dipiano/cta-sag-sci

max_issues_repo_name: ambra-dipiano/cta-sag-sci

max_forks_repo_name: ambra-dipiano/cta-sag-sci

reason: import numpy,from astropy


text:
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import dash_katex

import numpy as np
import plotly.express as px
from scipy import stats

from app import app

layout = html.Div([
    dash_katex.DashKatex(
        expression=r'f_X(x) = \frac{1}{b - a}',
        displayMode=True
    ),
    dcc.Graph(id='uniform_graph'),
    dash_katex.DashKatex(expression=r'a, b'),
    dcc.RangeSlider(
        id='uniform_interval',
        min=-10,
        max=10,
        marks={
            x: str(x) for x in range(-10, 11)
        },
        step=0.01,
        value=[2, 4],
        allowCross=False,
        tooltip={'placement': 'top'}
    )
])

@app.callback(
    Output('uniform_graph', 'figure'),
    [Input('uniform_interval', 'value')]
)
def plot(interval):
    a, b = interval
    x = np.linspace(a, b, 1000)
    y = stats.uniform.pdf(x, a, b - a)
    range_x=[-11, 11]
    range_y=[-0.2, max(1.2, max(y) + 0.2)]
    figure = px.line(x=x, y=y, range_x=range_x, range_y=range_y)
    return figure

max_stars_repo_name: leotappe/distributions

max_issues_repo_name: leotappe/distributions

max_forks_repo_name: leotappe/distributions

reason: import numpy,from scipy


text:
# set up environment
# define command line parameters
# define location of input data
# create output directories
# start the class FeatureCorrelations

import argparse
import os

import matplotlib
matplotlib.use("Agg")

import matplotlib.pyplot as plt
import csv
import pathlib
import numpy as np
import pandas as pd
import seaborn as sns
from datetime import datetime
from sklearn.model_selection import train_test_split

###############################################################################
#
#  define command line arguments
#
###############################################################################

def parse_command_line():
  '''defining the command line input to make it runable'''
  parser = argparse.ArgumentParser(description='various plots for feature analysis')

  parser.add_argument(
    '--input',
    type=str,
    dest="input",
    default="",
    help='The input CSV file')

  parser.add_argument(
    '--outdir',
    type=str,
    dest='outdir',
    default='',
    help='Specify output directory')

  args = parser.parse_args()
  if args.input == '':
    parser.print_help()
    exit(0)
  return args

###############################################################################
#
#  load the data from CSV file and creating output directory
#
###############################################################################

def load_data(csv_path):
  '''load the raw data as stored in CSV file'''
  return pd.read_csv(csv_path, na_filter=False, skipinitialspace=True, thousands=',')

def make_output_folder(outdir):
  output_dir = os.path.join(outdir, 'feature_analysis_plotting')
  os.makedirs(output_dir, exist_ok=True)
  return output_dir

###############################################################################
#
#  class to analyse correlations between features
#
###############################################################################

class FeatureAnalysisPlotting(object):
  '''A class to help analyse the data;
  try to identify linear correlations in the data;
  calculate Pearson Correlation Coefficient with and without
  p-values; create a scatter matrix; inout data must not contain
  any strings or NaN values; also remove any columns with
  categorical data or transform them first; remove any text except column labels'''

  def __init__(self, metrix, output_dir):
    self.metrix = metrix
    self.output_dir = output_dir
    self.prepare_metrix_data()
    self.split_data()
    self.create_itter()
    self.plot_hist()
    self.plot_ecdf()
    self.plot_density()

###############################################################################
#
#  creating smaller data frame
#
###############################################################################

  def prepare_metrix_data(self):
    '''Function to create smaller dataframe with columns of interest.
    ******
    Input: large data frame
    Output: smaller dataframe
    '''
    print('*' *80)
    print('*    Preparing input dataframe X_metrix')
    print('*' *80)

    #database plus manually added data
    self.X_metrix = self.metrix[['anomalousCC', 'IoverSigma', 'completeness',
                    'diffI', 'RmergeI', 'lowreslimit', 'RpimI', 'multiplicity',
                    'RmeasdiffI', 'anomalousslope', 'diffF', 'wilsonbfactor',
                    'RmeasI', 'highreslimit', 'RpimdiffI', 'anomalousmulti',
                    'RmergediffI', 'totalobservations', 'anomalouscompl',
                    'cchalf', 'totalunique', 'LLG', 'TFZ', 'PAK', 'mr_reso',
                    'RMSD', 'VRMS', 'eLLG', 'tncs', 'seq_ident', 'model_res',
                    'No_atom_chain', 'MW_chain', 'No_res_chain', 'No_res_asu',
                    'likely_sg_no', 'xia2_cell_volume', 'Vs', 'Vm',
                    'No_mol_asu', 'MW_asu', 'No_atom_asu']]

    self.X_metrix = self.X_metrix.fillna(0)

    with open(os.path.join(self.output_dir,
              'feature_analysis_plotting.txt'), 'a') as text_file:
      text_file.write('Created dataframe X_metrix \n')
      text_file.write('with columns: \n')
      text_file.write(str(self.X_metrix.columns)+ '\n')

###############################################################################
#
#  creating training and test set
#
###############################################################################

  def split_data(self):
    '''Function which splits the input data into training set and test set.
    ******
    Input: a dataframe that contains the features and labels in columns and the samples
          in rows
    Output: sets of training and test data with an 80/20 split; X_train, X_test, y_train,
            y_test
    '''
    print('*' *80)
    print('*    Splitting data into test and training set with test=20%')
    print('*' *80)

    y = self.metrix['MR_success']

#stratified split of samples
    X_metrix_train, X_metrix_test, y_train, y_test = train_test_split(self.X_metrix, y, test_size=0.2, random_state=42, stratify=y)

    assert self.X_metrix.columns.all() == X_metrix_train.columns.all()

    self.X_metrix_train = X_metrix_train
    self.X_metrix_test = X_metrix_test
    self.y_train = y_train
    self.y_test = y_test

    with open(os.path.join(self.output_dir,
              'feature_analysis_plotting.txt'), 'a') as text_file:
      text_file.write('Spliting into training and test set 80-20 \n')
      text_file.write('X_metrix: X_metrix_train, X_metrix_test \n')
      text_file.write('y(MR_success): y_train, y_test \n')

################################################################################
#
#  creating a global feature list for iteration
#
################################################################################

  def create_itter(self):
    itter = self.X_metrix_train.columns
    self.itter = itter

################################################################################
#
#  plotting histogram for each feature
#
################################################################################

  def plot_hist(self):
    '''Plot a histogram for each feature.'''
    print('*' *80)
    print('*    Plotting histogram for each feature')
    print('*' *80)

    datestring = datetime.strftime(datetime.now(), '%Y%m%d_%H%M')
    for name in self.itter:
      hist = plt.hist(self.X_metrix_train[name], bins=20)
      hist = plt.xlabel(name)
      hist = plt.ylabel('number of counts')
      plt.savefig(os.path.join(self.output_dir,
                      'histogram_feature_'+name+'_'+datestring+'.png'), dpi=600)
      plt.close()
      with open(os.path.join(self.output_dir,
                'feature_analysis_plotting.txt'), 'a') as text_file:
        text_file.write('Drawing histogram for feature %s \n' %name)

################################################################################
#
#  plotting empirical cumulative distribution function for each feature
#
################################################################################

  def plot_ecdf(self):
    '''Plot empirical cumulative distribution function (ECDF) for each feature.'''
    print('*' *80)
    print('*    Plotting ECDF for each feature')
    print('*' *80)

    datestring = datetime.strftime(datetime.now(), '%Y%m%d_%H%M')

    for name in self.itter:
      '''Compute ECDF for a one-dimensional array of measurements'''
      with open(os.path.join(self.output_dir,
                'feature_analysis_plotting.txt'), 'a') as text_file:
        text_file.write('Drawing ECDF for feature %s \n' %name)
      #Number of data points:n
      n = len(self.X_metrix_train[name])
      #x-data for the ECDF:x
      x = np.sort(self.X_metrix_train[name])
      #y-data for the ECDF;y
      y = np.arange(1, n+1)/n
      mean = np.mean(self.X_metrix_train[name])
      median = np.median(self.X_metrix_train[name])
      std = np.std(self.X_metrix_train[name])
      percentile = np.percentile(self.X_metrix_train[name], [25, 50, 75])
      info = self.X_metrix_train[name].describe()
      samples = np.random.normal(mean, std, size=10000)
      n_theor = len(samples)
      x_theor = np.sort(samples)
      y_theor = np.arange(1, n_theor+1)/n_theor

      with open(os.path.join(self.output_dir,
                'feature_analysis_plotting.txt'), 'a') as text_file:
        text_file.write('The basic stats for feature %s are: \n' %name)
        text_file.write('mean: '+str(info[1])+'\n')
        text_file.write('median: '+str(median)+'\n')
        text_file.write('std: '+str(info[2])+'\n')
        text_file.write('min: '+str(info[3])+'\n')
        text_file.write('25%: '+str(info[4])+'\n')
        text_file.write('50%: '+str(info[5])+'\n')
        text_file.write('75%: '+str(info[6])+'\n')
        text_file.write('max: '+str(info[7])+'\n')
      plt.plot(x, y, marker='.', linestyle='none', label='observed CDF')
      plt.plot(x_theor, y_theor, label='theoretical CDF')
      plt.axvline(mean, label='Mean', color='r', linestyle='--')
      plt.axvline(median, label='Median', color='g', linestyle='--')
      plt.text(mean, 0.9, 'Mean: %.2f' %mean)
      plt.text(median, 0.8, 'Median: %.2f' %median)
      plt.xlabel(name)
      plt.ylabel('(E)CDF')
      plt.margins(0.02)
      plt.legend(loc='best')
      plt.savefig(os.path.join(self.output_dir,
                           'ECDF_feature_'+name+'_'+datestring+'.png'), dpi=600)
      plt.close()

################################################################################
#
#  plotting probability density function for each feature to evaluate continous
#  data and its cumulative distribution function
#
################################################################################

  def plot_density(self):
    '''Plot probability density function (PDF) for each feature.'''
    print('*' *80)
    print('*    Plotting density curve for each feature')
    print('*' *80)

    datestring = datetime.strftime(datetime.now(), '%Y%m%d_%H%M')

    for name in self.itter:
      '''Compute ECDF for a one-dimensional array of measurements'''
      with open(os.path.join(self.output_dir,
                'feature_analysis_plotting.txt'), 'a') as text_file:
        text_file.write('Drawing ECDF for feature %s and its calculated theoretical curve\n' %name)
      mean = np.mean(self.X_metrix_train[name])
      median = np.median(self.X_metrix_train[name])
      sns.distplot(self.X_metrix_train[name], hist = False, kde = True,
                 kde_kws = {'linewidth': 3})
      plt.axvline(mean, label='Mean', color='r', linestyle='--')
      plt.axvline(median, label='Median', color='g', linestyle='--')
      plt.text(mean, 0.9, 'Mean: %.2f' %mean)
      plt.text(median, 0.8, 'Median: %.2f' %median)
      plt.legend(loc='best')
      plt.xlabel(name)
      plt.ylabel('Density')
      plt.savefig(os.path.join(self.output_dir,
                        'Density_feature_'+name+'_'+datestring+'.png'), dpi=600)
      plt.close()

def run():
  args = parse_command_line()

  ###############################################################################

  #look at the imported data to get an idea what we are working with
  matrix = load_data(args.input)

  output_dir = make_output_folder(args.outdir)

  ###############################################################################

  feature_analysis_plotting = FeatureAnalysisPlotting(matrix, output_dir)