text:
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
""" Computation of the canonical HRF used in fMRIstat, both the 2-term
spectral approximation and the Taylor series approximation, to a shifted
version of the canonical Glover HRF.

References
----------
Liao, C.H., Worsley, K.J., Poline, J-B., Aston, J.A.D., Duncan, G.H.,
    Evans, A.C. (2002). \'Estimating the delay of the response in fMRI
    data.\' NeuroImage, 16:593-606.
"""

import numpy as np
import numpy.linalg as npl
from scipy.interpolate import interp1d

from sympy.utilities.lambdify import implemented_function

from ..utils import T, lambdify_t
from .. import hrf
from .invert import invertR

def spectral_decomposition(hrf2decompose,
                           time=None,
                           delta=None,
                           ncomp=2):
    """ PCA decomposition of symbolic HRF shifted over time

    Perform a PCA expansion of a symbolic HRF, time shifted over the
    values in delta, returning the first ncomp components.

    This smooths out the HRF as compared to using a Taylor series
    approximation.

    Parameters
    ----------
    hrf2decompose : sympy expression
        An expression that can be lambdified as a function of 't'. This
        is the HRF to be expanded in PCA
    time : None or np.ndarray, optional
        None gives default value of np.linspace(-15,50,3251) chosen to
        match fMRIstat implementation.  This corresponds to a time
        interval of 0.02.  Presumed to be equally spaced.
    delta : None or np.ndarray, optional
        None results in default value of np.arange(-4.5, 4.6, 0.1)
        chosen to match fMRIstat implementation.
    ncomp : int, optional
        Number of principal components to retain.

    Returns
    -------
    hrf : [sympy expressions]
        A sequence length `ncomp` of symbolic HRFs that are the
        principal components.
    approx :
        TODO
    """
    if time is None:
        time = np.linspace(-15,50,3251)
    dt = time[1] - time[0]
    if delta is None:
        delta = np.arange(-4.5, 4.6, 0.1)
    # make numerical implementation from hrf function and symbol t.
    # hrft returns function values when called with values for time as
    # input.
    hrft = lambdify_t(hrf2decompose(T))
    # Create stack of time-shifted HRFs.  Time varies over row, delta
    # over column.
    ts_hrf_vals = np.array([hrft(time - d) for d in delta]).T
    ts_hrf_vals = np.nan_to_num(ts_hrf_vals)
    # PCA
    U, S, V = npl.svd(ts_hrf_vals, full_matrices=0)
    # make interpolators from the generated bases
    basis = []
    for i in range(ncomp):
        b = interp1d(time, U[:, i], bounds_error=False, fill_value=0.)
        # normalize components witn integral of abs of first component
        if i == 0:
            d = np.fabs((b(time) * dt).sum())
        b.y /= d
        basis.append(b)
    # reconstruct time courses for all bases
    W = np.array([b(time) for b in basis]).T
    # regress basis time courses against original time shifted time
    # courses, ncomps by len(delta) parameter matrix
    WH = np.dot(npl.pinv(W), ts_hrf_vals)
    # put these into interpolators to get estimated coefficients for any
    # value of delta
    coef = [interp1d(delta, w, bounds_error=False, fill_value=0.) for w in WH]
    # swap sign of first component to match that of input HRF.  Swap
    # other components if we swap the first, to standardize signs of
    # components across SVD implementations.
    if coef[0](0) < 0: # coefficient at time shift of 0
        for i in range(ncomp):
            coef[i].y *= -1.
            basis[i].y *= -1.

    def approx(time, delta):
        value = 0
        for i in range(ncomp):
            value += coef[i](delta) * basis[i](time)
        return value

    approx.coef = coef
    approx.components = basis
    (approx.theta,
     approx.inverse,
     approx.dinverse,
     approx.forward,
     approx.dforward) = invertR(delta, approx.coef)
    # construct aliased functions from bases
    symbasis = []
    for i, b in enumerate(basis):
        symbasis.append(
            implemented_function('%s%d' % (str(hrf2decompose), i), b))
    return symbasis, approx

def taylor_approx(hrf2decompose,
                  time=None,
                  delta=None):
    """ A Taylor series approximation of an HRF shifted by times `delta`

    Returns original HRF and gradient of HRF

    Parameters
    ----------
    hrf2decompose : sympy expression
        An expression that can be lambdified as a function of 't'. This
        is the HRF to be expanded in PCA
    time : None or np.ndarray, optional
        None gives default value of np.linspace(-15,50,3251) chosen to
        match fMRIstat implementation.  This corresponds to a time
        interval of 0.02.  Presumed to be equally spaced.
    delta : None or np.ndarray, optional
        None results in default value of np.arange(-4.5, 4.6, 0.1)
        chosen to match fMRIstat implementation.

    Returns
    -------
    hrf : [sympy expressions]
        Sequence length 2 comprising (`hrf2decompose`, ``dhrf``) where
        ``dhrf`` is the first derivative of `hrf2decompose`.
    approx :
        TODO

    References
    ----------
    Liao, C.H., Worsley, K.J., Poline, J-B., Aston, J.A.D., Duncan, G.H.,
    Evans, A.C. (2002). \'Estimating the delay of the response in fMRI
    data.\' NeuroImage, 16:593-606.
    """
    if time is None:
        time = np.linspace(-15,50,3251)
    dt = time[1] - time[0]
    if delta is None:
        delta = np.arange(-4.5, 4.6, 0.1)
    # make numerical implementation from hrf function and symbol t.
    # hrft returns function values when called with values for time as
    # input.
    hrft = lambdify_t(hrf2decompose(T))
    # interpolator for negative gradient of hrf
    dhrft = interp1d(time, -np.gradient(hrft(time), dt), bounds_error=False,
                    fill_value=0.)
    dhrft.y *= 2
    # Create stack of time-shifted HRFs.  Time varies over row, delta
    # over column.
    ts_hrf_vals = np.array([hrft(time - d) for d in delta]).T
    # hrf, dhrf
    W = np.array([hrft(time), dhrft(time)]).T
    # regress hrf, dhrf at times against stack of time-shifted hrfs
    WH = np.dot(npl.pinv(W), ts_hrf_vals)
    # put these into interpolators to get estimated coefficients for any
    # value of delta
    coef = [interp1d(delta, w, bounds_error=False,
                     fill_value=0.) for w in WH]

    def approx(time, delta):
        value = (coef[0](delta) * hrft(time)
                 + coef[1](delta) * dhrft(time))
        return value

    approx.coef = coef
    approx.components = [hrft, dhrft]
    (approx.theta,
     approx.inverse,
     approx.dinverse,
     approx.forward,
     approx.dforward) = invertR(delta, approx.coef)
    dhrf = implemented_function('d%s' % str(hrf2decompose), dhrft)
    return [hrf2decompose, dhrf], approx

canonical, canonical_approx = taylor_approx(hrf.glover)
spectral, spectral_approx = spectral_decomposition(hrf.glover)

max_stars_repo_name: arokem/nipy

max_issues_repo_name: arokem/nipy

max_forks_repo_name: arokem/nipy

reason: import numpy,from scipy,from sympy


text:
import streamlit as st
import pandas as pd
import json
import requests
import numpy as np

##########

# Present title
st.title('Company Default Estimator')

##########

st.header('1. Load csv report')
uploaded_file = st.file_uploader("Choose a file")
if uploaded_file is not None:
     df = pd.read_csv(uploaded_file)
     st.table(df)

##########
st.header('2. Run analysis')

if st.button('Press to run data through model'):
    #Create JSON Object
    data = df.to_numpy()
    data = json.dumps({"signature_name": "serving_default", "instances": data.tolist()})

    # Call tf server
    headers = {"content-type": "application/json"}
    json_response = requests.post('http://localhost:8501/v1/models/bankmodel:predict', data=data, headers=headers)
    predictions = json.loads(json_response.text)['predictions']

    # show results
    pred_round = [round(pred,3) for pred in sum(predictions, [])]
    st.write("0 ~ prediction no default and 1 ~ prediction will deault ")
    st.write(pred_round)

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Module for orbit response matrix calculation.
"""

import time
from functools import reduce

import numpy as np
import numpy.linalg as linalg

from phantasy.library.misc import epoch2human
from phantasy.library.misc import truncate_number

from .devices import process_devices

TS_FMT = "%Y-%m-%d %H:%M:%S"

def get_orm(correctors, monitors, **kws):
    """Calculate orbit response matrix (ORM) with defined input parameters.

    The length of *monitors* is ``m``, the length of correctors is ``n``,
    if both x and y directions are monitored, the length of *monitors* should
    be doubled. The final calculated ORM should be of the shape of ``(m, n)``.

    For each iteration of correctors updates, which could be defined by ``scan``
    keyword parameter, a serie of measured orbit data would be returned from
    :func:`get_orbit`, doing linear polynomial fitting against the feeded
    ``scan`` list of values should return one column of ORM data.
    The column-wise returned matrix is the final ORM.

    Parameters
    ----------
    correctors : List[CaElement]
        List of correctors from *lattice*.
    monitors : List[CaElement]
        List of monitors, usually BPMs, to measure the beam orbit.

    Keyword Arguments
    -----------------
    source : str
        Orbit data source, ``control`` (default) or ``model``. If ``control``
        is defined, ORM will be measured within *control* environment,
        or ORM will be from *model*, e.g. FLAME model.
    scan : array
        Scan array for correctors, unit: *rad*, default array is
        ``[-0.003, -0.0015, 0, 0.0015, 0.003]``.
    cor_field : str
        Field name for correctors, ``'ANG'`` by default.
    orb_field : tuple[str]
        Field names for monitors to retrieve orbit data, ``('X', 'Y')`` for
        *x* and *y* directions by default.
    wait : float
        Wait time after set value, in *sec*, 1.0 by default.
    reset_wait : float
        Wait time after set value to original value, in *sec*, 1.0 by default.
    xoy : str
        'x'('y') for monitoring 'x'('y') direction,'xy' for both (default).
    lattice : Lattice
        High-level lattice object.
    msg_queue : Queue
        A queue that keeps log messages.

    Returns
    -------
    ret : array
        ORM with shape ``(m, n)``.

    Todo
    ----
    case of ``source == 'model'``, require kws 'lattice'.
    """
    lattice = kws.get('lattice', None)
    source = kws.get('source', 'control')
    scan = kws.get('scan', np.linspace(-0.003, 0.003, 5))
    cor_field = kws.get('cor_field', 'ANG')
    xoy = kws.get('xoy', 'xy')
    wait = kws.get('wait', 1.0)
    reset_wait = kws.get('reset_wait', 1.0)
    q_msg = kws.get('msg_queue', None)

    m = len(monitors) * len(xoy)
    n = len(correctors)
    mat_mn = np.zeros([m, n])

    # i <-- 0, m-1
    # B[i] = \Sigma_j=0^{n-1} R[i,j] C[j]
    # ---> iff C[j] != 0 ==> B[i] = R[i,j] C[j]
    # ==> R[i,j] = B[i]/C[j]
    # ==> or: R[i,j] = polyfit(C[i], B[i], 1)[0]
    #
    ns = len(scan)
    for i, cor in enumerate(correctors):
        orbit_arr = np.zeros([len(scan), m])
        cor_val0 = cor.current_setting(cor_field)
        for iscan, val in enumerate(scan):
            setattr(cor, cor_field, val)
            time.sleep(wait)
            v_rd = getattr(cor, cor_field)
            msg = "[{0}] Set [{1:02d}] {2} [{3}]: {4:>10.6f} (RD: {5:>10.6f})".format(
                    epoch2human(time.time(), fmt=TS_FMT), i + 1, cor.name,
                    cor_field, val, v_rd)
            if q_msg is not None:
                q_msg.put(((i * ns + iscan) * 100.0 / n / ns, msg))
            print(msg)
            orbit_arr[iscan] = get_orbit(monitors, **kws)

        ## get mat_mn(k,i)
        #for k in range(m):
        #    mat_mn[k, i] = np.polyfit(scan, orbit_arr[:, k], 1)[0]
        mat_mn[:, i] = [np.polyfit(scan, orbit_arr[:, k], 1)[0] for k in range(m)]

        # reset cor and process next col
        setattr(cor, cor_field, cor_val0)
        time.sleep(reset_wait)
    return mat_mn

def get_orm_for_one_corrector(corrector, monitors, **kws):
    """Get column-wise ORM data for one corrector.

    Parameters
    ----------
    correctors : CaElement
        Selected corrector element.
    monitors : List[CaElement]
        List of monitors, usually BPMs, to measure the beam orbit.

    Keyword Arguments
    -----------------
    scan : array
        Scan array for correctors, unit: *rad*, default array is
        ``[-0.003, -0.0015, 0, 0.0015, 0.003]``.
    cor_field : str
        Field name for correctors, ``'ANG'`` by default.
    orb_field : tuple[str]
        Field names for monitors to retrieve orbit data, ``('X', 'Y')`` for
        *x* and *y* directions by default.
    wait : float
        Wait time after set value, in *sec*, 1.0 by default.
    reset_wait : float
        Wait time after set value to original value, in *sec*, 1.0 by default.
    xoy : str
        'x'('y') for monitoring 'x'('y') direction,'xy' for both (default).
    msg_queue : Queue
        A queue that keeps log messages.
    idx : int
        Index of selected corrector of all selected ones.
    ncor : int
        Total number of selected correctors.
    ndigits : int
        Number of effective digits to keep for a float number.
    keep_all : bool
        Return all measured data or not, default is False.
    tol : float
        Absolute tolerance between setpoint and readback.

    Returns
    -------
    ret : tuple
        Column-wised ORM data with the size of ``m``, if keep_all is True,
        return all data as the second element of tuple.

    See Also
    --------
    get_orm: Measurem ORM.
    """

    scan = kws.get('scan', np.linspace(-0.003, 0.003, 5))

    print("--- Scan Range: ", scan, "---")

    cor_field = kws.get('cor_field', 'ANG')
    xoy = kws.get('xoy', 'xy')
    wait = kws.get('wait', 1.0)
    reset_wait = kws.get('reset_wait', 1.0)
    idx = kws.get('idx', 0.0)  # index of correctors
    n = kws.get('ncor', 1)     # total number of correctors
    q_msg = kws.get('msg_queue', None)
    n_trun = kws.get('ndigits', 6)
    tol = kws.get('tol', 0.1)

    print(f"{corrector.name}, tol: {tol}")

    ns = len(scan)
    nn = n * ns

    m = len(monitors) * len(xoy)
    orbit_arr = np.zeros([len(scan), m])
    cor_val0 = corrector.current_setting(cor_field)
    scan_rdbk = []
    for iscan, val in enumerate(scan):
        v_to_set =  truncate_number(val, n_trun)

        # ensure put
        corrector.ensure_put(cor_field, v_to_set, tol=tol, timeout=wait)

        # setattr(corrector, cor_field, v_to_set)
        # time.sleep(wait)
        scan_rdbk.append(getattr(corrector, cor_field))

        msg = "[{0}] Set [{1:02d}] {2} [{3}]: {4:>10.6f} (RD: {5:>10.6f})".format(
                epoch2human(time.time(), fmt=TS_FMT), idx + 1, corrector.name,
                cor_field, v_to_set, getattr(corrector, cor_field))
        if q_msg is not None:
            q_msg.put(((ns * idx + iscan)* 100.0 / nn, msg))
        print(msg)
        orbit_arr[iscan] = get_orbit(monitors, **kws)

    # ensure put
    corrector.ensure_put(cor_field, cor_val0, tol=tol, timeout=reset_wait)

    # setattr(corrector, cor_field, cor_val0)
    # time.sleep(reset_wait)

    msg = "[{0}] Set [{1:02d}] {2} [{3}]: {4:>10.6f} (RD: {5:>10.6f}) [RESET]".format(
            epoch2human(time.time(), fmt=TS_FMT), idx + 1, corrector.name,
            cor_field, cor_val0, getattr(corrector, cor_field))
    if q_msg is not None:
        q_msg.put((-1, msg))
    print(msg)

    r = np.asarray([np.polyfit(scan_rdbk, orbit_arr[:, k], 1)[0] for k in range(m)])

    if kws.get('keep_all', False):
        return r, np.asarray(orbit_arr)
    else:
        return r, None

def inverse_matrix(m):
    """Get inverse matrix of *m* based on SVD approach, *m* should be general
    rectangle shaped.

    Parameters
    ----------
    m : array
        Matrix.

    Returns
    -------
    ret : array
        Inverse matrix.

    See Also
    --------
    get_orm : Measure orbit response matrix.
    """
    U, s, V = linalg.svd(m)
    min_shape = min(m.shape)
    S_inv = np.zeros(m.shape[::-1], dtype=float)
    S_inv[:min_shape, :min_shape] = np.diag(1.0/s)
    m_inv = reduce(np.dot, (V.T, S_inv, U.T))
    return m_inv

def get_orbit(monitors, **kws):
    """Get beam orbit from defined *monitors*.

    Parameters
    ----------
    monitors : List[CaElement]
        List of monitors, usually BPMs, to measure the beam orbit.

    Keyword Arguments
    -----------------
    orb_field : tuple[str]
        Field names for monitors to retrieve orbit data, ``('X', 'Y')`` for
        *x* and *y* directions by default.
    xoy : str
        'x'('y') for monitoring 'x'('y') direction,'xy' for both (default).
    rate : int
        DAQ frequency, default is 1.
    nshot : int
        Total shot number for DAQ process, default is 1.
    slow_mode_on : bool
        If set, apply processor for slow device, e.g. PM, default is True,
        set False might be useful for orbit evaluation of ORM correction.

    Returns
    -------
    ret : array
        Monitors readings as beam orbit.

    Note
    ----
    The returned array lines all 'X' readings first, then follows all 'Y's.
    """
    nshot = kws.get('nshot', 1)
    rate = kws.get('rate', 1)
    delt = 1.0 / rate
    orb_field = kws.get('orb_field', ('X', 'Y'))
    xoy = kws.get('xoy', 'xy')
    slow_mode_on = kws.get('slow_mode_on', True)
    xyfld = list(zip(range(len(xoy)), orb_field))
    arr = np.zeros((nshot, len(xoy) * len(monitors)))
    #
    if slow_mode_on:
        process_devices(monitors, **kws)
    #
    for i in range(nshot):
        a = [[getattr(elem, fld) for elem in monitors] for _, fld in xyfld]
        arr[i, :] = np.asarray(a).flatten()
        time.sleep(delt)
    return arr.mean(axis=0)

def get_correctors_settings(orm, orbit_diff, inverse=False):
    """Calculate new settings of correctors regarding to BPM readings after
    correction.

    Parameters
    ----------
    orm : array
        Orbit response matrix.
    orbit_diff : array
        Orbit (BPM readings) difference after and before correction.
    inverse : True or False
        If *orm* has already been inversed or not (default).

    Returns
    -------
    ret : array
        Array of corrector settings that should be added to original ones.

    See Also
    --------
    get_orm : Measure orbit response matrix.
    inverse_matrix : Inverse matrix by SVD.
    """
    if not inverse:
        m_inv = inverse_matrix(orm)
    else:
        m_inv = orm

    return np.dot(m_inv, orbit_diff)

def get_index_grid(lattice, correctors, monitors, xoy='xy'):
    """Return the index grid of sub ORM from global ORM for selected
    *correctors* and *monitors* w/ *xoy*.

    The global ORM is measured with all correctors (H + V) and BPMs for
    both x and y direction.

    Parameters
    ----------
    lattice : Lattice
        High-level lattice object.
    correctors : List[CaElement]
        List of correctors from *lattice*.
    monitors : List[CaElement]
        List of monitors, usually BPMs, to measure the beam orbit.
    xoy : str
        'x'('y') for monitoring 'x'('y') direction,'xy' for both (default).

    See Also
    --------
    get_orm : Measure orbit response matrix.
    """
    all_cors = lattice.get_elements(type='HCOR') +  \
               lattice.get_elements(type='VCOR')
    all_bpms = lattice.get_elements(type='BPM')
    # selected correctors columns
    col_idx = [all_cors.index(i) for i in correctors]
    # selected BPM rows, x,x,x....y,y,y...
    bpm_idx = [all_bpms.index(i) for i in monitors]
    if xoy == 'xy':
        bpm_cnt = len(all_bpms)
        row_idx = bpm_idx + [i+bpm_cnt for i in bpm_idx]
    elif xoy == 'x':
        row_idx = bpm_idx
    elif xoy == 'y':
        bpm_cnt = len(all_bpms)
        row_idx = [i+bpm_cnt for i in bpm_idx]
    return np.ix_(row_idx, col_idx)

max_stars_repo_name: phantasy-project/phantasy

max_issues_repo_name: phantasy-project/phantasy

max_forks_repo_name: archman/phantasy


text:
import math
import random
import numpy as np
import os
from collections import deque
from datetime import datetime
from tqdm import tqdm

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as FDQN

from model_breakout import DQN, ReplayMemory, ActionSelector, FrameProcessor, fp
from utils import wrap_deepmind, make_atari

def main():
    # 1. GPU settings
    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
    device = torch.device(
        "cuda" if torch.cuda.is_available() else "cpu"
    )  # if gpu is to be used

    env_name = "Breakout"
    env_raw = make_atari("{}NoFrameskip-v4".format(env_name))
    env = wrap_deepmind(
        env_raw, frame_stack=False, episode_life=True, clip_rewards=True
    )

    c, h, w = fp(env.reset()).shape
    n_actions = env.action_space.n

    print(f"Running {env_name} with {n_actions} actions")

    # 4. Network reset
    policy_net = DQN(h, w, n_actions, device).to(device)
    target_net = DQN(h, w, n_actions, device).to(device)
    policy_net.apply(policy_net.init_weights)
    target_net.load_state_dict(policy_net.state_dict())
    target_net.eval()

    # 5. DQN hyperparameters
    BATCH_SIZE = 32
    GAMMA = 0.99
    EPS_START = 1.0
    EPS_END = 0.1
    EPS_DECAY = 500000
    TARGET_UPDATE = 10000
    NUM_STEPS = 50000000
    M_SIZE = 500000
    POLICY_UPDATE = 4
    EVALUATE_FREQ = 200000
    optimizer = optim.Adam(policy_net.parameters(), lr=0.0000625, eps=1.5e-4)

    # replay memory and action selector
    memory = ReplayMemory(M_SIZE, [5, h, w], n_actions, device)
    sa = ActionSelector(EPS_START, EPS_END, policy_net, EPS_DECAY, n_actions, device)

    steps_done = 0

    def optimize_model(train):
        if not train:
            return
        (
            state_batch,
            action_batch,
            reward_batch,
            n_state_batch,
            done_batch,
        ) = memory.sample(BATCH_SIZE)

        q = policy_net(state_batch).gather(1, action_batch)
        nq = target_net(n_state_batch).max(1)[0].detach()

        # Compute the expected Q values
        expected_state_action_values = (nq * GAMMA) * (
            1.0 - done_batch[:, 0]
        ) + reward_batch[:, 0]

        # Compute Huber loss
        loss = F.smooth_l1_loss(q, expected_state_action_values.unsqueeze(1))

        # Optimize the model
        optimizer.zero_grad()
        loss.backward()
        for param in policy_net.parameters():
            param.grad.data.clamp_(-1, 1)
        optimizer.step()

    def evaluate(step, policy_net, device, env, n_actions, eps=0.05, num_episode=5):
        env = wrap_deepmind(env)
        sa = ActionSelector(eps, eps, policy_net, EPS_DECAY, n_actions, device)
        e_rewards = []
        q = deque(maxlen=5)
        for i in range(num_episode):
            env.reset()
            e_reward = 0
            for _ in range(10):  # no-op
                n_frame, _, done, _ = env.step(0)
                n_frame = fp(n_frame)
                q.append(n_frame)

            while not done:
                state = torch.cat(list(q))[1:].unsqueeze(0)
                action, eps = sa.select_action(state, train)
                n_frame, reward, done, info = env.step(action)
                n_frame = fp(n_frame)
                q.append(n_frame)

                e_reward += reward
            e_rewards.append(e_reward)

        f = open("file.txt", "a")
        f.write(
            "%f, %d, %d\n"
            % (float(sum(e_rewards)) / float(num_episode), step, num_episode)
        )
        f.close()
        return float(sum(e_rewards)) / float(num_episode)

    q = deque(maxlen=5)
    done = True
    eps = 0
    episode_len = 0

    episode = 0
    latest_test_reward = 0

    progressive = tqdm(range(NUM_STEPS), total=NUM_STEPS, unit="b")
    for step in progressive:
        if done:  # life reset !!!
            episode += 1
            env.reset()
            sum_reward = 0
            episode_len = 0
            img, _, _, _ = env.step(1)  # BREAKOUT specific !!!
            for i in range(10):  # no-op
                n_frame, _, _, _ = env.step(0)
                n_frame = fp(n_frame)
                q.append(n_frame)

        train = len(memory) > 50000
        # Select and perform an action
        state = torch.cat(list(q))[1:].unsqueeze(0)
        action, eps = sa.select_action(state, train)
        n_frame, reward, done, info = env.step(action)
        n_frame = fp(n_frame)

        # 5 frame as memory
        q.append(n_frame)
        memory.push(
            torch.cat(list(q)).unsqueeze(0), action, reward, done
        )  # here the n_frame means next frame from the previous time step
        episode_len += 1

        # Perform one step of the optimization (on the target network)
        if step % POLICY_UPDATE == 0:
            optimize_model(train)

        # Update the target network, copying all weights and biases in DQN
        if step % TARGET_UPDATE == 0:
            target_net.load_state_dict(policy_net.state_dict())

        if step % EVALUATE_FREQ == 0:
            latest_test_reward = evaluate(
                step,
                policy_net,
                device,
                env_raw,
                n_actions,
                eps=0.05,
                num_episode=episode,
            )

        progressive.set_description(
            f"Step {step} Latest test reward: {latest_test_reward}"
        )

if __name__ == "__main__":
    main()

max_stars_repo_name: jiwidi/MIARFID

max_issues_repo_name: jiwidi/MIARFID

max_forks_repo_name: jiwidi/MIARFID


text:
#####################################################
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.03 #
#####################################################

import abc
import math
import copy
import random
import numpy as np
from collections import OrderedDict

from typing import Optional, Text

__all__ = ["_EPS", "Space", "Categorical", "Integer", "Continuous"]

_EPS = 1e-9

class Space(metaclass=abc.ABCMeta):
    """Basic search space describing the set of possible candidate values for hyperparameter.
    All search space must inherit from this basic class.
    """

    def __init__(self):
        # used to avoid duplicate sample
        self._last_sample = None
        self._last_abstract = None

    @abc.abstractproperty
    def xrepr(self, depth=0) -> Text:
        raise NotImplementedError

    def __repr__(self) -> Text:
        return self.xrepr()

    @abc.abstractproperty
    def abstract(self, reuse_last=False) -> "Space":
        raise NotImplementedError

    @abc.abstractmethod
    def random(self, recursion=True, reuse_last=False):
        raise NotImplementedError

    @abc.abstractmethod
    def clean_last_sample(self):
        raise NotImplementedError

    @abc.abstractmethod
    def clean_last_abstract(self):
        raise NotImplementedError

    def clean_last(self):
        self.clean_last_sample()
        self.clean_last_abstract()

    @abc.abstractproperty
    def determined(self) -> bool:
        raise NotImplementedError

    @abc.abstractmethod
    def has(self, x) -> bool:
        """Check whether x is in this search space."""
        assert not isinstance(
            x, Space
        ), "The input value itself can not be a search space."

    @abc.abstractmethod
    def __eq__(self, other):
        raise NotImplementedError

    def copy(self) -> "Space":
        return copy.deepcopy(self)

class VirtualNode(Space):
    """For a nested search space, we represent it as a tree structure.

    For example,
    """

    def __init__(self, id=None, value=None):
        super(VirtualNode, self).__init__()
        self._id = id
        self._value = value
        self._attributes = OrderedDict()

    @property
    def value(self):
        return self._value

    def append(self, key, value):
        if not isinstance(key, str):
            raise TypeError(
                "Only accept string as a key instead of {:}".format(type(key))
            )
        if not isinstance(value, Space):
            raise ValueError("Invalid type of value: {:}".format(type(value)))
        # if value.determined:
        #    raise ValueError("Can not attach a determined value: {:}".format(value))
        self._attributes[key] = value

    def xrepr(self, depth=0) -> Text:
        strs = [self.__class__.__name__ + "(value={:}".format(self._value)]
        for key, value in self._attributes.items():
            strs.append(key + " = " + value.xrepr(depth + 1))
        strs.append(")")
        if len(strs) == 2:
            return "".join(strs)
        else:
            space = "  "
            xstrs = (
                [strs[0]]
                + [space * (depth + 1) + x for x in strs[1:-1]]
                + [space * depth + strs[-1]]
            )
            return ",\n".join(xstrs)

    def abstract(self, reuse_last=False) -> Space:
        if reuse_last and self._last_abstract is not None:
            return self._last_abstract
        node = VirtualNode(id(self))
        for key, value in self._attributes.items():
            if not value.determined:
                node.append(value.abstract(reuse_last))
        self._last_abstract = node
        return self._last_abstract

    def random(self, recursion=True, reuse_last=False):
        if reuse_last and self._last_sample is not None:
            return self._last_sample
        node = VirtualNode(None, self._value)
        for key, value in self._attributes.items():
            node.append(key, value.random(recursion, reuse_last))
        self._last_sample = node  # record the last sample
        return node

    def clean_last_sample(self):
        self._last_sample = None
        for key, value in self._attributes.items():
            value.clean_last_sample()

    def clean_last_abstract(self):
        self._last_abstract = None
        for key, value in self._attributes.items():
            value.clean_last_abstract()

    def has(self, x) -> bool:
        for key, value in self._attributes.items():
            if value.has(x):
                return True
        return False

    def __contains__(self, key):
        return key in self._attributes

    def __getitem__(self, key):
        return self._attributes[key]

    @property
    def determined(self) -> bool:
        for key, value in self._attributes.items():
            if not value.determined:
                return False
        return True

    def __eq__(self, other):
        if not isinstance(other, VirtualNode):
            return False
        for key, value in self._attributes.items():
            if not key in other:
                return False
            if value != other[key]:
                return False
        return True

class Categorical(Space):
    """A space contains the categorical values.
    It can be a nested space, which means that the candidate in this space can also be a search space.
    """

    def __init__(self, *data, default: Optional[int] = None):
        super(Categorical, self).__init__()
        self._candidates = [*data]
        self._default = default
        assert self._default is None or 0 <= self._default < len(
            self._candidates
        ), "default >= {:}".format(len(self._candidates))
        assert len(self) > 0, "Please provide at least one candidate"

    @property
    def candidates(self):
        return self._candidates

    @property
    def default(self):
        return self._default

    @property
    def determined(self):
        if len(self) == 1:
            return (
                not isinstance(self._candidates[0], Space)
                or self._candidates[0].determined
            )
        else:
            return False

    def __getitem__(self, index):
        return self._candidates[index]

    def __len__(self):
        return len(self._candidates)

    def clean_last_sample(self):
        self._last_sample = None
        for candidate in self._candidates:
            if isinstance(candidate, Space):
                candidate.clean_last_sample()

    def clean_last_abstract(self):
        self._last_abstract = None
        for candidate in self._candidates:
            if isinstance(candidate, Space):
                candidate.clean_last_abstract()

    def abstract(self, reuse_last=False) -> Space:
        if reuse_last and self._last_abstract is not None:
            return self._last_abstract
        if self.determined:
            result = VirtualNode(id(self), self)
        else:
            # [TO-IMPROVE]
            data = []
            for candidate in self.candidates:
                if isinstance(candidate, Space):
                    data.append(candidate.abstract())
                else:
                    data.append(VirtualNode(id(candidate), candidate))
            result = Categorical(*data, default=self._default)
        self._last_abstract = result
        return self._last_abstract

    def random(self, recursion=True, reuse_last=False):
        if reuse_last and self._last_sample is not None:
            return self._last_sample
        sample = random.choice(self._candidates)
        if recursion and isinstance(sample, Space):
            sample = sample.random(recursion, reuse_last)
        if isinstance(sample, VirtualNode):
            sample = sample.copy()
        else:
            sample = VirtualNode(None, sample)
        self._last_sample = sample
        return self._last_sample

    def xrepr(self, depth=0):
        del depth
        xrepr = "{name:}(candidates={cs:}, default_index={default:})".format(
            name=self.__class__.__name__, cs=self._candidates, default=self._default
        )
        return xrepr

    def has(self, x):
        super().has(x)
        for candidate in self._candidates:
            if isinstance(candidate, Space) and candidate.has(x):
                return True
            elif candidate == x:
                return True
        return False

    def __eq__(self, other):
        if not isinstance(other, Categorical):
            return False
        if len(self) != len(other):
            return False
        if self.default != other.default:
            return False
        for index in range(len(self)):
            if self.__getitem__(index) != other[index]:
                return False
        return True

class Integer(Categorical):
    """A space contains the integer values."""

    def __init__(self, lower: int, upper: int, default: Optional[int] = None):
        if not isinstance(lower, int) or not isinstance(upper, int):
            raise ValueError(
                "The lower [{:}] and uppwer [{:}] must be int.".format(lower, upper)
            )
        data = list(range(lower, upper + 1))
        self._raw_lower = lower
        self._raw_upper = upper
        self._raw_default = default
        if default is not None and (default < lower or default > upper):
            raise ValueError("The default value [{:}] is out of range.".format(default))
            default = data.index(default)
        super(Integer, self).__init__(*data, default=default)

    def xrepr(self, depth=0):
        del depth
        xrepr = "{name:}(lower={lower:}, upper={upper:}, default={default:})".format(
            name=self.__class__.__name__,
            lower=self._raw_lower,
            upper=self._raw_upper,
            default=self._raw_default,
        )
        return xrepr

np_float_types = (np.float16, np.float32, np.float64)
np_int_types = (
    np.uint8,
    np.int8,
    np.uint16,
    np.int16,
    np.uint32,
    np.int32,
    np.uint64,
    np.int64,
)

class Continuous(Space):
    """A space contains the continuous values."""

    def __init__(
        self,
        lower: float,
        upper: float,
        default: Optional[float] = None,
        log: bool = False,
        eps: float = _EPS,
    ):
        super(Continuous, self).__init__()
        self._lower = lower
        self._upper = upper
        self._default = default
        self._log_scale = log
        self._eps = eps

    @property
    def lower(self):
        return self._lower

    @property
    def upper(self):
        return self._upper

    @property
    def default(self):
        return self._default

    @property
    def use_log(self):
        return self._log_scale

    @property
    def eps(self):
        return self._eps

    def abstract(self, reuse_last=False) -> Space:
        if reuse_last and self._last_abstract is not None:
            return self._last_abstract
        self._last_abstract = self.copy()
        return self._last_abstract

    def random(self, recursion=True, reuse_last=False):
        del recursion
        if reuse_last and self._last_sample is not None:
            return self._last_sample
        if self._log_scale:
            sample = random.uniform(math.log(self._lower), math.log(self._upper))
            sample = math.exp(sample)
        else:
            sample = random.uniform(self._lower, self._upper)
        self._last_sample = VirtualNode(None, sample)
        return self._last_sample

    def xrepr(self, depth=0):
        del depth
        xrepr = "{name:}(lower={lower:}, upper={upper:}, default_value={default:}, log_scale={log:})".format(
            name=self.__class__.__name__,
            lower=self._lower,
            upper=self._upper,
            default=self._default,
            log=self._log_scale,
        )
        return xrepr

    def convert(self, x):
        if isinstance(x, np_float_types) and x.size == 1:
            return float(x), True
        elif isinstance(x, np_int_types) and x.size == 1:
            return float(x), True
        elif isinstance(x, int):
            return float(x), True
        elif isinstance(x, float):
            return float(x), True
        else:
            return None, False

    def has(self, x):
        super().has(x)
        converted_x, success = self.convert(x)
        return success and self.lower <= converted_x <= self.upper

    @property
    def determined(self):
        return abs(self.lower - self.upper) <= self._eps

    def clean_last_sample(self):
        self._last_sample = None

    def clean_last_abstract(self):
        self._last_abstract = None

    def __eq__(self, other):
        if not isinstance(other, Continuous):
            return False
        if self is other:
            return True
        else:
            return (
                self.lower == other.lower
                and self.upper == other.upper
                and self.default == other.default
                and self.use_log == other.use_log
                and self.eps == other.eps
            )


text:
from __future__ import print_function
# Version : dec 2nd
from builtins import str
from builtins import range
import os
import re
import numpy as np

# Get a list of all exciton files, make a new file with x=time, y1 = exc A1, y2 = exc A2

# List of exciton files
dirfiles = os.listdir('.')
files=[]
for f in dirfiles:
    if '_E_' in f:
        files.append(f)

files.sort()
print(files)

# Get times
times=[]
for f in files:
    time=re.search('[0-9]{3,}',f).group()
    times.append(time)

# Get excitons
excitons=[]
i_thr=0.005 # intensity threshold
exc_list = (1,2) # excitons wanted (here 1st and 2nd one that respect the threshold)

for p,f in enumerate(files):
    n=1
    print(files[p],times[p])
    content = np.loadtxt(f)
    for e,i,index in content:
        if i>i_thr:
            if n in exc_list:
                n=n+1
                excitons.append((times[p],e))
# excitons is an array of the form
# excitons[(time0,energy_n),(time0,energy_n+1),...,(timep,energy_n+m)]

f = open('output.dat','w')
for i in range(0,len(excitons)):
    t = excitons[i][0]
    e = excitons[i][1]
    line = str(t)+'\t'+str(e)+'\n'
    f.write(line)

f.close()

max_stars_repo_name: QU-XIAO/yambopy

max_issues_repo_name: alexmoratalla/yambopy

max_forks_repo_name: alexmoratalla/yambopy


text:
from tensorflow.python.ops import nn_ops
from tensorflow.python.keras import activations

import tensorflow as tf
import numpy as np

class ConvRNNCell(tf.nn.rnn_cell.RNNCell):
    """A RNN cell with convolutions instead of multiplications."""
    def __init__(self, input_shape, output_channels, kernel_shape, activation=None, reuse=None, name="conv_rnn_cell"):
        """Construct ConvGRUCell.

        Args:
            input_shape: (int, int, int) Shape of the input as int tuple, excluding the batch size
            output_channels: (int) number of output channels of the conv LSTM
            kernel_shape: (int, int) Shape of kernel as in tuple of size 2
            activation: Activation function.
            reuse: (bool) whether to reuse the weights of a previous layer by the same name.
            name: Name of the module
        Raises:
            ValueError: If data_format is not 'channels_first' or 'channels_last'
        """
        super(ConvRNNCell, self).__init__(_reuse=reuse, name=name)
        self._input_shape = input_shape
        self._output_channels = output_channels
        self._kernel_shape = kernel_shape

        if activation:
            self._activation = activations.get(activation)
        else:
            self._activation = tf.tanh

        self._output_size = tf.TensorShape([self._output_channels] + self._input_shape[1:])

    @property
    def output_size(self):
        return self._output_size

    @property
    def state_size(self):
        return self._output_size

    def call(self, inputs, state, scope=None):
        args = [inputs, state]

        total_arg_size_depth = 0
        shapes = [a.get_shape().as_list() for a in args]
        shape_length = len(shapes[0])
        for shape in shapes:
            if len(shape) != 4:
                raise ValueError("Conv Linear expects 4D arguments: %s" % str(shapes))
            if len(shape) != len(shapes[0]):
                raise ValueError("Conv Linear expects all args "
                                 "to be of same Dimension: %s" % str(shapes))
            else:
                total_arg_size_depth += shape[1]
        dtype = [a.dtype for a in args][0]

        inputs = tf.concat(args, axis=1)

        strides = shape_length * [1]
        kernel = tf.get_variable('kernel', self._kernel_shape + [total_arg_size_depth, self._output_channels],
                                 dtype=dtype)

        new_hidden = tf.nn.conv2d(inputs, kernel, strides, padding='SAME', data_format='NCHW')

        output = self._activation(new_hidden)
        return output, output

class ConvGRUCell(tf.nn.rnn_cell.RNNCell):
    """A GRU cell with convolutions instead of multiplications."""
    def __init__(self, input_shape, output_channels, kernel_shape,
                 use_bias=True,
                 activation=tf.tanh,
                 normalize=True,
                 data_format='channels_last',
                 reuse=None):
        """Construct ConvGRUCell.

        Args:
            input_shape: (int, int, int) Shape of the input as int tuple, excluding the batch size
            output_channels: (int) number of output channels of the conv LSTM
            kernel_shape: (int, int) Shape of kernel as in tuple of size 2
            use_bias: (bool) whether the convolutions use biases
            activation: Activation function.
            normalize: (bool) whether to layer normalize the conv output
            data_format: A string, one of 'channels_last' (default) or 'channels_first'. The ordering of the dimensions in
                the inputs. channels_last corresponds to inputs with shape (batch, height, width, channels) while
                channels_first corresponds to inputs with shape (batch, channels, height, width)
            reuse: (bool) whether to reuse the weights of a previous layer by the same name.
        Raises:
            ValueError: If data_format is not 'channels_first' or 'channels_last'
        """
        super(ConvGRUCell, self).__init__(_reuse=reuse)
        self._filters = output_channels
        self._kernel = kernel_shape
        self._use_bias = use_bias
        self._activation = activation
        self._normalize = normalize

        self._channel_first_dict = {1: 'NCW', 2: 'NCHW', 3: 'NCDHW'}

        if data_format == 'channels_last':
            self._size = tf.TensorShape(input_shape + [self._filters])
            self._feature_axis = self._size.ndims
            self._data_format = None
        elif data_format == 'channels_first':
            self._size = tf.TensorShape([self._filters] + input_shape)
            self._feature_axis = 1
            self._data_format = self._channel_first_dict[len(self._kernel)]
        else:
            raise ValueError('Unknown data_format')

    @property
    def state_size(self):
        return self._size

    @property
    def output_size(self):
        return self._size

    def call(self, x, h):
        channels = x.shape[self._feature_axis].value

        with tf.variable_scope('gates'):
            inputs = tf.concat([x, h], axis=self._feature_axis)
            n = channels + self._filters
            m = 2 * self._filters if self._filters > 1 else 2
            W = tf.get_variable('kernel', self._kernel + [n, m])
            y = tf.nn.convolution(inputs, W, 'SAME', data_format=self._data_format)
            if self._normalize:
                r, u = tf.split(y, 2, axis=self._feature_axis)
                r = tf.contrib.layers.layer_norm(r)
                u = tf.contrib.layers.layer_norm(u)
            elif self._use_bias:
                y += tf.get_variable('bias', [m], initializer=tf.ones_initializer())
                r, u = tf.split(y, 2, axis=self._feature_axis)
            else:
                r, u = tf.split(y, 2, axis=self._feature_axis)
            r, u = tf.sigmoid(r), tf.sigmoid(u)

        with tf.variable_scope('candidate'):
            inputs = tf.concat([x, r * h], axis=self._feature_axis)
            n = channels + self._filters
            m = self._filters
            W = tf.get_variable('kernel', self._kernel + [n, m])
            y = tf.nn.convolution(inputs, W, 'SAME', data_format=self._data_format)
            if self._normalize:
                y = tf.contrib.layers.layer_norm(y)
            elif self._use_bias:
                y += tf.get_variable('bias', [m], initializer=tf.zeros_initializer())
            h = u * h + (1 - u) * self._activation(y)

        return h, h

class MyConv2DLSTMCell(tf.nn.rnn_cell.RNNCell):
    """A LSTM cell with convolutions instead of multiplications.
    Reference:
        Xingjian, S. H. I., et al. "Convolutional LSTM network: A machine learning approach for precipitation
            nowcasting." Advances in Neural Information Processing Systems. 2015.
    """
    def __init__(self,
                 input_shape,
                 output_channels,
                 kernel_shape,
                 use_bias=True,
                 forget_bias=1.0,
                 activation=tf.tanh,
                 data_format='channels_last',
                 reuse=None,
                 name="conv_2d_lstm_cell"):
        """Construct Conv2DLSTMCell.

        Args:
            input_shape: (int, int, int) Shape of the input as int tuple, excluding the batch size
            output_channels: (int) number of output channels of the conv LSTM
            kernel_shape: (int, int) Shape of kernel as in tuple of size 2
            use_bias: (bool) whether the convolutions use biases
            forget_bias: (float) Forget bias
            activation: Activation function.
            data_format: A string, one of 'channels_last' (default) or 'channels_first'. The ordering of the dimensions in
                the inputs. channels_last corresponds to inputs with shape (batch, height, width, channels) while
                channels_first corresponds to inputs with shape (batch, channels, height, width)
            reuse: (bool) whether to reuse the weights of a previous layer by the same name.
            name: Name of the module
        Raises:
            ValueError: If data_format is not 'channels_first' or 'channels_last'
        """
        super(MyConv2DLSTMCell, self).__init__(_reuse=reuse, name=name)

        self._input_shape = input_shape
        self._output_channels = output_channels
        self._kernel_shape = kernel_shape
        self._use_bias = use_bias
        self._forget_bias = forget_bias
        self._activation = activation

        if data_format == 'channels_last':
            state_size = tf.TensorShape(self._input_shape[:-1] + [self._output_channels])
            self._state_size = tf.nn.rnn_cell.LSTMStateTuple(state_size, state_size)
            self._output_size = state_size

            self._feature_axis = self.state_size.ndims
            self._data_format = None
        elif data_format == 'channels_first':
            state_size = tf.TensorShape([self._output_channels] + self._input_shape[1:])
            self._state_size = tf.nn.rnn_cell.LSTMStateTuple(state_size, state_size)
            self._output_size = state_size

            self._feature_axis = 1
            self._data_format = 'NCHW'
        else:
            raise ValueError('Unknown data_format')

    @property
    def state_size(self):
        return self._state_size

    @property
    def output_size(self):
        return self._output_size

    def call(self, inputs, state, scope=None):
        cell, hidden = state
        args = [inputs, hidden]

        total_arg_size_depth = 0
        shapes = [a.get_shape().as_list() for a in args]
        shape_length = len(shapes[0])
        for shape in shapes:
            if len(shape) != 4:
                raise ValueError("Conv Linear expects 4D arguments: %s" % str(shapes))
            if len(shape) != len(shapes[0]):
                raise ValueError("Conv Linear expects all args "
                                 "to be of same Dimension: %s" % str(shapes))
            else:
                total_arg_size_depth += shape[self._feature_axis]
        dtype = [a.dtype for a in args][0]

        inputs = tf.concat(args, axis=self._feature_axis)

        num_features = 4 * self._output_channels if self._output_channels > 1 else 4
        strides = shape_length * [1]

        kernel = tf.get_variable('kernel', self._kernel_shape + [total_arg_size_depth, num_features], dtype=dtype)

        new_hidden = nn_ops.conv2d(inputs, kernel, strides, padding='SAME', data_format=self._data_format)
        if self._use_bias:
            new_hidden += tf.get_variable('bias', [num_features], initializer=tf.zeros_initializer(), dtype=dtype)

        gates = tf.split(new_hidden, 4, axis=self._feature_axis)

        input_gate, new_input, forget_gate, output_gate = gates
        new_cell = tf.sigmoid(forget_gate + self._forget_bias) * cell
        new_cell += tf.sigmoid(input_gate) * self._activation(new_input)
        output = self._activation(new_cell) * tf.sigmoid(output_gate)

        new_state = tf.nn.rnn_cell.LSTMStateTuple(new_cell, output)
        return output, new_state

class BNConvGRUCell(tf.nn.rnn_cell.RNNCell):
    """A GRU cell with convolutions instead of multiplications."""
    def __init__(self, input_shape, output_channels, kernel_shape, max_bn_steps, training,
                 use_bias=False,
                 activation=tf.tanh,
                 momentum=0.95,
                 initial_scale=0.1,
                 reuse=None):
        """Construct ConvGRUCell.

        Args:
            input_shape: (int, int, int) Shape of the input as int tuple, excluding the batch size
            output_channels: (int) number of output channels of the conv LSTM
            kernel_shape: (int, int) Shape of kernel as in tuple of size 2
            activation: Activation function.
            reuse: (bool) whether to reuse the weights of a previous layer by the same name.
        Raises:
            ValueError: If data_format is not 'channels_first' or 'channels_last'
        """
        super(BNConvGRUCell, self).__init__(_reuse=reuse)
        self._filters = output_channels
        self._kernel = kernel_shape
        self._max_bn_steps = max_bn_steps
        self._training = tf.constant(training, tf.bool)
        self._use_bias = use_bias
        self._activation = activation
        self._momentum = momentum
        self._initial_scale = initial_scale

        self._size = tf.TensorShape(input_shape + [self._filters])
        self._feature_axis = self._size.ndims

    @property
    def state_size(self):
        return self._size

    @property
    def output_size(self):
        return self._size

    def _batch_norm(self, x, name_scope, step, epsilon=1e-5):
        with tf.variable_scope(name_scope):
            size = x.get_shape().as_list()[-1]

            scale = tf.get_variable('scale', [size], initializer=tf.constant_initializer(self._initial_scale))
            offset = 0

            pop_mean_all_steps = tf.get_variable('pop_mean', [self._max_bn_steps, size],
                                                 initializer=tf.zeros_initializer(), trainable=False)
            pop_var_all_steps = tf.get_variable('pop_var', [self._max_bn_steps, size],
                                                initializer=tf.ones_initializer(), trainable=False)

            step = tf.minimum(step, self._max_bn_steps - 1)

            pop_mean = pop_mean_all_steps[step]
            pop_var = pop_var_all_steps[step]

            batch_mean, batch_var = tf.nn.moments(x, [0, 1, 2])

            def batch_statistics():
                pop_mean_new = pop_mean * self._momentum + batch_mean * (1 - self._momentum)
                pop_var_new = pop_var * self._momentum + batch_var * (1 - self._momentum)
                with tf.control_dependencies([pop_mean.assign(pop_mean_new), pop_var.assign(pop_var_new)]):
                    return tf.nn.batch_normalization(x, batch_mean, batch_var, offset, scale, epsilon)

            def population_statistics():
                return tf.nn.batch_normalization(x, pop_mean, pop_var, offset, scale, epsilon)

            return tf.cond(self._training, batch_statistics, population_statistics)

    def call(self, x, state, scope=None):
        h, step = state

        with tf.variable_scope('gates'):
            channels = x.shape[-1].value
            m = 2 * self._filters

            kernel_x = tf.get_variable('kernel_x', self._kernel + [channels, m])
            kernel_h = tf.get_variable('kernel_h', self._kernel + [self._filters, m])

            rux = tf.nn.convolution(x, kernel_x, 'SAME')
            ruh = tf.nn.convolution(h, kernel_h, 'SAME')

            rux = self._batch_norm(rux, 'rux', step)
            ruh = self._batch_norm(ruh, 'ruh', step)

            if self._use_bias:
                bias = tf.get_variable('bias_r', [m], initializer=tf.ones_initializer())
                ru = tf.nn.bias_add(rux + ruh, bias)
            else:
                ru = rux + ruh

            ru = tf.sigmoid(ru)
            r, u = tf.split(ru, 2, axis=-1)

        with tf.variable_scope('candidate'):
            channels = x.shape[-1].value
            m = self._filters

            kernel_x = tf.get_variable('kernel_x', self._kernel + [channels, m])
            kernel_h = tf.get_variable('kernel_h', self._kernel + [self._filters, m])

            hx = tf.nn.convolution(x, kernel_x, 'SAME')
            hh = tf.nn.convolution(r * h, kernel_h, 'SAME')

            hx = self._batch_norm(hx, 'hx', step)
            hh = self._batch_norm(hh, 'hh', step)

            if self._use_bias:
                bias_h = tf.get_variable('bias', [self._filters], initializer=tf.zeros_initializer())
                hc = tf.nn.bias_add(hx + hh, bias_h)
            else:
                hc = hx + hh

            h = u * h + (1 - u) * self._activation(hc)

        return h, (h, step+1)

max_stars_repo_name: yesArjan/Playing-Go-with-RNNs

max_issues_repo_name: yesArjan/Playing-Go-with-RNNs

max_forks_repo_name: yesArjan/Playing-Go-with-RNNs


text:
#!/usr/bin/env python
import sys
import os
import rospy
import numpy as np
import cv2
import pcl
import glob
import math
import std_msgs.msg
import sensor_msgs.point_cloud2 as pc2
from sensor_msgs.msg import PointCloud2
from parse_xml import parseXML

def load_pc_from_pcd(pcd_path):
    """Load PointCloud data from pcd file."""
    p = pcl.load(pcd_path)
    return np.array(list(p), dtype=np.float32)

def load_pc_from_bin(bin_path):
    """Load PointCloud data from pcd file."""
    obj = np.fromfile(bin_path, dtype=np.float32).reshape(-1, 4)
    return obj

def read_label_from_txt(label_path):
    """Read label from txt file."""
    text = np.fromfile(label_path)
    bounding_box = []
    with open(label_path, "r") as f:
        labels = f.read().split("\n")
        for label in labels:
            if not label:
                continue
            label = label.split(" ")
            if (label[0] == "DontCare"):
                continue

            if label[0] == ("Car" or "Van"): #  or "Truck"
                bounding_box.append(label[8:15])

    if bounding_box:
        data = np.array(bounding_box, dtype=np.float32)
        return data[:, 3:6], data[:, :3], data[:, 6]
    else:
        return None, None, None

def read_label_from_xml(label_path):
    """Read label from xml file.

    # Returns:
        label_dic (dictionary): labels for one sequence.
        size (list): Bounding Box Size. [l, w. h]?
    """
    labels = parseXML(label_path)
    label_dic = {}
    for label in labels:
        first_frame = label.firstFrame
        nframes = label.nFrames
        size = label.size
        obj_type = label.objectType
        for index, place, rotate in zip(range(first_frame, first_frame+nframes), label.trans, label.rots):
            if index in label_dic.keys():
                label_dic[index]["place"] = np.vstack((label_dic[index]["place"], place))
                label_dic[index]["size"] = np.vstack((label_dic[index]["size"], np.array(size)))
                label_dic[index]["rotate"] = np.vstack((label_dic[index]["rotate"], rotate))
            else:
                label_dic[index] = {}
                label_dic[index]["place"] = place
                label_dic[index]["rotate"] = rotate
                label_dic[index]["size"] = np.array(size)
    return label_dic, size

def read_calib_file(calib_path):
    """Read a calibration file."""
    data = {}
    with open(calib_path, 'r') as f:
        for line in f.readlines():
            if not line or line == "\n":
                continue
            key, value = line.split(':', 1)
            try:
                data[key] = np.array([float(x) for x in value.split()])
            except ValueError:
                pass
    return data

def proj_to_velo(calib_data):
    """Projection matrix to 3D axis for 3D Label"""
    rect = calib_data["R0_rect"].reshape(3, 3)
    velo_to_cam = calib_data["Tr_velo_to_cam"].reshape(3, 4)
    inv_rect = np.linalg.inv(rect)
    inv_velo_to_cam = np.linalg.pinv(velo_to_cam[:, :3])
    return np.dot(inv_velo_to_cam, inv_rect)

def filter_camera_angle(places):
    """Filter camera angles for KiTTI Datasets"""
    bool_in = np.logical_and((places[:, 1] < places[:, 0] - 0.27), (-places[:, 1] < places[:, 0] - 0.27))
    # bool_in = np.logical_and((places[:, 1] < places[:, 0]), (-places[:, 1] < places[:, 0]))
    return places[bool_in]

def create_publish_obj(obj, places, rotates, size):
    """Create object of correct data for publisher"""
    for place, rotate, sz in zip(places, rotates, size):
        x, y, z = place
        obj.append((x, y, z))
        h, w, l = sz
        if l > 10:
            continue
        for hei in range(0, int(h*100)):
            for wid in range(0, int(w*100)):
                for le in range(0, int(l*100)):
                    a = (x - l / 2.) + le / 100.
                    b = (y - w / 2.) + wid / 100.
                    c = (z) + hei / 100.
                    obj.append((a, b, c))
    return obj

def get_boxcorners(places, rotates, size):
    """Create 8 corners of bounding box from bottom center."""
    corners = []
    for place, rotate, sz in zip(places, rotates, size):
        x, y, z = place
        h, w, l = sz
        if l > 10:
            continue

        corner = np.array([
            [x - l / 2., y - w / 2., z],
            [x + l / 2., y - w / 2., z],
            [x - l / 2., y + w / 2., z],
            [x - l / 2., y - w / 2., z + h],
            [x - l / 2., y + w / 2., z + h],
            [x + l / 2., y + w / 2., z],
            [x + l / 2., y - w / 2., z + h],
            [x + l / 2., y + w / 2., z + h],
        ])

        corner -= np.array([x, y, z])

        rotate_matrix = np.array([
            [np.cos(rotate), -np.sin(rotate), 0],
            [np.sin(rotate), np.cos(rotate), 0],
            [0, 0, 1]
        ])

        a = np.dot(corner, rotate_matrix.transpose())
        a += np.array([x, y, z])
        corners.append(a)
    return np.array(corners)

def publish_pc2(pc, obj):
    """Publisher of PointCloud data"""
    pub = rospy.Publisher("/points_raw", PointCloud2, queue_size=1000000)
    rospy.init_node("pc2_publisher")
    header = std_msgs.msg.Header()
    header.stamp = rospy.Time.now()
    header.frame_id = "velodyne"
    points = pc2.create_cloud_xyz32(header, pc[:, :3])

    pub2 = rospy.Publisher("/points_raw1", PointCloud2, queue_size=1000000)
    header = std_msgs.msg.Header()
    header.stamp = rospy.Time.now()
    header.frame_id = "velodyne"
    points2 = pc2.create_cloud_xyz32(header, obj)

    r = rospy.Rate(0.1)
    while not rospy.is_shutdown():
        pub.publish(points)
        pub2.publish(points2)
        r.sleep()

def raw_to_voxel(pc, resolution=0.50, x=(0, 90), y=(-50, 50), z=(-4.5, 5.5)):
    """Convert PointCloud2 to Voxel"""
    logic_x = np.logical_and(pc[:, 0] >= x[0], pc[:, 0] < x[1])
    logic_y = np.logical_and(pc[:, 1] >= y[0], pc[:, 1] < y[1])
    logic_z = np.logical_and(pc[:, 2] >= z[0], pc[:, 2] < z[1])
    pc = pc[:, :3][np.logical_and(logic_x, np.logical_and(logic_y, logic_z))]
    pc =((pc - np.array([x[0], y[0], z[0]])) / resolution).astype(np.int32)
    voxel = np.zeros((int((x[1] - x[0]) / resolution), int((y[1] - y[0]) / resolution), int(round((z[1]-z[0]) / resolution))))
    voxel[pc[:, 0], pc[:, 1], pc[:, 2]] = 1
    return voxel

def center_to_sphere(places, size, resolution=0.50, min_value=np.array([0., -50., -4.5]), scale=4, x=(0, 90), y=(-50, 50), z=(-4.5, 5.5)):
    """Convert object label to Training label for objectness loss"""
    x_logical = np.logical_and((places[:, 0] < x[1]), (places[:, 0] >= x[0]))
    y_logical = np.logical_and((places[:, 1] < y[1]), (places[:, 1] >= y[0]))
    z_logical = np.logical_and((places[:, 2] < z[1]), (places[:, 2] >= z[0]))
    xyz_logical = np.logical_and(x_logical, np.logical_and(y_logical, z_logical))
    center = places.copy()
    center[:, 2] = center[:, 2] + size[:, 0] / 2.
    sphere_center = ((center[xyz_logical] - min_value) / (resolution * scale)).astype(np.int32)
    return sphere_center

def sphere_to_center(p_sphere, resolution=0.5, scale=4, min_value=np.array([0., -50., -4.5])):
    """from sphere center to label center"""
    center = p_sphere * (resolution*scale) + min_value
    return center

def voxel_to_corner(corner_vox, resolution, center):#TODO
    """Create 3D corner from voxel and the diff to corner"""
    corners = center + corner_vox
    return corners

def read_labels(label_path, label_type, calib_path=None, is_velo_cam=False, proj_velo=None):
    """Read labels from xml or txt file.
    Original Label value is shifted about 0.27m from object center.
    So need to revise the position of objects.
    """
    if label_type == "txt": #TODO
        places, size, rotates = read_label_from_txt(label_path)
        if places is None:
            return None, None, None
        rotates = np.pi / 2 - rotates
        dummy = np.zeros_like(places)
        dummy = places.copy()
        if calib_path:
            places = np.dot(dummy, proj_velo.transpose())[:, :3]
        else:
            places = dummy
        if is_velo_cam:
            places[:, 0] += 0.27

    elif label_type == "xml":
        bounding_boxes, size = read_label_from_xml(label_path)
        places = bounding_boxes[30]["place"]
        rotates = bounding_boxes[30]["rotate"][:, 2]
        size = bounding_boxes[30]["size"]

    return places, rotates, size

def create_label(places, size, corners, resolution=0.50, x=(0, 90), y=(-50, 50), z=(-4.5, 5.5), scale=4, min_value=np.array([0., -50., -4.5])):
    """Create training Labels which satisfy the range of experiment"""
    x_logical = np.logical_and((places[:, 0] < x[1]), (places[:, 0] >= x[0]))
    y_logical = np.logical_and((places[:, 1] < y[1]), (places[:, 1] >= y[0]))
    z_logical = np.logical_and((places[:, 2] + size[:, 0]/2. < z[1]), (places[:, 2] + size[:, 0]/2. >= z[0]))
    xyz_logical = np.logical_and(x_logical, np.logical_and(y_logical, z_logical))

    center = places.copy()
    center[:, 2] = center[:, 2] + size[:, 0] / 2. # Move bottom to center
    sphere_center = ((center[xyz_logical] - min_value) / (resolution * scale)).astype(np.int32)

    train_corners = corners[xyz_logical].copy()
    anchor_center = sphere_to_center(sphere_center, resolution=resolution, scale=scale, min_value=min_value) #sphere to center
    for index, (corner, center) in enumerate(zip(corners[xyz_logical], anchor_center)):
        train_corners[index] = corner - center
    return sphere_center, train_corners

def corner_to_train(corners, sphere_center, resolution=0.50, x=(0, 90), y=(-50, 50), z=(-4.5, 5.5), scale=4, min_value=np.array([0., -50., -4.5])):
    """Convert corner to Training label for regression loss"""
    x_logical = np.logical_and((corners[:, :, 0] < x[1]), (corners[:, :, 0] >= x[0]))
    y_logical = np.logical_and((corners[:, :, 1] < y[1]), (corners[:, :, 1] >= y[0]))
    z_logical = np.logical_and((corners[:, :, 2] < z[1]), (corners[:, :, 2] >= z[0]))
    xyz_logical = np.logical_and(x_logical, np.logical_and(y_logical, z_logical)).all(axis=1)
    train_corners = corners[xyz_logical].copy()
    sphere_center = sphere_to_center(sphere_center, resolution=resolution, scale=scale, min_value=min_value) #sphere to center
    for index, (corner, center) in enumerate(zip(corners[xyz_logical], sphere_center)):
        train_corners[index] = corner - center
    return train_corners

def corner_to_voxel(voxel_shape, corners, sphere_center, scale=4):
    """Create final regression label from corner"""
    corner_voxel = np.zeros((voxel_shape[0] / scale, voxel_shape[1] / scale, voxel_shape[2] / scale, 24))
    corner_voxel[sphere_center[:, 0], sphere_center[:, 1], sphere_center[:, 2]] = corners
    return corner_voxel

def create_objectness_label(sphere_center, resolution=0.5, x=90, y=100, z=10, scale=4):
    """Create Objectness label"""
    obj_maps = np.zeros((int(x / (resolution * scale)), int(y / (resolution * scale)), int(round(z / (resolution * scale)))))
    obj_maps[sphere_center[:, 0], sphere_center[:, 1], sphere_center[:, 2]] = 1
    return obj_maps

def process(velodyne_path, label_path=None, calib_path=None, dataformat="pcd", label_type="txt", is_velo_cam=False):
    p = []
    pc = None
    bounding_boxes = None
    places = None
    rotates = None
    size = None
    proj_velo = None

    if dataformat == "bin":
        pc = load_pc_from_bin(velodyne_path)
    elif dataformat == "pcd":
        pc = load_pc_from_pcd(velodyne_path)

    if calib_path:
        calib = read_calib_file(calib_path)
        proj_velo = proj_to_velo(calib)[:, :3]

    if label_path:
        places, rotates, size = read_labels(label_path, label_type, calib_path=calib_path, is_velo_cam=is_velo_cam, proj_velo=proj_velo)

    corners = get_boxcorners(places, rotates, size)
    print("################", len(pc))
    pc = filter_camera_angle(pc)
    # obj = []
    # obj = create_publish_obj(obj, places, rotates, size)

    p.append((0, 0, 0))
    p.append((0, 0, -1))
    print (pc.shape)
    print (1)
    # publish_pc2(pc, obj)
    a = center_to_sphere(places, size, resolution=0.25)
    print (places)
    print (a)
    print (sphere_to_center(a, resolution=0.25))
    bbox = (sphere_to_center(a, resolution=0.25))
    print (corners.shape)
    # publish_pc2(pc, bbox.reshape(-1, 3))
    publish_pc2(pc, corners.reshape(-1, 3))

if __name__ == "__main__":
    # pcd_path = "../data/training/velodyne/000012.pcd"
    # label_path = "../data/training/label_2/000012.txt"
    # calib_path = "../data/training/calib/000012.txt"
    # process(pcd_path, label_path, calib_path=calib_path, dataformat="pcd")

    # bin_path = "../data/2011_09_26/2011_09_26_drive_0001_sync/velodyne_points/data/0000000030.bin"
    # xml_path = "../data/2011_09_26/2011_09_26_drive_0001_sync/tracklet_labels.xml"
    # process(bin_path, xml_path, dataformat="bin", label_type="xml")

    pcd_path = "/home/katou01/download/training/velodyne/000410.bin"
    label_path = "/home/katou01/download/training/label_2/000410.txt"
    calib_path = "/home/katou01/download/training/calib/000410.txt"
    process(pcd_path, label_path, calib_path=calib_path, dataformat="bin", is_velo_cam=True)


text:
#!/usr/bin/env python3
from extras.transformations.camera import transform_img, eon_intrinsics
from extras.transformations.model import medmodel_intrinsics
import numpy as np
from tqdm import tqdm
from tools.lib.framereader_littlemountainman import FrameReader, MP4FrameReader, MKVFrameReader
import matplotlib
import matplotlib.pyplot as plt

import cv2
from tensorflow.keras.models import load_model
from modelparser import parser
import cv2
import sys
camerafile = sys.argv[1]
supercombo = load_model('../../models/supercombo.keras')

MAX_DISTANCE = 140.
LANE_OFFSET = 1.8
MAX_REL_V = 10.

LEAD_X_SCALE = 10
LEAD_Y_SCALE = 10

fr = FrameReader(camerafile)
cap = cv2.VideoCapture(camerafile)

imgs = []
raw_imgs = []
for i in tqdm(range(1000)):
  imgs.append(fr.get(i, pix_fmt='yuv420p')[0].reshape((874*3//2, 1164)))
  ret, frame = cap.read()
  raw_imgs.append(frame)

raw_imgs = np.array(raw_imgs)
print(raw_imgs.shape)
def frames_to_tensor(frames):
  H = (frames.shape[1]*2)//3
  W = frames.shape[2]
  in_img1 = np.zeros((frames.shape[0], 6, H//2, W//2), dtype=np.uint8)

  in_img1[:, 0] = frames[:, 0:H:2, 0::2]
  in_img1[:, 1] = frames[:, 1:H:2, 0::2]
  in_img1[:, 2] = frames[:, 0:H:2, 1::2]
  in_img1[:, 3] = frames[:, 1:H:2, 1::2]
  in_img1[:, 4] = frames[:, H:H+H//4].reshape((-1, H//2,W//2))
  in_img1[:, 5] = frames[:, H+H//4:H+H//2].reshape((-1, H//2,W//2))
  return in_img1

imgs_med_model = np.zeros((len(imgs), 384, 512), dtype=np.uint8)
for i, img in tqdm(enumerate(imgs)):
  imgs_med_model[i] = transform_img(img, from_intr=eon_intrinsics, to_intr=medmodel_intrinsics, yuv=True,
                                    output_size=(512,256))
frame_tensors = frames_to_tensor(np.array(imgs_med_model)).astype(np.float32)/128.0 - 1.0

state = np.zeros((1,512))
desire = np.zeros((1,8))

for i in tqdm(range(len(frame_tensors) - 1)):
  inputs = [np.vstack(frame_tensors[i:i+2])[None], desire, state]
  outs = supercombo.predict(inputs)
  parsed = parser(outs)
  # Important to refeed the state
  state = outs[-1]
  pose = outs[-2]

  # Show raw camera image
  cv2.imshow("modeld", raw_imgs[i])
  # Clean plot for next frame
  plt.clf()
  plt.title("lanes and path")
  # lll = left lane line
  plt.scatter(parsed["lll"], range(0,192), c="b")
  # rll = right lane line
  plt.scatter(parsed["rll"], range(0, 192), c="r")
  # path = path cool isn't it ?
  plt.scatter(parsed["path"], range(0, 192), c="g")
  print(np.array(pose[0,:3]).shape)
  #plt.scatter(pose[0,:3], range(3), c="y")

  # Needed to invert axis because standart left lane is positive and right lane is negative, so we flip the x axis
  plt.gca().invert_xaxis()
  plt.pause(0.05)
  if cv2.waitKey(10) & 0xFF == ord('q'):
        break

plt.show()

max_stars_repo_name: galenzhao/openpilot

max_issues_repo_name: galenzhao/openpilot

max_forks_repo_name: galenzhao/openpilot


text:
import numpy as np
import torch

def already_ohe(y):
    if len(y.shape) <= 1:
        return False
    return (y.sum(axis=1)-np.ones(y.shape[0])).sum() == 0

def one_hot_encode(y, nb_classes):
    if isinstance(y, (np.ndarray, torch.Tensor)):
        if len(y.shape) == 1:
            y = np.expand_dims(y, 0)
    else:
        y = np.expand_dims(np.array(y), 0)
    try:
        res = np.eye(nb_classes)[y.reshape(-1)]
        res = res.reshape(list(y.shape)+[nb_classes]).squeeze()
        return res
    except:
        return y

def soften_label(y, num_classes=None):
    if not num_classes:
        if isinstance(y, int):
            num_classes = max(2, y + 1)
        elif isinstance(y, list):
            num_classes = len(y)
        elif isinstance(y, np.ndarray):
            if len(y.shape) == 1:
                num_classes = len(y)
            else:
                num_classes = y.shape[-1]
    return one_hot_encode(y, num_classes)

def invert_label(y, soften=False, num_classes=None):
    if not isinstance(y, np.ndarray):
        y = soften_label(y, num_classes)
    y = y[::-1]
    if not soften:
        y = np.argmax(y)
    return y

def interpolate_label(y1, y2, x1=None, x2=None, num_classes=None, y_weights=None):
    if isinstance(y_weights, (list, tuple)):
        mass_y1 = y_weights[0]
        mass_y2 = y_weights[1]
    elif x1 and x2:
        mass_y1 = len(x1) / (len(x1) + len(x2))
        mass_y2 = 1 - mass_y1
    else:
        mass_y1 = 1
        mass_y2 = 1
    y1 = soften_label(y1, num_classes) * mass_y1
    y2 = soften_label(y2, num_classes) * mass_y2
    return (y1 + y2) / (y1 + y2).sum()

def weight_label(y, y_weights=None):
    if not y_weights:
        y_weights = np.ones_like(y)
    y = y * np.array(y_weights)
    return y / y.sum()

def smooth_label(y, factor=0.1):
    if not isinstance(y, np.ndarray):
        y = soften_label(y)
    y = y * (1. - factor)
    y = y + (factor / y.shape[-1])
    return y

# labels1 = [0, 1]
# labels2 = np.array([0, 1])

# hard_label_int   = 3
# hard_labels_list = [0,1,2,3]
# hard_labels_arr  = np.array([0,1,2,3])
# soft_labels_arr  = np.array([0.45035461, 0., 0., 0.54964539])

# print(one_hot_encode(labels1, 2))
# print(one_hot_encode(labels2, 2))

# print(one_hot_encode(hard_label_int, 4))
# print(one_hot_encode(hard_labels_list, 4))
# print(one_hot_encode(hard_labels_arr, 4))
# print(one_hot_encode(soft_labels_arr, 4))

max_stars_repo_name: UCLA-SEAL/Sibyl

max_issues_repo_name: UCLA-SEAL/Sibyl

max_forks_repo_name: UCLA-SEAL/Sibyl


text:
import os
import swhlab
import matplotlib.pyplot as plt
import numpy as np
import pickle

if __name__=="__main__":
    abfPath=r"X:\Data\2P01\2016\2016-09-01 PIR TGOT"
    abfFile=os.path.join(abfPath,"16d14060.abf")
    abf=swhlab.ABF(abfFile)
    Xs=np.arange(abf.sweeps)*abf.sweepLength

    # figure out if we should calculate or load phasic data
    phasicFname=abf.filename+".phasic.npy"
    if not os.path.exists(phasicFname):
        data=np.empty((abf.sweeps,3))
        for sweep in abf.setsweeps():
            print("analyzing sweep %d of %d"%(sweep,abf.sweeps))
            data[sweep]=abf.phasic(returnSeparated=True)
        np.save(phasicFname,data)
    else:
        data = np.load(phasicFname)

    # plot the result with comments
    plt.figure(figsize=(5,5))

    plt.subplot(211)
    plt.plot(Xs,data[:,2],'.',color='k',alpha=.1)
    plt.plot(Xs,swhlab.common.lowpass(data[:,2],40), '-',color='k',alpha=.5,lw=2)
    for t in abf.comment_times:
        plt.axvline(t,color='r',ls='--',alpha=.5)
    plt.axhline(0,color='r',ls='--',alpha=.5)
    plt.margins(0,.1)
    plt.tight_layout()
    plt.ylabel("net excitation (pA)")
    plt.title(abf.ID)

    plt.subplot(212)
    plt.plot(Xs,data[:,0],'.',color='r',alpha=.1)
    plt.plot(Xs,swhlab.common.lowpass(data[:,0],40), '-',color='r',alpha=.5,lw=2)
    plt.plot(Xs,data[:,1],'.',color='b',alpha=.1)
    plt.plot(Xs,swhlab.common.lowpass(data[:,1],40), '-',color='b',alpha=.5,lw=2)
    for t in abf.comment_times:
        plt.axvline(t,color='r',ls='--',alpha=.5)
    plt.margins(0,.1)
    plt.tight_layout()
    plt.ylabel("separated excitation (pA)")
    plt.xlabel("experiment duration (sec)")

    plt.savefig(abf.ID+".png")
    plt.show()

    print("DONE")

max_stars_repo_path: doc/uses/EPSCs-and-IPSCs/variance method/2016-12-14 epsc ipsc report.py

max_stars_repo_name: swharden/SWHLab

max_issues_repo_name: swharden/SWHLab

max_forks_repo_name: swharden/SWHLab


text:
# -*- coding: utf-8 -*-
# @File    : image_caption_with_attention_r9.1/image_cap_dataset_prepare.py
# @Info    : @ TSMC-SIGGRAPH, 2018/9/26
# @Desc    :
# -.-.. - ... -- -.-. .-.. .- -... .---.   -.-- ..- .-.. --- -. --.   ..-. .- -.

import json
import os

import numpy as np
import tensorflow as tf
# # Scikit-learn includes many helpful utilities
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from tqdm import tqdm

from configuration import ModelConfig

os.environ["CUDA_VISIBLE_DEVICES"] = '3'  # only /gpu:gpu_id is visible

class ImageCapDataBase(object):
    """Image Caption Data Base"""

    def __init__(self, config: ModelConfig):
        """
        :param config: an instance of class ModelConfig
        """
        self.config = config
        self.annotation_file = '/dataset/cocodataset/annotations/captions_train2014.json'
        self.img_metadata_dir = os.path.join('/dataset/cocodataset', 'train2014')
        self.cache_dir = '/dataset/cocodataset/inceptionv3_conv_feature'

        self.train_captions = None
        self.img_name_vector = None

        # holding the preprocessed and tokenize captions.
        self.tokenizer = None
        # mapping ( index -> word )
        self.index_word = None

        # pad_sequence
        self.cap_vector = None

        # pre-trained model
        self.image_features_extract_model = None

        # Split the data into training and testing
        self.img_name_train, self.img_name_val, self.cap_train, self.cap_val = None, None, None, None

        # batch dataset for training
        self.dataset_train = None

        # images full path
        self.batch_path = None
        self.batch_features = None

    def load_json_file(self):
        # read the json file
        with open(self.annotation_file, 'r') as f:
            annotations = json.load(f)

        # storing the captions and the image name in vectors
        all_captions = []
        all_img_name_vector = []

        for annot in annotations['annotations']:
            caption = '<start> ' + annot['caption'] + ' <end>'
            image_id = annot['image_id']
            full_coco_image_path = os.path.join(self.img_metadata_dir, 'COCO_train2014_' + '%012d.jpg' % image_id)

            all_img_name_vector.append(full_coco_image_path)
            all_captions.append(caption)

        # shuffling the captions and image_names together
        # setting a random state,  guaranteed to have the same random sequence if you use the same seed.
        train_captions, img_name_vector = shuffle(all_captions, all_img_name_vector, random_state=1)

        self.train_captions = train_captions[:self.config.limit_num_examples]
        self.img_name_vector = img_name_vector[:self.config.limit_num_examples]
        print("[load_json_file] (captions,img) tuples random select ", len(self.train_captions), len(self.img_name_vector))

    @staticmethod
    def load_image(image_path):
        """convert the images into the format inceptionV3 expects
        resizing the image to (299,299),using the `preprocess_input` method to place the pixels in the range of -1 to 1
        :param image_path: image path, a `Tensor` of type `string`.
        :return:
        """
        img = tf.read_file(image_path)
        img = tf.image.decode_jpeg(img, channels=3)
        img = tf.image.resize_images(img, (299, 299))
        img = tf.keras.applications.inception_v3.preprocess_input(img)
        return img, image_path

    def build_inception_model(self):
        """Initialize InceptionV3 and load the pretrained Imagenet weights
        :return:
        """
        # we'll create a tf.keras model where the output layer is the last convolutional layer in the InceptionV3 architecture.
        image_model = tf.keras.applications.InceptionV3(include_top=False, weights='imagenet')
        image_model.trainable = False
        new_input = image_model.input
        hidden_layer = image_model.layers[-1].output
        # hidden_layer = image_model.get_layer("mixed10").output
        self.image_features_extract_model = tf.keras.models.Model(inputs=new_input, outputs=hidden_layer)

    def caching_inception_feature(self):
        """
        After all the images are passed through the network, we pickle the dictionary and save it to disk.
        Caching the features extracted from InceptionV3
        """
        # getting the unique images
        encode_train = sorted(set(self.img_name_vector))
        image_dataset = tf.data.Dataset.from_tensor_slices(encode_train).map(self.load_image).batch(self.config.batch_size)
        # follow snippets need eager execution
        for img, path in tqdm(image_dataset):
            batch_features = self.image_features_extract_model(img)
            batch_features = tf.reshape(batch_features, (batch_features.shape[0], -1, batch_features.shape[3]))
            for bf, p in zip(batch_features, path):
                path_of_img = p.numpy().decode('utf-8')
                filename = os.path.basename(path_of_img)
                path_of_feature = os.path.join(self.cache_dir, filename)
                np.save(path_of_feature, bf.numpy())

        print(image_dataset.output_types)
        print(image_dataset.output_shapes)

    @staticmethod
    def calc_max_length(tensor):
        """
        find the maxinum length of any caption in our dataset
        :param tensor:  an array with shape (n,m)
        :return:
        """
        return max(len(t) for t in tensor)

    def map_func(self, img_name, cap):
        filename = os.path.basename(img_name.decode('utf-8'))
        filename = os.path.join(self.cache_dir, filename + '.npy')
        img_tensor = np.load(filename)
        return img_tensor, cap

    def process_captions(self):
        """
        Preprocess and tokenize the captions.
        1). First, we'll tokenize the captions (e.g., by splitting on spaces).
        This will give us a vocabulary of all the unique words in the data (e.g., "surfing", "football", etc).

        2). Next, we'll limit the vocabulary size to the top 5,000 words to save memory.
        We'll replace all other words with the token "UNK" (for unknown).

        3). Finally, we create a word --> index mapping and vice-versa.
        We will then pad all sequences to the be same length as the longest one.
        :return:
        """
        # The steps above is a general process of dealing with text processing
        # choosing the top k words from the vocabulary
        self.tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=self.config.top_k,
                                                               filters='!"#$%&()*+.,-/:;=?@[\]^_`{|}~ ',
                                                               oov_token="<unk>")
        self.tokenizer.fit_on_texts(self.train_captions)

        # choose the top_k words
        self.tokenizer.word_index = {key: value for key, value in self.tokenizer.word_index.items()
                                     if value <= self.config.top_k}
        # putting <unk> token in the word2idx dictionary
        self.tokenizer.word_index[self.tokenizer.oov_token] = self.config.top_k + 1
        self.tokenizer.word_index['<pad>'] = 0

        self.config.vocab_size = len(self.tokenizer.word_index)

        # creating the tokenized vectors
        train_seqs = self.tokenizer.texts_to_sequences(self.train_captions)

        # we truncate captions longer than 20 words for COCO
        truncate_train_seqs = []
        for line in train_seqs:
            if len(line) > 22:
                line = line[:22]    # don't need to add '<end>' symbol
            truncate_train_seqs.append(line)

        # creating a reverse mapping ( index -> word )
        self.index_word = {value: key for key, value in self.tokenizer.word_index.items()}

        # padding each vector to the max_length of the captions
        # if the max_length parameter is not provided, pad_sequences calculates that automatically.
        self.cap_vector = tf.keras.preprocessing.sequence.pad_sequences(truncate_train_seqs, padding='post')

        # calculating the max_length
        # used to store the attention weights
        max_length = self.calc_max_length(truncate_train_seqs)
        print("[process_captions] the longest captions length: {}".format(max_length))

    def build_dataset(self):
        # Split the data into training and testing using 0.9/0.1 split
        self.img_name_train, self.img_name_val, self.cap_train, self.cap_val = train_test_split(self.img_name_vector,
                                                                                self.cap_vector,
                                                                                test_size=0.1,
                                                                                random_state=0)
        print("[build_dataset] train examples: {}, val examples : {}".format(len(self.img_name_train), len(self.img_name_val)))

        dataset_train = tf.data.Dataset.from_tensor_slices((self.img_name_train, self.cap_train))

        # using map to load the numpy file in parallel
        # https://www.tensorflow.org/api_docs/python/tf/py_func
        # Given a python function func, which takes numpy arrays as its arguments and returns numpy arrays as its outputs,
        # wrap this function as an operation in a TensorFlow graph.
        dataset_train = dataset_train.map(lambda item1, item2:
                                          tf.py_func(self.map_func, [item1, item2], [tf.float32, tf.int32]),
                                          num_parallel_calls=2)

        # shuffling and batching
        dataset_train = dataset_train.shuffle(self.config.buffer_size)
        dataset_train = dataset_train.repeat(self.config.epoch)
        self.dataset_train = dataset_train.batch(self.config.batch_size)
        # self.dataset_train = dataset_train.prefetch(1)

    def build(self):
        self.load_json_file()
        self.build_inception_model()
        # self.caching_inception_feature()
        self.process_captions()

max_stars_repo_name: yulongfan/tryEverything

max_issues_repo_name: yulongfan/tryEverything

max_forks_repo_name: yulongfan/tryEverything


text:
# helpers.py

import math
import numpy as np

### Influence functions and their parents. ###

def rho_catnew(u):
    '''
    Parent derived from the influence function of
    Catoni and Giulini (2017)
    '''
    return np.where((np.abs(u) > math.sqrt(2)),
                    (np.abs(u)*2*math.sqrt(2)/3-0.5),
                    (u**2/2-u**4/24))

def psi_catnew(u):
    '''
    Influence function of Catoni and Giulini (2017).
    '''
    return np.where((np.abs(u) > math.sqrt(2)),
                    (np.sign(u)*2*math.sqrt(2)/3),
                    (u-u**3/6))

def psi_catnew_deriv(u):
    '''
    Derivative of influence function of Catoni and Giulini (2017).
    '''
    return np.where((np.abs(u) >= math.sqrt(2)), 0., (1.-u**2/2.))

CONST_catnew = math.sqrt(81/32)

def rho_lcosh(u):
    '''
    A classic example of a Catoni-type function (parent).
    '''
    return np.log(np.cosh(u))

def psi_lcosh(u):
    '''
    A classic example of a Cat-type influence function.
    '''
    return np.tanh(u)

def rho_hinge(u):
    '''
    The hinge function used to enforce relaxed constraints
    in a margin-maximization setting.
    '''
    return np.where((u>=0), u, 0)

def psi_hinge(u):
    '''
    The first-order derivative of the hinge function used to
    enforce relaxed constraints in a margin-maximization
    setting.
    '''
    return np.where((u>=0), 1, 0)

max_stars_repo_name: feedbackward/catcube

max_issues_repo_name: feedbackward/catcube

max_forks_repo_name: feedbackward/catcube


text:
# -*- coding: utf-8 -*-
"""
"""
from __future__ import division, print_function, unicode_literals

#import pytest
import declarative
#import numpy as np
import numpy.testing as test

from phasor.utilities.future_from_2 import super
import phasor.signals as signals
import phasor.readouts as readouts
import phasor.system as system
from phasor import base

class VCOTest(signals.SignalElementBase):
    f_dict = None

    @declarative.dproperty
    def F_AOM1(self):
        val = base.Frequency(
            F_Hz  = 200e6,
            order = 1,
        )
        return val

    @declarative.dproperty
    def generate(self):
        val = signals.SignalGenerator(
            F = self.F_AOM1,
            amplitude = 1,
        )
        return val

    @declarative.dproperty
    def modulate(self):
        val = signals.Modulator()
        return val

    @declarative.dproperty
    def mix(self):
        val = signals.Mixer()
        return val

    @declarative.dproperty
    def AC_I(self, val = None):
        val = readouts.ACReadout(
            portN = self.mix.ps_R_I.o,
            portD  = self.modulate.Mod_amp.i,
        )
        return val

    @declarative.dproperty
    def AC_IQ(self, val = None):
        val = readouts.ACReadout(
            portN = self.mix.ps_R_Q.o,
            portD  = self.modulate.Mod_amp.i,
        )
        return val

    @declarative.dproperty
    def AC_QI(self, val = None):
        val = readouts.ACReadout(
            portN = self.mix.ps_R_Q.o,
            portD  = self.modulate.Mod_amp.i,
        )
        return val

    @declarative.dproperty
    def AC_Q(self, val = None):
        val = readouts.ACReadout(
            portN = self.mix.ps_R_Q.o,
            portD  = self.modulate.Mod_phase.i,
        )
        return val

    def __build__(self):
        super().__build__()
        self.generate.ps_Out.bond(
            self.modulate.ps_In,
        )
        self.mix.LO.bond(self.generate.ps_Out)
        self.modulate.ps_Out.bond(
            self.mix.ps_In,
        )

def test_VCO():
    db = declarative.DeepBunch()
    db.environment.F_AC.order = 1
    sys = system.BGSystem(
        ctree = db,
        F_AC = 1e3,
    )
    sys.own.test = VCOTest()
    db = sys.ctree_shadow()

    print(sys.test.AC_I.AC_sensitivity)
    print(sys.test.AC_Q.AC_sensitivity)
    print(sys.test.AC_IQ.AC_sensitivity)
    print(sys.test.AC_QI.AC_sensitivity)

    test.assert_almost_equal(sys.test.AC_I.AC_sensitivity / 0.5, 1)
    test.assert_almost_equal(sys.test.AC_Q.AC_sensitivity / 0.5, 1)
    test.assert_almost_equal(sys.test.AC_IQ.AC_sensitivity, 0)
    test.assert_almost_equal(sys.test.AC_QI.AC_sensitivity, 0)
    return

if __name__ == '__main__':
    test_VCO()

max_stars_repo_name: mccullerlp/OpenLoop

max_issues_repo_name: mccullerlp/OpenLoop

max_forks_repo_name: mccullerlp/OpenLoop