text:
import sys
import os
sys.path.append(os.getcwd())

import numpy as np
import torch
import torch.nn.functional as F
import os
import tqdm
import time
import h5py
import argparse
import json
from collections import defaultdict
import subprocess

from diffdvr import Renderer, CameraOnASphere, Settings, setup_default_settings, \
    fibonacci_sphere, renderer_dtype_torch, renderer_dtype_np, VolumeDensities, \
    TfTexture, SmoothnessPrior, toCHW, VolumePreshaded
from losses import LossBuilder
import pyrenderer

class Config:
    def __init__(self):
        pass

    def init_parser(self, parser: argparse.ArgumentParser):
        parser.add_argument('outputFile', type=str, help="Output .hdf5 file")
        # dataset
        parser_group = parser.add_argument_group("Data")
        parser.add_argument('settingsFile', type=str, help="Settings .json file")
        parser_group.add_argument('-v', '--views', default=1, type=int, help="Number of views")
        parser_group.add_argument('-r', '--imgres', default=512, type=int, help="Image resolution")
        parser_group.add_argument('-tf', '--tfmode', default='texture',
                                  choices=['identity', 'texture', 'linear', 'gauss'])
        parser_group.add_argument('--opacityScaling', default=3, type=float,
                                  help="Opacity scaling, only used for identity TF")
        # model
        parser_group = parser.add_argument_group("Model")
        parser_group.add_argument('-m', '--meanStepsize', default=1, type=float, help="Mean step size in voxels")
        parser_group.add_argument('--ignoreEmpty', action='store_true', help="Ignore empty pixels during mean calculation")
        parser_group.add_argument('-s', '--scale', default=1, type=float, help="Downscaling/smoothing factor of the stepsize")
        parser_group.add_argument('--onlyOpacity', action="store_true", help="Only consider opacity when computing the gradient norm")
        parser_group.add_argument('--blendWhite', action="store_true",
                                  help="Blend to white before computing the norm. Excludes --onlyOpacity")
        # training
        parser_group = parser.add_argument_group("Training")
        parser_group.add_argument('-lr', default=1.2, type=float, help="The learning rate > 1")
        parser_group.add_argument('-i', '--iterations', default=50, type=int,
                                  help="The number of iterations in the training")
        parser_group.add_argument('--noCuda', action='store_true', help='Disable cuda')
        parser_group.add_argument('--seed', type=int, default=124, help='random seed to use. Default=124')

    def parse(self, parse_args):
        opt_dict: dict = vars(parse_args)
        self.opt_dict = opt_dict
        for (key, value) in opt_dict.items():
            setattr(self, key, value)
        return opt_dict

def train():
    # Settings
    parser = argparse.ArgumentParser(
        description='Stepsize optimization',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    cfg = Config()
    cfg.init_parser(parser)
    opt = parser.parse_args()
    opt_dict = cfg.parse(opt)
    torch.manual_seed(cfg.seed)
    np.random.seed(cfg.seed)
    torch.set_num_threads(4)

    if cfg.onlyOpacity and cfg.blendWhite:
        raise ValueError("cannot specifiy both --onlyOpacity and --blendWhite at the same time")

    # settings
    s = Settings(cfg.settingsFile)
    volume = s.load_dataset()
    if cfg.noCuda:
        volume_data = volume.getDataCpu(0)
    else:
        volume.copy_to_gpu()
        volume_data = volume.getDataGpu(0)
    device = volume_data.device
    rs: pyrenderer.RendererInputs = setup_default_settings(
        volume, cfg.imgres, cfg.imgres,
        s.get_stepsize(), not cfg.noCuda)
    rs.volume = volume_data

    # camera
    camera_config = s.get_camera()
    if cfg.views == 1:
        # use test view
        camera_pitch_cpu = np.array([camera_config.pitch_radians], dtype=renderer_dtype_np)
        camera_yaw_cpu = np.array([camera_config.yaw_radians], dtype=renderer_dtype_np)
    else:
        # random views
        camera_pitch_cpu, camera_yaw_cpu = fibonacci_sphere(cfg.views, dtype=renderer_dtype_np)
    camera_distance_cpu = camera_config.distance * np.ones((cfg.views,), dtype=renderer_dtype_np)
    camera_center_cpu = np.stack([camera_config.center] * cfg.views, axis=0).astype(dtype=renderer_dtype_np)
    camera_fov_radians = camera_config.fov_y_radians
    camera_module = CameraOnASphere(camera_config.orientation)
    cameras = camera_module(
        torch.from_numpy(camera_center_cpu).to(device=device),
        torch.from_numpy(camera_yaw_cpu).to(device=device).unsqueeze(1),
        torch.from_numpy(camera_pitch_cpu).to(device=device).unsqueeze(1),
        torch.from_numpy(camera_distance_cpu).to(device=device).unsqueeze(1))
    rs.camera_mode = pyrenderer.CameraMode.ReferenceFrame
    rs.camera = pyrenderer.CameraReferenceFrame(cameras, camera_fov_radians)

    # tf
    if cfg.tfmode == 'texture':
        rs.tf_mode = pyrenderer.TFMode.Texture
        _tf_points = s.get_tf_points()
        tf_reference = pyrenderer.TFUtils.get_texture_tensor(_tf_points, 256)
        tf_reference = tf_reference.to(device=device, dtype=renderer_dtype_torch)
    elif cfg.tfmode == 'linear':
        rs.tf_mode = pyrenderer.TFMode.Linear
        _tf_points = s.get_tf_points()
        tf_reference = pyrenderer.TFUtils.get_piecewise_tensor(_tf_points)
        tf_reference = tf_reference.to(device=device, dtype=renderer_dtype_torch)
    elif cfg.tfmode == "identity":
        rs.tf_mode = pyrenderer.TFMode.Identity
        tf_reference = torch.tensor([[
            # r,g,b,a,pos
            [cfg.opacityScaling, 1]
        ]], dtype=renderer_dtype_torch, device=device)
    elif cfg.tfmode == "gauss":
        rs.tf_mode = pyrenderer.TFMode.Gaussian
        tf_reference = s.get_gaussian_tensor().to(device=device, dtype=renderer_dtype_torch)
    else:
        raise ValueError("unknown tfmode: " + cfg.tfmode)
    rs.tf = tf_reference

    # stepsize
    ignore_empty = opt.ignoreEmpty
    volume_resolution = volume_data.shape[-1]
    mean_stepsize = cfg.meanStepsize / volume_resolution
    mean_cost = 1 / mean_stepsize
    initial_stepsize = mean_stepsize * torch.ones(cfg.views, cfg.imgres, cfg.imgres,
                                                  device=device, dtype=renderer_dtype_torch)
    lr = cfg.lr
    assert lr>1, "learning rate must be >1"

    # input / output structures
    fds = pyrenderer.ForwardDifferencesSettings()
    fds.D = 1
    fds.d_stepsize = 0

    output_color = torch.empty(
        cfg.views, cfg.imgres, cfg.imgres, 4, dtype=volume_data.dtype, device=volume_data.device)
    output_termination_index = torch.empty(
        cfg.views, cfg.imgres, cfg.imgres, dtype=torch.int32, device=volume_data.device)
    outputs = pyrenderer.RendererOutputs(output_color, output_termination_index)

    forward_gradients_out = torch.zeros(cfg.views, cfg.imgres, cfg.imgres, fds.D, 4,
                                        dtype=volume_data.dtype, device=volume_data.device)

    # create output
    epochs = cfg.iterations
    print("Create output file", cfg.outputFile)
    outputDir = os.path.split(cfg.outputFile)[0]
    os.makedirs(outputDir, exist_ok=True)
    with h5py.File(cfg.outputFile, 'w') as hdf5_file:
        for k, v in cfg.opt_dict.items():
            try:
                hdf5_file.attrs[k] = v
            except TypeError as ex:
                print("Exception", ex, "while saving attribute", k, "=", str(v))
        try:
            git_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip()
            hdf5_file.attrs['git'] = git_commit
            print("git commit", git_commit)
        except:
            print("unable to get git commit")
        hdf5_file.create_dataset(
            "reference_tf", data=tf_reference.cpu().numpy())
        times = hdf5_file.create_dataset("times", (epochs,), dtype=renderer_dtype_np)
        min_stepsizes = hdf5_file.create_dataset("min_stepsizes", (epochs,), dtype=renderer_dtype_np)
        max_stepsizes = hdf5_file.create_dataset("max_stepsizes", (epochs,), dtype=renderer_dtype_np)
        stepsizes = hdf5_file.create_dataset("stepsizes", (epochs + 1,cfg.views, cfg.imgres, cfg.imgres), dtype=renderer_dtype_np)
        gradient_norms = hdf5_file.create_dataset("gradient_norm", (epochs + 1, cfg.views, cfg.imgres, cfg.imgres),
                                             dtype=renderer_dtype_np)
        stepsizes[0,...] = initial_stepsize.cpu().numpy()

        print("Optimize for", epochs, "iterations")
        current_stepsize = initial_stepsize.clone()
        start_time = time.time()
        #with tqdm.tqdm(epochs + 1) as iteration_bar:
        for iteration in range(epochs):
            # render with forward gradients
            rs.step_size = current_stepsize
            pyrenderer.Renderer.render_forward_gradients(
                rs, fds, outputs, forward_gradients_out)

            # in the first iteration, compute mask where the ray hit the bounding box
            if iteration==0:
                mask = output_termination_index>0
                fill_rate = torch.mean(mask*1.0).item()
                if cfg.scale > 1:
                    mask = torch.nn.functional.interpolate(mask.unsqueeze(0)*1.0, scale_factor=1 / cfg.scale, mode='bilinear')[0] > 0.5
                print("fill rate:", fill_rate)
            # compute gradient norm
            def nan_to_num(tensor):
                return tensor # not needed anymore, NaNs are fixed
                #return torch.where(torch.logical_or(torch.isnan(tensor), torch.isinf(tensor)), torch.zeros_like(tensor), tensor)

            forward_gradients_out2 = nan_to_num(forward_gradients_out[:,:,:,0,:])
            if cfg.onlyOpacity:
                gradient_norm = torch.abs(forward_gradients_out2[...,3])
            elif cfg.blendWhite:
                forward_gradients_out3 = forward_gradients_out2[...,:3]*output_color[...,3:] + (output_color[...,:3]-1)*forward_gradients_out2[...,3:]
                gradient_norm = torch.linalg.norm(forward_gradients_out3, dim=3)
            else:
                gradient_norm = torch.linalg.norm(forward_gradients_out2,dim=3)
            if cfg.scale>1:
                _, origSizeX, origSizeY = gradient_norm.shape
                gradient_norm = torch.nn.functional.interpolate(gradient_norm.unsqueeze(0), scale_factor=1/cfg.scale, mode='bilinear')[0]
                current_stepsize = torch.nn.functional.interpolate(current_stepsize.unsqueeze(0), scale_factor=1 / cfg.scale, mode='bilinear')[0]
            # per-pixel change
            per_pixel_change = current_stepsize * gradient_norm
            max_change = torch.max(per_pixel_change).item()
            # now decrease the stepsize where this value is high
            step_pre = current_stepsize / torch.pow(lr, per_pixel_change/max_change)
            # normalize
            #current_stepsize = step_pre * mean_stepsize / torch.mean(step_pre)
            cost_pre = 1 / step_pre
            if ignore_empty:
                cost_pre_filtered = torch.where(mask, cost_pre, mean_cost*torch.ones_like(cost_pre))
                #print("mean(cost_pre):", torch.mean(cost_pre).item(), ", mean(cost_pre_filtered):", torch.mean(cost_pre_filtered).item())
                current_stepsize = 1 / (cost_pre_filtered * mean_cost / torch.mean(cost_pre_filtered))
            else:
                current_stepsize = 1 / (cost_pre * mean_cost / torch.mean(cost_pre))
            if cfg.scale>1:
                current_stepsize = torch.nn.functional.interpolate(current_stepsize.unsqueeze(0), size=(origSizeX, origSizeY), mode='bilinear')[0]
                gradient_norm = torch.nn.functional.interpolate(gradient_norm.unsqueeze(0), size=(origSizeX, origSizeY), mode='bilinear')[0]
            # save
            min_stepsize = torch.min(current_stepsize).item()
            max_stepsize = torch.max(current_stepsize).item()
            stepsizes[iteration+1, ...] = current_stepsize.cpu().numpy()
            end_time = time.time()
            gradient_norms[iteration+1, ...] = gradient_norm.cpu().numpy()
            times[iteration] = end_time - start_time
            min_stepsizes[iteration] = min_stepsize
            max_stepsizes[iteration] = max_stepsize

            print("[Epoch %03d] change: %7.5f, min: %7.5f, max: %7.5f" % (
                iteration, max_change, min_stepsize*volume_resolution, max_stepsize*volume_resolution))
            if not np.isfinite(max_change):
                print("Inf or NaN!")
                break

                #iteration_bar.update(1)
                #iteration_bar.set_description("change: %7.5f, min: %7.5f, max: %7.5f" % (
                #    max_change, min_stepsize*volume_resolution, max_stepsize*volume_resolution))
            #break

    print("Done")

if __name__ == '__main__':
    train()

max_stars_repo_name: shamanDevel/DiffDVR

max_issues_repo_name: shamanDevel/DiffDVR

max_forks_repo_name: shamanDevel/DiffDVR


text:
"""
Defines the soccer module parameters.
"""

import numpy as np

TARGETS = {
    'H': lambda score1, score2: score1 > score2,
    'A': lambda score1, score2: score2 > score1,
    'D': lambda score1, score2: score1 == score2,
    'H+D': lambda score1, score2: score1 >= score2,
    'A+D': lambda score1, score2: score2 >= score1,
    'H+A': lambda score1, score2: score1 != score2,
    'over_2.5': lambda score1, score2: score1 + score2 > 2.5,
    'under_2.5': lambda score1, score2: score1 + score2 < 2.5
}

ext: py

max_stars_repo_name: Lyianu/sports-betting

max_issues_repo_name: Lyianu/sports-betting

max_forks_repo_name: Lyianu/sports-betting

reason: import numpy


text:
import os
import numpy as np
import datetime
from mikeio import dfs0 as dfs0
from mikeio.eum import TimeStep
from datetime import timedelta

def test_simple_create():

    dfs0File = r"simple.dfs0"

    data = []

    nt = 100
    d = np.random.random([nt])
    data.append(d)

    dfs = dfs0.dfs0()

    dfs.create(filename=dfs0File, data=data)

    assert True
    os.remove(dfs0File)

def test_multiple_create():

    dfs0File = r"zeros_ones.dfs0"

    data = []

    nt = 10
    d1 = np.zeros(nt)
    data.append(d1)
    d2 = np.ones(nt)
    data.append(d2)

    names = ["Zeros", "Ones"]

    dfs = dfs0.dfs0()

    dfs.create(filename=dfs0File, data=data, names=names, title="Zeros and ones")

    assert True
    os.remove(dfs0File)

def test_create_timestep_7days():

    dfs0File = r"zeros_ones.dfs0"

    data = []

    nt = 10
    d1 = np.zeros(nt)
    data.append(d1)
    d2 = np.ones(nt)
    data.append(d2)

    names = ["Zeros", "Ones"]

    dfs = dfs0.dfs0()

    dfs.create(filename=dfs0File, data=data, names=names, title="Zeros and ones",
               timeseries_unit=TimeStep.DAY, dt=7)

    assert True

    res = dfs.read(dfs0File)

    dt = res.time[1] - res.time[0]

    assert dt == timedelta(days=7)

    os.remove(dfs0File)

def test_create_equidistant_calendar():

    dfs0file = r'random.dfs0'
    d1 = np.random.random([1000])
    d2 = np.random.random([1000])
    data = []
    data.append(d1)
    data.append(d2)
    start_time = datetime.datetime(2017, 1, 1)
    timeseries_unit = 1402
    title = 'Hello Test'
    names = ['VarFun01', 'NotFun']
    variable_type = [100000, 100000]
    unit = [1000, 1000]
    data_value_type = [0, 1]
    dt = 5
    dfs = dfs0.dfs0()
    dfs.create(filename=dfs0file, data=data,
               start_time=start_time,
               timeseries_unit=timeseries_unit, dt=dt,
               names=names, title=title,
               variable_type=variable_type,
               unit=unit, data_value_type=data_value_type)

    os.remove(dfs0file)
    assert True

def test_create_non_equidistant_calendar():
    dfs0file = r'neq.dfs0'
    d1 = np.random.random([1000])
    d2 = np.random.random([1000])
    data = []
    data.append(d1)
    data.append(d2)
    start_time = datetime.datetime(2017, 1, 1)
    time_vector = []
    for i in range(1000):
        time_vector.append(start_time + datetime.timedelta(hours=i*0.1))
    title = 'Hello Test'
    names = ['VarFun01', 'NotFun']
    variable_type = [100000, 100000]
    unit = [1000, 1000]
    data_value_type = [0, 1]

    dfs = dfs0.dfs0()
    dfs.create(filename=dfs0file, data=data,
               datetimes=time_vector,
               names=names, title=title,
               variable_type=variable_type, unit=unit,
               data_value_type=data_value_type)

    assert True
    os.remove(dfs0file)

def test_read_dfs0_to_pandas_single_item():

    dfs0file = r'tests/testdata/random.dfs0'

    dfs = dfs0.dfs0()
    df = dfs.read_to_pandas(dfs0file, item_numbers=[1])

    assert df.shape[1] == 1

def test_read_dfs0_single_item():

    dfs0file = r'tests/testdata/random.dfs0'

    dfs = dfs0.dfs0()
    (data, t, names) = dfs.read(dfs0file, item_numbers=[1])

    assert len(data) == 1

def test_read_dfs0_single_item_named_access():

    dfs0file = r'tests/testdata/random.dfs0'

    dfs = dfs0.dfs0()
    res = dfs.read(dfs0file, item_numbers=[1])
    data = res.data

    assert len(data) == 1

def test_read_dfs0_to_pandas():

    dfs0file = r'tests/testdata/random.dfs0'

    dfs = dfs0.dfs0()
    pd = dfs.read_to_pandas(dfs0file)

    assert np.isnan(pd[pd.columns[0]][2])

def test_read_dfs0_to_matrix():
    dfs0file = r'tests/testdata/random.dfs0'

    dfs = dfs0.dfs0()
    (data, t, names) = dfs.read(filename=dfs0file)

    assert len(data) == 2

max_stars_repo_name: laurafroelich/py-dhi-dfs

max_issues_repo_name: laurafroelich/py-dhi-dfs

max_forks_repo_name: laurafroelich/py-dhi-dfs


text:
import math
from functools import partial
import numpy as np
import matplotlib.pyplot as plt
import cv2

def zeropadding(filename, srcbitdepth, dstbitdepth):
    if srcbitdepth == 10:
        image = cv2.imread(filename, 3)
        print(image.min(), image.max())
        image = image // np.power(2, srcbitdepth-dstbitdepth) * np.power(2, srcbitdepth-dstbitdepth)
        print(image.min(), image.max())
    newname = filename[:-4] + "_zp.png"
    cv2.imwrite(newname, image)

def scale_bitdepth(img, srcbitdepth, dstbitdepth):
    if dstbitdepth >= srcbitdepth:
        return img*np.power(2, dstbitdepth-srcbitdepth)
    else:
        return img // np.power(2, srcbitdepth-dstbitdepth)

def readyuv420(filename, bitdepth, height, width, startfrm, save=True, show=False):
    fp = open(filename, 'rb')

    framesize = height * width * 3 // 2
    h_h = height // 2
    h_w = width // 2

    fp.seek(0, 2) # locate the end
    ps = fp.tell() # return current location
    numfrm = ps // framesize

    bytesPerPixel = math.ceil(bitdepth / 8)
    seekPixels = startfrm * framesize
    fp.seek(bytesPerPixel * seekPixels)

    bytes2num = partial(int.from_bytes, byteorder='little', signed=False)

    if bitdepth == 8:
        Y = np.zeros((numfrm, height, width), np.uint8)
        U = np.zeros((numfrm, h_h, h_w), np.uint8)
        V = np.zeros((numfrm, h_h, h_w), np.uint8)
    elif bitdepth == 10:
        Y = np.zeros((numfrm, height, width), np.uint16)
        U = np.zeros((numfrm, h_h, h_w), np.uint16)
        V = np.zeros((numfrm, h_h, h_w), np.uint16)
    for i in range(numfrm - startfrm):
        for m in range(height):
            for n in range(width):
                if bitdepth == 8:
                    Y[i, m, n] = np.uint8(bytes2num(fp.read(1)))
                elif bitdepth == 10:
                    Y[i, m, n] = np.uint16(bytes2num(fp.read(2)))
        for m in range(h_h):
            for n in range(h_w):
                if bitdepth == 8:
                    U[i, m, n] = np.uint8(bytes2num(fp.read(1)))
                elif bitdepth == 10:
                    U[i, m, n] = np.uint16(bytes2num(fp.read(2)))
        for m in range(h_h):
            for n in range(h_w):
                if bitdepth == 8:
                    V[i, m, n] = np.uint8(bytes2num(fp.read(1)))
                elif bitdepth == 10:
                    V[i, m, n] = np.uint16(bytes2num(fp.read(2)))
        #plt.ion()
        #plt.ioff()
        if show:
            print(i)
            plt.subplot(131)
            plt.imshow(Y[i, :, :], cmap='gray')
            plt.subplot(132)
            plt.imshow(U[i, :, :], cmap='gray')
            plt.subplot(133)
            plt.imshow(V[i, :, :], cmap='gray')
            plt.show()
            #plt.pause(1)
            #plt.pause(0.001)
        if save:
            if bitdepth == 10:
                y = scale_bitdepth(Y[i, :, :], 10, 16)  # scale is necessary for display, otherwise image is dark
                u = scale_bitdepth(U[i, :, :], 10, 16)
                cv2.imwrite("../data/yuv2rgb/%03d_Y.png" % (i+1), y)
                cv2.imwrite("../data/yuv2rgb/%03d_U.png" % (i+1), u)
                yuv444 = yuv420to444(Y[i, :, :], U[i, :, :], V[i, :, :])  # CV_U16 only supported by YUV444 to BGR
                yuv = scale_bitdepth(yuv444, 10, 16)  # scale bit-depth for display in the YUV space
                rgb_img = cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR)
                cv2.imwrite("../data/yuv2rgb/%03d.png"%(i+1), rgb_img)
                print("Extract frame %d" % (i+1))
            elif bitdepth == 8:
                pass
                #img = np.concatenate((Yt.reshape(-1), Ut.reshape(-1), Vt.reshape(-1)))
                #img = img.reshape(height * 3 // 2, width)
                #rgb_img = cv2.cvtColor(img, cv2.COLOR_YUV2RGB_I420)
    fp.close()
    print("job done!")
    return None

def yuv420to444(y420, u420, v420):
    height, width = y420.shape
    yuv444 = np.zeros(shape=(height, width, 3), dtype=y420.dtype)
    for m in range(height):
        for n in range(width):
            yuv444[m, n, 0] = y420[m, n]
    for m in range(0, height, 2):
        for n in range(0, width, 2):
            yuv444[m, n, 1] = u420[m//2, n//2]
            yuv444[m, n+1, 1] = u420[m//2, n//2]
            yuv444[m+1, n, 1] = u420[m//2, n//2]
            yuv444[m+1, n+1, 1] = u420[m//2, n//2]

            yuv444[m, n, 2] = v420[m//2, n//2]
            yuv444[m, n+1, 2] = v420[m//2, n//2]
            yuv444[m+1, n, 2] = v420[m//2, n//2]
            yuv444[m+1, n+1, 2] = v420[m//2, n//2]
    return yuv444

if __name__ == '__main__':
    readyuv420("../data/Bosphorus_3840x2160_120fps_420_10bit_YUV.yuv", 10, 2160, 3840, 0, True, True)


text:
"""
Solve and animate the Schrodinger equation

First presented at http://jakevdp.github.com/blog/2012/09/05/quantum-python/

Author: Jake Vanderplas <vanderplas@astro.washington.edu>
License: BSD

Edited by: npgh2009
"""

import matplotlib.pyplot as plt
from matplotlib import animation
import numpy as np
from schrodinger import Schrodinger
from HarmonicOscillator import HarmonicOscillator

# specify time steps and duration
dt = 0.01
N_steps = 50
t_max = 120
frames = int(t_max / float(N_steps * dt))

# specify constants
hbar = 1.0   # planck's constant
m = 0.5      # particle mass

# specify range in x coordinate
N = 20
dx = 0.1
nx = int(N/dx)*2 + 1
x = np.linspace(-N, N, nx)

# specify potential
omega = 0.1
qho = HarmonicOscillator(x = x, omega = omega, hbar = hbar, m = m)
V_x = qho.potential
alpha = m * omega / hbar
L = 1/np.sqrt(alpha) #length scale

# # specify initial velocity for coherence state
v0 = 0.3*omega*L #should not be much larger than omega*L to keep wavefunction shape coherent
p0 = m * v0
k0 = p0 / hbar

# specify initial wavefunction

psi_x0 = qho.eigenFunction(n = 0)
#psi_x0 = qho.coherenceState(v0, n = 0)
#psi_x0 = qho.squeezeState(omegasq = 0.1)

# define the Schrodinger object which performs the calculations
S = Schrodinger(x=x,
                psi_x0=psi_x0,
                V_x=V_x,
                hbar=hbar,
                m=m,
                k0=-50)

######################################################################
# Set up plot
fig = plt.figure()

# plotting limits
xlim = (-N, N)
#klim = (-5, 5)
ymin = 0
ymax = 10*hbar*omega

# choose plotting functions
PROB_AMPLITUDE = 1 #probability amplitude (absolute value)
REAL_AND_IMAG = 2 #real and imaginary part
REAL_IMAG_AND_ABS = 3

PLOTFUNCTION = REAL_IMAG_AND_ABS

if PLOTFUNCTION == PROB_AMPLITUDE:
    ax1 = fig.add_subplot(111, xlim=xlim,
                                ylim=(ymin - 0.2 * (ymax - ymin),
                                ymax + 0.2 * (ymax - ymin)))
    psi_x_line, = ax1.plot([], [], c='r', label=r'$|\psi(x)|$')
    V_x_line, = ax1.plot([], [], c='k', label=r'$V(x)$')

    ax1.legend(prop=dict(size=12))
    ax1.set_xlabel(r'$x$')
    ax1.set_ylabel(r'$|\psi(x)|$')
    V_x_line.set_data(S.x, S.V_x)

    # Functions to Animate the plot
    def init():
        psi_x_line.set_data([], [])
        V_x_line.set_data([], [])

        return (psi_x_line, V_x_line)

    def animate(i):
        S.time_step(dt, N_steps)
        psi_x_line.set_data(S.x, 2*abs(S.psi_x))
        V_x_line.set_data(S.x, S.V_x)

        return (psi_x_line, V_x_line)

    # call the animator.
    # blit=True means only re-draw the parts that have changed.
    anim = animation.FuncAnimation(fig, animate, init_func=init,
                                   frames=frames, interval=30, blit=True)
    plt.show()

elif PLOTFUNCTION == REAL_AND_IMAG:
    ax1 = fig.add_subplot(111, xlim=xlim,
                                ylim=(ymin - 0.8 * (ymax - ymin),
                                ymax + 0.2 * (ymax - ymin)))
    psi_x_real_line, = ax1.plot([], [], c='r', label=r'$Re(\psi(x))$')
    psi_x_imag_line, = ax1.plot([], [], c='b', label=r'$Im(\psi(x))$')
    V_x_line, = ax1.plot([], [], c='k', label=r'$V(x)$')

    ax1.legend(prop=dict(size=12))
    ax1.set_xlabel(r'$x$')
    ax1.set_ylabel(r'$\psi(x)$')
    V_x_line.set_data(S.x, S.V_x)

    # Functions to Animate the plot
    def init():
        psi_x_real_line.set_data([], [])
        psi_x_imag_line.set_data([], [])
        V_x_line.set_data([], [])

        return (psi_x_real_line, psi_x_imag_line, V_x_line)

    def animate(i):
        S.time_step(dt, N_steps)
        psi_x_real_line.set_data(S.x, 2*np.real(S.psi_x))
        psi_x_imag_line.set_data(S.x, 2*np.imag(S.psi_x))
        V_x_line.set_data(S.x, S.V_x)

        return (psi_x_real_line, psi_x_imag_line, V_x_line)

    # call the animator.
    # blit=True means only re-draw the parts that have changed.
    anim = animation.FuncAnimation(fig, animate, init_func=init,
                                   frames=frames, interval=30, blit=True)
    plt.show()

elif PLOTFUNCTION == REAL_IMAG_AND_ABS:
    ax1 = fig.add_subplot(111, xlim=xlim,
                                ylim=(ymin - 0.8 * (ymax - ymin),
                                ymax + 0.2 * (ymax - ymin)))
    psi_x_real_line, = ax1.plot([], [], c='r', label=r'$Re(\psi(x))$')
    psi_x_imag_line, = ax1.plot([], [], c='b', label=r'$Im(\psi(x))$')
    psi_x_line, = ax1.plot([], [], c='g', label=r'$|\psi(x)|$')
    V_x_line, = ax1.plot([], [], c='k', label=r'$V(x)$')

    ax1.legend(prop=dict(size=12))
    ax1.set_xlabel(r'$x$')
    ax1.set_ylabel(r'$\psi(x)$')
    V_x_line.set_data(S.x, S.V_x)

    # Functions to Animate the plot
    def init():
        psi_x_real_line.set_data([], [])
        psi_x_imag_line.set_data([], [])
        psi_x_line.set_data([], [])
        V_x_line.set_data([], [])

        return (psi_x_real_line, psi_x_imag_line, psi_x_line, V_x_line)

    def animate(i):
        S.time_step(dt, N_steps)
        psi_x_real_line.set_data(S.x, 2*np.real(S.psi_x))
        psi_x_imag_line.set_data(S.x, 2*np.imag(S.psi_x))
        psi_x_line.set_data(S.x, 2*abs(S.psi_x))
        V_x_line.set_data(S.x, S.V_x)

        return (psi_x_real_line, psi_x_imag_line, psi_x_line, V_x_line)

    # call the animator.
    # blit=True means only re-draw the parts that have changed.
    anim = animation.FuncAnimation(fig, animate, init_func=init,
                                   frames=frames, interval=30, blit=True)
    plt.show()

# bottom axes show the k-space data
# ymin = abs(S.psi_k).min()
# ymax = abs(S.psi_k).max()
# ax2 = fig.add_subplot(212, xlim=klim,
#                       ylim=(ymin - 0.2 * (ymax - ymin),
#                             ymax + 0.2 * (ymax - ymin)))
# psi_k_line, = ax2.plot([], [], c='r', label=r'$|\psi(k)|$')

# p0_line1 = ax2.axvline(-p0 / hbar, c='k', ls=':', label=r'$\pm p_0$')
# p0_line2 = ax2.axvline(p0 / hbar, c='k', ls=':')
# mV_line = ax2.axvline(np.sqrt(2 * V0) / hbar, c='k', ls='--',
#                       label=r'$\sqrt{2mV_0}$')
# ax2.legend(prop=dict(size=12))
# ax2.set_xlabel('$k$')
# ax2.set_ylabel(r'$|\psi(k)|$')

######################################################################
# Functions to Animate the plot
# def init():
#     psi_x_line.set_data([], [])
#     V_x_line.set_data([], [])

#     title.set_text("")
#     return (psi_x_line, V_x_line, title)

# def animate(i):
#     S.time_step(dt, N_steps)
#     psi_x_line.set_data(S.x, abs(S.psi_x))
#     V_x_line.set_data(S.x, S.V_x)

#     title.set_text("t = %.2f" % S.t)
#     return (psi_x_line, V_x_line, title)

# # call the animator.
# # blit=True means only re-draw the parts that have changed.
# anim = animation.FuncAnimation(fig, animate, init_func=init,
#                                frames=frames, interval=30, blit=True)

# # uncomment the following line to save the video in mp4 format.  This
# # requires either mencoder or ffmpeg to be installed on your system
# #anim.save('schrodinger_barrier.mp4', fps=15,
# #          extra_args=['-vcodec', 'libx264'])

# plt.show()


text:
# -*- coding: utf-8 -*-
"""
Copyright (c) 2021 Showa Denko Materials co., Ltd. All rights reserved.

This software is for non-profit use only.

THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THIS SOFTWARE OR THE USE OR OTHER DEALINGS IN THIS SOFTWARE.
"""

import numpy as np
def get_non_dominated_solutions(X):
    non_dominated = []
    X = np.array(X)
    for x in X:
        diff = np.array(X) - x
        bool_dominated1 = (diff <= 0).all(axis=1)
        bool_dominated2 = (diff < 0).any(axis=1)
        bool_dominated = np.array([b1 and b2 for b1, b2 in zip(bool_dominated1, bool_dominated2)])
        if not True in bool_dominated:
            non_dominated.append(x)
    return np.array(non_dominated)

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_licenses: MIT Unlicense


text:
#!/usr/bin/env python

from __future__ import division
import sys
from mu_sims_module import *
from numpy.random import randint

# # # random start
if __name__ == '__main__':
	iter_start = int(sys.argv[1])
	iter_end = int(sys.argv[2])
	np.random.seed(iter_start+1)

	# import mut_matrix
	mut_file = "../data/mu-matrix-7mers-scaled_0.txt"
	mut_matrix = get_mut_matrix(mut_file)

	# output files
	run_file = "../results/simulations/random_scaled_0_"+str(iter_start)+"_"+str(iter_end)
	with open(run_file+"_track_seq.txt", "w+") as track_seq, open(run_file+"_track_mut.txt", "w+") as track_mut, open(run_file+"_track_verbose.txt", "w+") as track_verbose:

		# add header
		prefix = "init_cond"+"\t"+"constr_cond"+"\t"+"run_number"
		track_seq.write(prefix+"\t"+"seq_length"+"\t"+"mutation"+"\t"+"mut_scaled"+"\t"+"mutation_tot"+"\t"+"mut_scaled_tot"+"\t"+"sequence"+"\n")
		track_mut.write(prefix+"\t"+"seq_length"+"\t"+"mutation"+"\t"+"mut_scaled"+"\t"+"mutation_tot"+"\t"+"mut_scaled_tot"+"\t"+"mut_mean"+"\t"+"esr_mean"+"\n")
		track_verbose.write(prefix+"\t"+"seq_length"+"mutation"+"\t"+"mut_scaled"+"\t"+"mutation_tot"+"\t"+"mut_scaled_tot"+"\t"+"codonBefore"+"\t" "codonAfter"+"\t"+"wtMotif"+"\t"+"mtMotif"+"\t"+"mutRate"+"\t"+"aminoBefore"+"\t"+"aminoAfter"+"\t"+"aminoCheck"+"\n")

		for i in range(iter_start, iter_end):
			init_cond = "random"
			constr_cond = "neutral"
			run_number = str(i)
			print init_cond+"_"+constr_cond+"_"+run_number
			seq_sequence = initiate_seq_sequence(999)
			muSimsConstraintGranthamSimulation(mut_matrix, track_seq, track_mut, track_verbose, init_cond, constr_cond, run_number, seq_sequence, 9990, 50, 300, False)

		for i in range(iter_start, iter_end):
			init_cond = "random"
			constr_cond = "identity"
			run_number = str(i)
			print init_cond+"_"+constr_cond+"_"+run_number
			seq_sequence = initiate_seq_sequence(999)
			muSimsConstraintGranthamSimulation(mut_matrix, track_seq, track_mut, track_verbose, init_cond, constr_cond, run_number, seq_sequence, 9990, 50, 0, False)

		# for i in range(iter_start, iter_end):
		# 	init_cond = "random"
		# 	constr_cond = "grantham"
		# 	run_number = str(i)
		# 	print init_cond+"_"+constr_cond+"_"+run_number
		# 	seq_sequence = initiate_seq_sequence(999)
		# 	muSimsConstraintGranthamSimulation(mut_matrix, track_seq, track_mut, track_verbose, init_cond, constr_cond, run_number, seq_sequence, 9990, 50, 30, False)

	# track kmer freqs
	for kmer_size in range(6, 7):
		out_seq_kmers(run_file+"_track_seq.txt", run_file+"_track_kmers"+str(kmer_size)+".txt", kmer_size, False)

	# track mut, esr, and rosenberg scores
	out_seq_mut(run_file+"_track_seq.txt", run_file+"_track_mut.txt")

max_stars_repo_name: stephenrong/mutation-paper

max_issues_repo_name: stephenrong/mutation-paper

max_forks_repo_name: stephenrong/mutation-paper

reason: from numpy


text:
from typing import List, Optional
import numpy as np
from ...text_process.tokenizer import Tokenizer, get_default_tokenizer
from ...utils import check_language, get_language, language_by_name
from ...attack_assist.substitute.word import WordSubstitute, get_default_substitute
from ..classification import ClassificationAttacker, Classifier, ClassifierGoal
from ...tags import TAG_English, Tag
from ...exceptions import WordNotInDictionaryException
from ...attack_assist.filter_words import get_default_filter_words

class FDAttacker(ClassificationAttacker):
    @property
    def TAGS(self):
        return { self.__lang_tag, Tag("get_pred", "victim"), Tag("get_grad", "victim"), Tag("get_embedding", "victim") }

    def __init__(self,
            substitute : Optional[WordSubstitute] = None,
            tokenizer : Optional[Tokenizer] = None,
            token_unk : str = "<UNK>",
            max_iter : int = 100,
            lang : Optional[str] = None,
            filter_words : List[str] = None
        ):
        """
        Crafting Adversarial Input Sequences For Recurrent Neural Networks. Nicolas Papernot, Patrick McDaniel, Ananthram Swami, Richard Harang. MILCOM 2016.
        `[pdf] <https://arxiv.org/pdf/1604.08275.pdf>`__

        Args:
            substitute: A substitute that will be used during the attack procedure. Must be an instance of :py:class:`.WordSubstitute`
            tokenizer: A tokenizer that will be used during the attack procedure. Must be an instance of :py:class:`.Tokenizer`
            token_unk: The token id or the token name for out-of-vocabulary words in victim model. **Default:** ``"<UNK>"``
            max_iter: Maximum number of iterations in attack procedure.
            lang: The language used in attacker. If is `None` then `attacker` will intelligently select the language based on other parameters.
            filter_words: A list of words that will be preserved in the attack procesudre.

        :Classifier Capacity:
            * get_pred
            * get_grad
            * get_embedding

        """

        if substitute is not None and tokenizer is not None:
            self.__lang_tag = get_language([substitute, tokenizer])
        if substitute is not None:
            self.__lang_tag = get_language([substitute])
        elif tokenizer is not None:
            self.__lang_tag = get_language([tokenizer])
        else:
            if lang is None:
                self.__lang_tag = TAG_English
            else:
                self.__lang_tag = language_by_name(lang)
                if self.__lang_tag is None:
                    raise ValueError("Unknown language `%s`" % lang)

        if substitute is None:
            substitute = get_default_substitute(self.__lang_tag)

        self.substitute = substitute

        if tokenizer is None:
            tokenizer = get_default_tokenizer(self.__lang_tag)
        self.tokenizer = tokenizer

        if filter_words is None:
            filter_words = get_default_filter_words(self.__lang_tag)
        self.filter_words = set(filter_words)

        check_language([self.tokenizer, self.substitute], self.__lang_tag)

        self.token_unk = token_unk
        self.max_iter = max_iter

    def attack(self, victim: Classifier, x_orig, goal: ClassifierGoal):
        x_orig = x_orig.lower()

        sent = self.tokenizer.tokenize(x_orig, pos_tagging=False)

        victim_embedding = victim.get_embedding()

        for i in range(self.max_iter):
            curr_sent = self.tokenizer.detokenize(sent)
            pred = victim.get_pred([ curr_sent ])[0]
            if goal.check(curr_sent, pred):
                return curr_sent

            iter_cnt = 0
            while True:
                idx = np.random.choice(len(sent))
                iter_cnt += 1
                if iter_cnt > 5 * len(sent):    # Failed to find a substitute word
                    return None
                if sent[idx] in self.filter_words:
                    continue
                try:
                    reps = list(map(lambda x:x[0], self.substitute(sent[idx], None)))
                except WordNotInDictionaryException:
                    continue
                reps = list(filter(lambda x: x in victim_embedding.word2id, reps))
                if len(reps) > 0:
                    break

            prob, grad = victim.get_grad([sent], [goal.target])
            grad = grad[0]
            prob = prob[0]
            if grad.shape[0] != len(sent) or grad.shape[1] != victim_embedding.embedding.shape[1]:
                raise RuntimeError("Sent %d != Gradient %d" % (len(sent), grad.shape[0]))
            s1 = np.sign(grad[idx])

            mn = None
            mnwd = None

            for word in reps:
                s0 = np.sign(victim_embedding.transform(word, self.token_unk) - victim_embedding.transform(sent[idx], self.token_unk))
                v = np.abs(s0 - s1).sum()
                if goal.targeted:
                    v = -v

                if (mn is None) or v < mn:
                    mn = v
                    mnwd = word

            if mnwd is None:
                return None
            sent[idx] = mnwd
        return None

max_stars_repo_name: e-tornike/OpenAttack

max_issues_repo_name: e-tornike/OpenAttack

max_forks_repo_name: e-tornike/OpenAttack


text:
#!/usr/bin/python2.7
# -*- coding:utf-8 -*-

# Author: NetworkRanger
# Date: 2018/11/4 下午2:00

# 3.3 用TensorFlow 实现矩阵分解

# 1. 导入编程库，初始化计算图，生成数据集
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
ops.reset_default_graph()
sess = tf.Session()
x_vals = np.linspace(0, 10, 100)
y_vals = x_vals + np.random.normal(0, 1, 100)
x_vals_column = np.transpose(np.matrix(x_vals))
ones_column = np.transpose(np.matrix(np.repeat(1, 100)))
A = np.column_stack((x_vals_column, ones_column))
b = np.transpose(np.matrix(y_vals))
A_tensor = tf.constant(A)
b_tensor = tf.constant(b)

# 2. 找到方阵的Cholesky矩阵分解，A.T*A
"""
注意：TensorFlow 的cholesky()函数仅仅返回矩阵分解的下三角矩阵，因为上三角矩阵是下三角矩阵的转置矩阵。
"""
tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor)
L = tf.cholesky(tA_A)
tA_b = tf.matmul(tf.transpose(A_tensor), b)
sol1 = tf.matrix_solve(L, tA_b)
sol2 = tf.matrix_solve(tf.transpose(L), sol1)

# 3. 抽取系数
solution_eval = sess.run(sol2)
slope = solution_eval[0][0]
y_intercept = solution_eval[1][0]
print('slope: ' + str(slope))
print('y_intercept: '+ str(y_intercept))

"""
slope: 0.9593805254436375
y_intercept: 0.06843132207087535
"""

best_fit = []
for i in x_vals:
    best_fit.append(slope*i+y_intercept)

plt.plot(x_vals, y_vals, 'o', label='Data')
plt.plot(x_vals, best_fit, 'r-', label='Best fit lien', linewidth=3)
plt.legend(loc='upper left')
plt.show()

max_stars_repo_name: OsbornHu/tensorflow-ml

max_issues_repo_name: OsbornHu/tensorflow-ml

max_forks_repo_name: OsbornHu/tensorflow-ml


text:
# -*- coding: utf-8 -*-
import numpy as np
import cv2

from matchernet import RewardSystem

class Reward(object):
    def __init__(self, pos, is_good, radius=0.1):
        self.pos = pos
        self.is_good = is_good
        self.radius = radius

    def contains(self, agent_pos):
        d = agent_pos - self.pos
        return np.sum(d**2) < self.radius**2

    def render(self, image):
        image_width = image.shape[1]
        render_scale = image_width / 2.0
        render_offset = image_width / 2.0

        rx = int(self.pos[0] * render_scale + render_offset)
        ry = int(self.pos[1] * render_scale + render_offset)
        if self.is_good:
            reward_color = (1,0,0)
        else:
            reward_color = (0,0,0)
        image = cv2.circle(image,
                           (rx, ry), int(self.radius * render_scale),
                           reward_color, -1)
        return image

    def evaluate(self, x, dt):
        pos = x[:2]
        if self.contains(pos):
            if self.is_good:
                return 1.0 * dt
            else:
                return -1.0 * dt
        else:
            return 0.0

class ObstacleRewardSystem(RewardSystem):
    """
    Reward definition for rectangle region environment with
    obstacle and reward objects.
    """
    def __init__(self):
        super(ObstacleRewardSystem, self).__init__()
        self.rewards = []

    def reset(self):
        self.locate_rewards()

    def locate_rewards(self):
        self.rewards = []

        # Locate rewards to random locations
        x0 = np.random.uniform(-0.8, 0.8, 1)
        y0 = np.random.uniform(-0.8, 0.8, 1)
        x1 = np.random.uniform(-0.8, 0.8, 1)
        y1 = np.random.uniform(-0.8, 0.8, 1)

        good_reward = Reward(pos=np.array((x0, y0), dtype=np.float32),
                             is_good=True)
        bad_reward = Reward(pos=np.array((x1, y1), dtype=np.float32),
                            is_good=False)
        self.rewards.append(good_reward)
        self.rewards.append(bad_reward)

    def render(self, image):
        """ Render rewards """
        for reward in self.rewards:
            reward.render(image)
        return image

    def evaluate(self, x, dt):
        """ Evaluate reward based on current state """
        evaluated_reward = 0.0
        for reward in self.rewards:
            evaluated_reward += reward.evaluate(x, dt)
        return evaluated_reward

max_stars_repo_name: shigeyukioba/matchernet

max_issues_repo_name: shigeyukioba/matchernet

max_forks_repo_name: shigeyukioba/matchernet


text:
"""

Tests module util.bulk .

Currently tests only bulk module..

# Author: Vladan Lucic
# $Id$
"""
from __future__ import unicode_literals

__version__ = "$Revision$"

import os
import sys
import unittest

import numpy
import numpy.testing as np_test

import pyto.util.bulk

class TestBulk(np_test.TestCase):

    def setUp(self):
        """
        """

        # path of this module
        self.this_dir, self.this_base = os.path.split(os.path.abspath(__file__))

    def testRunPath(self):
        """
        Tests run_path
        """

        # tests calling run_path one by one
        for dir_ in ['dir_1', 'dir_2', 'dir_1']:

            # check that module is loaded and executed
            module_executed = False
            try:
                path = os.path.join(self.this_dir, dir_, 'script.py')
                #before_module_index = pyto.util.bulk._module_index
                pyto.util.bulk.run_path([path])
            except pyto.util.test.test_bulk._TestBulkError:
                module_executed = True
            np_test.assert_equal(module_executed, True)

            # check module directly
            #module_index = pyto.util.bulk._module_index
            #if module_index - before_module_index == 1:
            #    mod = sys.modules['script_' + str(module_index)]
            #else:
            #    print "something's funny"

    def tearDown(self):
        """
        """

        # nothing for now
        pass

class _TestBulkError(Exception):
    """
    Exception class used to pass info from executed module
    """
    pass

if __name__ == '__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(TestBulk)
    unittest.TextTestRunner(verbosity=2).run(suite)


text:
import os.path as osp
import numpy as np
import torch
import torch.nn as nn
import torch.utils.data
import torchvision.transforms as transforms
import torchvision.datasets as dset
from pdb import set_trace as bp
from datasets import CUTOUT, Dataset2Class, ImageNet16, ImageNet, MetaImageNet, TieredImageNet, MetaTieredImageNet
from operator import mul
from functools import reduce
from PIL import Image

class RandChannel(object):
    # randomly pick channels from input
    def __init__(self, num_channel):
        self.num_channel = num_channel

    def __repr__(self):
        return ('{name}(num_channel={num_channel})'.format(name=self.__class__.__name__, **self.__dict__))

    def __call__(self, img):
        channel = img.size(0)
        channel_choice = sorted(np.random.choice(list(range(channel)), size=self.num_channel, replace=False))
        return torch.index_select(img, 0, torch.Tensor(channel_choice).long())

def get_datasets(args, input_size, cutout=-1):

    name = args.dataset
    root = args.data_path

    assert len(input_size) in [3, 4]
    if len(input_size) == 4:
        input_size = input_size[1:]
    assert input_size[1] == input_size[2]

    if name == 'cifar10':
        mean = [x / 255 for x in [125.3, 123.0, 113.9]]
        std  = [x / 255 for x in [63.0, 62.1, 66.7]]
    elif name == 'cifar100':
        mean = [x / 255 for x in [129.3, 124.1, 112.4]]
        std  = [x / 255 for x in [68.2, 65.4, 70.4]]
    elif name.startswith('imagenet-1k'):
        mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
    elif name.startswith('ImageNet16'):
        mean = [x / 255 for x in [122.68, 116.66, 104.01]]
        std  = [x / 255 for x in [63.22,  61.26 , 65.09]]
    elif name in ['MiniImageNet', 'MetaMiniImageNet', 'TieredImageNet', 'MetaTieredImageNet']:
        pass
    else:
        raise TypeError("Unknow dataset : {:}".format(name))

    # Data Argumentation
    if name == 'cifar10' or name == 'cifar100':
        lists = [transforms.RandomCrop(input_size[1], padding=0), transforms.ToTensor(), transforms.Normalize(mean, std), RandChannel(input_size[0])]
        if cutout > 0 : lists += [CUTOUT(cutout)]
        train_transform = transforms.Compose(lists)
        test_transform  = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
    elif name.startswith('ImageNet16'):
        lists = [transforms.RandomCrop(input_size[1], padding=0), transforms.ToTensor(), transforms.Normalize(mean, std), RandChannel(input_size[0])]
        if cutout > 0 : lists += [CUTOUT(cutout)]
        train_transform = transforms.Compose(lists)
        test_transform  = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
    elif name.startswith('imagenet-1k'):
        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        if name == 'imagenet-1k':
            xlists    = []
            xlists.append(transforms.Resize((32, 32), interpolation=2))
            xlists.append(transforms.RandomCrop(input_size[1], padding=0))
        elif name == 'imagenet-1k-s':
            xlists = [transforms.RandomResizedCrop(32, scale=(0.2, 1.0))]
            xlists = []
        else: raise ValueError('invalid name : {:}'.format(name))
        xlists.append(transforms.ToTensor())
        xlists.append(normalize)
        xlists.append(RandChannel(input_size[0]))
        train_transform = transforms.Compose(xlists)
        test_transform = transforms.Compose([transforms.Resize(40), transforms.CenterCrop(32), transforms.ToTensor(), normalize])
    elif name in ['MiniImageNet', 'MetaMiniImageNet', 'TieredImageNet', 'MetaTieredImageNet']:
        mean = [120.39586422 / 255.0, 115.59361427 / 255.0, 104.54012653 / 255.0]
        std = [70.68188272 / 255.0, 68.27635443 / 255.0, 72.54505529 / 255.0]
        normalize = transforms.Normalize(mean=mean, std=std)
        xlists    = []
        xlists.append(lambda x: Image.fromarray(x))
        xlists.append(transforms.Resize((32, 32), interpolation=2))
        xlists.append(transforms.RandomCrop(input_size[1], padding=0))
        xlists.append(lambda x: np.asarray(x))
        xlists.append(transforms.ToTensor())
        xlists.append(normalize)
        xlists.append(RandChannel(input_size[0]))
        train_transform = transforms.Compose(xlists)
        test_transform = transforms.Compose([lambda x: Image.fromarray(x),
            transforms.Resize(40), transforms.CenterCrop(32), transforms.ToTensor(), normalize])
    else:
        raise TypeError("Unknow dataset : {:}".format(name))

    if name == 'cifar10':
        train_data = dset.CIFAR10 (root, train=True , transform=train_transform, download=True)
        test_data  = dset.CIFAR10 (root, train=False, transform=test_transform , download=True)
        assert len(train_data) == 50000 and len(test_data) == 10000
    elif name == 'cifar100':
        train_data = dset.CIFAR100(root, train=True , transform=train_transform, download=True)
        test_data  = dset.CIFAR100(root, train=False, transform=test_transform , download=True)
        assert len(train_data) == 50000 and len(test_data) == 10000
    elif name.startswith('imagenet-1k'):
        train_data = dset.ImageFolder(osp.join(root, 'train'), train_transform)
        test_data  = dset.ImageFolder(osp.join(root, 'val'),   test_transform)
    elif name == 'ImageNet16':
        train_data = ImageNet16(root, True , train_transform)
        test_data  = ImageNet16(root, False, test_transform)
        assert len(train_data) == 1281167 and len(test_data) == 50000
    elif name == 'ImageNet16-120':
        train_data = ImageNet16(root, True , train_transform, 120)
        test_data  = ImageNet16(root, False, test_transform , 120)
        assert len(train_data) == 151700 and len(test_data) == 6000
    elif name == 'ImageNet16-150':
        train_data = ImageNet16(root, True , train_transform, 150)
        test_data  = ImageNet16(root, False, test_transform , 150)
        assert len(train_data) == 190272 and len(test_data) == 7500
    elif name == 'ImageNet16-200':
        train_data = ImageNet16(root, True , train_transform, 200)
        test_data  = ImageNet16(root, False, test_transform , 200)
        assert len(train_data) == 254775 and len(test_data) == 10000
    elif name == 'MiniImageNet':
        train_data = ImageNet(args=args, partition='train', transform=train_transform)
        test_data  = ImageNet(args=args, partition='val', transform=test_transform)
    elif name == 'MetaMiniImageNet':
        train_data = MetaImageNet(args=args, partition='test', train_transform=train_transform, test_transform=test_transform)
        test_data  = MetaImageNet(args=args, partition='val', train_transform=train_transform, test_transform=test_transform)
    elif name == 'TieredImageNet':
        train_data = TieredImageNet(args=args, partition='train', transform=train_transform)
        test_data  = TieredImageNet(args=args, partition='train_phase_val', transform=test_transform)
    elif name == 'MetaTieredImageNet':
        train_data = MetaTieredImageNet(args=args, partition='test', train_transform=train_transform, test_transform=test_transform)
        test_data  = MetaTieredImageNet(args=args, partition='val', train_transform=train_transform, test_transform=test_transform)

    else: raise TypeError("Unknow dataset : {:}".format(name))

    class_num = Dataset2Class[name]
    return train_data, test_data, class_num

class LinearRegionCount(object):
    """Computes and stores the average and current value"""
    def __init__(self, n_samples):
        self.ActPattern = {}
        self.n_LR = -1
        self.n_samples = n_samples
        self.ptr = 0
        self.activations = None

    @torch.no_grad()
    def update2D(self, activations):
        n_batch = activations.size()[0]
        n_neuron = activations.size()[1]
        self.n_neuron = n_neuron
        if self.activations is None:
            self.activations = torch.zeros(self.n_samples, n_neuron).cuda()
        self.activations[self.ptr:self.ptr+n_batch] = torch.sign(activations)  # after ReLU
        self.ptr += n_batch

    @torch.no_grad()
    def calc_LR(self):
        res = torch.matmul(self.activations.half(), (1-self.activations).T.half())
        res += res.T
        res = 1 - torch.sign(res)
        res = res.sum(1)
        res = 1. / res.float()
        self.n_LR = res.sum().item()
        del self.activations, res
        self.activations = None
        torch.cuda.empty_cache()

    @torch.no_grad()
    def update1D(self, activationList):
        code_string = ''
        for key, value in activationList.items():
            n_neuron = value.size()[0]
            for i in range(n_neuron):
                if value[i] > 0:
                    code_string += '1'
                else:
                    code_string += '0'
        if code_string not in self.ActPattern:
            self.ActPattern[code_string] = 1

    def getLinearReginCount(self):
        if self.n_LR == -1:
            self.calc_LR()
        return self.n_LR

class Linear_Region_Collector:
    def __init__(self, args, models=[], input_size=(64, 3, 32, 32), sample_batch=100, dataset='cifar100', data_path=None, seed=0):
        self.models = []
        self.input_size = input_size  # BCHW
        self.sample_batch = sample_batch
        self.input_numel = reduce(mul, self.input_size, 1)
        self.interFeature = []
        self.dataset = dataset
        self.data_path = data_path
        self.seed = seed
        self.args = args
        self.reinit(models, input_size, sample_batch, seed)

    def reinit(self,  models=None, input_size=None, sample_batch=None, seed=None):
        if models is not None:
            assert isinstance(models, list)
            del self.models
            self.models = models
            for model in self.models:
                self.register_hook(model)
            self.LRCounts = [LinearRegionCount(self.input_size[0]*self.sample_batch) for _ in range(len(models))]
        if input_size is not None or sample_batch is not None:
            if input_size is not None:
                self.input_size = input_size  # BCHW
                self.input_numel = reduce(mul, self.input_size, 1)
            if sample_batch is not None:
                self.sample_batch = sample_batch
            if self.data_path is not None:
                self.train_data, _, class_num = get_datasets(self.args, self.input_size, -1)
                self.train_loader = torch.utils.data.DataLoader(self.train_data, batch_size=self.input_size[0], num_workers=16, pin_memory=True, drop_last=True, shuffle=True)
                self.loader = iter(self.train_loader)
        if seed is not None and seed != self.seed:
            self.seed = seed
            torch.manual_seed(seed)
            torch.cuda.manual_seed(seed)
        del self.interFeature
        self.interFeature = []
        torch.cuda.empty_cache()

    def clear(self):
        self.LRCounts = [LinearRegionCount(self.input_size[0]*self.sample_batch) for _ in range(len(self.models))]
        del self.interFeature
        self.interFeature = []
        torch.cuda.empty_cache()

    def register_hook(self, model):
        for m in model.modules():
            if isinstance(m, nn.ReLU):
                m.register_forward_hook(hook=self.hook_in_forward)

    def hook_in_forward(self, module, input, output):
        if isinstance(input, tuple) and len(input[0].size()) == 4:
            self.interFeature.append(output.detach())  # for ReLU

    def forward_batch_sample(self):
        for _ in range(self.sample_batch):
            try:
                datapair = self.loader.next()
                inputs = datapair[0]
                targets = datapair[1]
            except Exception:
                del self.loader
                self.loader = iter(self.train_loader)
                datapair = self.loader.next()
                inputs = datapair[0]
                targets = datapair[1]
            for model, LRCount in zip(self.models, self.LRCounts):
                self.forward(model, LRCount, inputs)
        return [LRCount.getLinearReginCount() for LRCount in self.LRCounts]

    def forward(self, model, LRCount, input_data):
        self.interFeature = []
        with torch.no_grad():
            model.forward(input_data.cuda())
            if len(self.interFeature) == 0: return
            feature_data = torch.cat([f.view(input_data.size(0), -1) for f in self.interFeature], 1)
            LRCount.update2D(feature_data)

max_stars_repo_name: YiteWang/TENAS

max_issues_repo_name: YiteWang/TENAS

max_forks_repo_name: YiteWang/TENAS


text:
#!/usr/bin/python
# coding=utf-8
"""
fmri data simulation
"""
__author__ = "Oliver Contier"

import numpy as np
from mvpa2.datasets.mri import fmri_dataset

# TODO: problem is, BRAINIAK demands python 3.4
from brainiak.utils.fmrisim import generate_stimfunction, double_gamma_hrf

from mvpa2.misc.data_generators import autocorrelated_noise, simple_hrf_dataset
import csv
from nipype.interfaces import fsl
from nipype.interfaces.fsl.utils import ConvertXFM
from os.path import join
import os

"""

General procedure:

- get conditions
- get design parameters
    for each expected effect / condition...
    . onsets
    . durations
    . amplitudes
    . roi(s)
    . lag
- Model Specifications:
    . within- & between run effect over time
    . type of neural signal model
    . which convolution
    . noise specifications
- make neueral signal model (e.g. boxcar)
- convolve to bold model (e.g. with double gamma)
- load volume and perform motion correction
- apply signal to volume
- generate noise
- add noise to volume
- output nifti image

"""

def load_and_mc(infile, workdir):
    """
    Perform motion correction on a dataset and load the result in pymvpa
    """
    # perform motion correction using mcflirt implemented by nipype.
    fsl.FSLCommand.set_default_output_type('NIFTI_GZ')

    infilename = infile.split('/')[-1]
    mcfile = os.path.join(workdir, infilename.replace('.nii.gz', '_mc.nii.gz'))

    mcflt = fsl.MCFLIRT(in_file=infile,
                        out_file=mcfile)
    mcflt.run()

    # load the preprocessed nifti as pymvpa data set
    dataset = fmri_dataset(mcfile)
    tr = float(dataset.sa['time_coords'][1] - dataset.sa['time_coords'][0])
    nvolumes = len(dataset.samples)

    return dataset, tr, nvolumes

def mni2bold(bold, anat, standard,  mask, workdir):
    """
    Use fsl.FLIRT to transform a mask from MNI to subject space
    """

    os.makedirs(workdir)

    # bold to anat
    bold2anat = fsl.FLIRT(
        dof=6, no_clamp=True,
        in_file=bold,
        reference=anat,
        out_matrix_file=join(workdir, 'bold2anat.txt'),
        out_file=join(workdir, 'bold2anat.nii.gz'))
    bold2anat.run()

    # anat to mni
    anat2mni = fsl.FLIRT(
        dof=12,
        in_file=anat,
        reference=standard,
        out_matrix_file=join(workdir, 'anat2mni.txt'),
        out_file=join(workdir, sub, run, '%s_%s_anat2mni.nii.gz' % (sub, run)))
    anat2mni.run()

    # concatinate matrices
    concat = ConvertXFM(
        concat_xfm=True,
        in_file2=join(workdir, sub, run, '%s_%s_bold2anat.txt' % (sub, run)),
        in_file=join(workdir, sub, run, '%s_%s_anat2mni.txt' % (sub, run)),
        out_file=join(workdir, sub, run, '%s_%s_bold2mni.txt' % (sub, run)))
    concat.run()

    # inverse transmatrix
    inverse = ConvertXFM(
        in_file=join(workdir, sub, run, '%s_%s_bold2mni.txt' % (sub, run)),
        out_file=join(workdir, sub, run, '%s_%s_mni2bold.txt' % (sub, run)),
        invert_xfm=True)
    inverse.run()

    # apply to mask
    mni2bold = fsl.FLIRT(
        interp='nearestneighbour',
        apply_xfm=True,
        in_matrix_file=join(workdir, sub, run, '%s_%s_mni2bold.txt' % (sub, run)),
        in_file=join(mask),
        reference=join(data_basedir, sub, 'BOLD', run, 'bold.nii.gz'),
        out_file=join(workdir, sub, run, '%s_%s_roimask.nii.gz' % (sub, run)))
    mni2bold.run()

    mask_subjspace = join(workdir, sub, run, '%s_%s_roimask.nii.gz' % (sub, run))

    # return path of created mask
    return mask_subjspace

def get_conditions_openfmri(ds, model_id):
    """
    get names of conditions from participants_key.txt
    along with corresponding filenames for the onsets.
    """
    conditions_key_file = os.path.join(
        ds.basedir, 'models', 'model%03d' % (model_id), 'participants_key.txt')

    with open(conditions_key_file) as f:
        reader = csv.reader(f, delimiter='\t')
        condfiles = list(zip(*reader))[1]
        condnames = list(zip(*reader))[2]

    return condfiles, condnames

def get_events_info_openfmri(ds, model_id, task_id, sub, run,
                             condfiles, condnames):
    """
    for given run of given subject in an openfmri dataset, get the onsets and durations.
    get_conditions_openfmri should be run before.
    """
    events_dir = os.path.join(ds.basedir, sub, 'model', 'model%03d' % model_id,
                              'onsets', 'task%03d_run%03d' % (task_id, run), 'onset')  # /'condXX.txt'

    if set(condfiles) != set(sorted(os.listdir(events_dir))):
        raise ValueError('the list of conditions in participants_key.txt does not match'
                         'the conditions specified in the onsets directories.')
    else:

        design_spec = []

        for cf, cn in zip(condfiles, condnames):
            conddict = {}
            conddict['trial_type'] = cn
            with open(os.path.join(events_dir, '%s.txt' % cf)) as f:
                reader = csv.reader(f, delimiter='\t')
                conddict['onsets'] = list(zip(*reader))[0]
                conddict['durations'] = list(zip(*reader))[1]
            design_spec.append(conddict)

        return design_spec

def get_conditions_bids(infile):
    """
    read onsets, duration and amplitude from 3 column format file.
    (e.g. events.tsv in BIDS format)
    """

    spec = []

    if infile.endswith('.tsv'):

        # read tsv file
        with open(infile) as f:
            reader = csv.reader(f, delimiter='\t')
            rows = [row for row in reader]
            header = rows[0]
            trial_types = set([row[header.index('trial_type')] for row in rows])

        # create a dict for each trial type
        for event in trial_types:
            event_dict = {
                'trial_type': event,
                'onsets': [float(row[header.index('onset')]) for row in rows[1:]
                           if row[header.index('trial_type') == event]],
                'durations': [float(row[header.index('duration')]) for row in rows[1:]
                              if row[header.index('trial_type') == event]]
            }
            spec.append(event_dict)
    return spec

def get_signal_spec(infile, spec):
    """
    add info about amplitude, rois, changes in amplitude over time, and lag to spec
    (for each event / trial_type)
    """
    with open(infile) as f:
        reader = csv.reader(f, delimiter='\t')
        rows = [row for row in reader if not row[0].startswith('#')]
        header = rows[0]
        trial_types = set([row[header.index('trial_type')] for row in rows])

    for trial_type in trial_types:
        for event in spec:
            for row in rows:
                if event['trial_type'] == trial_type:
                    event['amplitude'] = row[header.index('amplitude')]
                    event['lag'] = row[header.index('lag')]
                    event['sample_scalar'] = row[header.index('sample_scalar')]
                    event['run_scalar'] = row[header.index('run_scalar')]
                    event['rois'] = row[header.index('rois')].split(',')
                    event['signal_type'] = row[header.index('signal_type')].split(',')
                else:
                    continue
    return spec

def univ_neural_signal(spec, tr, nvolumes):
    """
    For each event/condition in the input spec,
    generate a neural signal function (e.g. boxcar)
    and append it to the spec
    """

    for event in spec:
        # TODO: add option for lag
        if event['signal_type'] == 'boxcar':
            event['neural_signal'] = generate_stimfunction(event['onsets'],
                                                           event['durations'],
                                                           (nvolumes * tr),
                                                           weights=event['amplitudes'])

    return spec

    # TODO: More kinds of univariate signals

def neural_signal2bold(spec, tr):
    """
    For each event in spec, convolute the neural signal function with the specified
    convolutional model and append it to spec as 'bold_model'.
    """

    for event in spec:
        if event['convolution_model'] == 'double_gamma':
            event['bold_model'] = double_gamma_hrf(neural_signal, tr)

    return spec

    # TODO: more convolutional models

def make_noise(bold_dataset, tr, noisetype='autocorrelated',
               lfnl=3, hfnl=.5, seedval=1):
    """
    Simulate a functional image containing only noise using an existing
    image as a pedestal.
    """

    seed(seedval)

    if noisetype == 'autocorrelated':
        # convert to sampling rate in Hz
        sr = 1.0 / tr
        cutoff = sr / 4

        with_noise = autocorrelated_noise(bold_dataset, sr, cutoff, lfnl=lfnl, hfnl=hfnl)
        return with_noise

        # TODO: more types of noise

def add_signal(ds, ms, spec):
    """
    Basically the same as with add_signal_custom but onsets are shifted by lag.
    """

    # TODO: project mask into subject space!

    with_signal = ds.copy()

    for cond in spec:
        roivalue = cond['roivalue']
        bold_model = cond['bold_model']

        # get voxel indices for roi
        roi_indices = np.where(ms.samples[0] == roivalue)[0]

        # add bold activation model to noise
        for sample, activation in zip(with_signal.samples, bold_model):
            sample[roi_indices] += activation

        # TODO: add option for multiplicative vs. additive noise

    return with_signal

max_stars_repo_name: oliver-contier/shambrain

max_issues_repo_name: oliver-contier/shambrain

max_forks_repo_name: oliver-contier/shambrain


text:
import os, sys, time
import numpy as np
import random
import torch
import ctypes
import multiprocessing as mp
import torch
from torch.autograd import Variable
from collections import deque

def shm_as_tensor(mp_array, shape = None):
    '''
    Given a multiprocessing.Array, returns an ndarray pointing to the same data.
    '''
    if mp_array._type_ == ctypes.c_float:
        result = torch.FloatTensor(np.asarray(np.frombuffer(mp_array, dtype=np.float32)))
    elif mp_array._type_ == ctypes.c_long:
        result = torch.LongTensor(np.asarray(np.frombuffer(mp_array, dtype=np.int32)))
    else:
        print('only support float32 or int32')

    if shape is not None:
        result = result.view(*shape)

    return result

def tensor_to_shm(array, data_type='float32', lock = False):
    '''
    Generate an 1D multiprocessing.Array containing the data from the passed ndarray.
    The data will be *copied* into shared memory.
    '''
    array1d = array.view(array.numel())
    if data_type == 'float32':
        c_type = ctypes.c_float
    elif data_type == 'int32':
        c_type = ctypes.c_long
    result = mp.Array(c_type, array.numel(), lock = lock)
    shm_as_tensor(result)[:] = array1d
    return result

class SharedTensor(object):
    def __init__(self, shape, dtype='float32'):
        if dtype == 'float32':
            self.shm_array = tensor_to_shm(torch.zeros(*shape))
        elif dtype == 'int32':
            self.shm_array = tensor_to_shm(torch.LongTensor(*shape).zero_(), data_type='int32')
        else:
            print('only support float32 and int32')
            exit(0)

        if len(shape) > 1:
            self.shm_tensor = shm_as_tensor(self.shm_array, shape=shape)
        else:
            self.shm_tensor = shm_as_tensor(self.shm_array)

        self.inventory = mp.Queue()
        #self.flag = mp.Value('i', 0)

    def recv(self):
        output = self.inventory.get()
        output = output.clone()
        return output

    def send(self, tensor):
        self.shm_tensor[:] = tensor
        self.inventory.put(self.shm_tensor[:])

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

def compute_shapes(model, args):
    model.eval()
    inputs = torch.FloatTensor(1, args.nz, 1, 1)
    output = model(inputs)
    size = output.size()
    shape = [args.batch_size, ] + [x for x in size[1:]]
    return shape


text:
# -*- coding: utf-8 -*-

""" This module offers util functions to be called and used
    in other modules
"""

from datetime import datetime
import os
import json
import pickle
import string
import random
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn import tree

def id_generator(size=6, chars=string.ascii_lowercase + string.digits):
    """GENERATE A RANDOM STRING TO BE USED AS AN ID

    Args:
        size (int, optional): size of the string. Defaults to 6.
        chars (str, optional): charachters to be used to generate the string.
                            Defaults to string.ascii_lowercase+string.digits.

    Returns:
        [str]: a random chain of charachters
    """
    return "".join(random.choice(chars) for _ in range(size))

def save_model(path, model):
    """SAVE MODEL INTO PICKLE FILE

    Args:
        path (str): path where to save the model
        model (binary): the model to be saved
    """

    with open(path, "wb") as file:
        pickle.dump(model, file)

def update_history(models_hist_path, model_id, model_name, model, params):
    """SAVE METADATA RELATED TO THE TRAINED MODEL INTO THE HISTORY FILE

    Args:
        models_hist_path (str): path to the history file
        model_id (str): unique id of the model
        model_name (str): model name = "model_"+model_id+".pkl"
        model (binary): binary file of the model
        params (dict): dictionnary containing the hyper-parameters
                        used to fit the model
    """

    model_metadata = dict()
    model_metadata["trained"] = str(datetime.now())
    model_metadata["model_type"] = type(model).__name__
    model_metadata["model_id"] = model_id
    model_metadata["params"] = params
    print(model_metadata)

    with open(models_hist_path, "r+") as outfile:
        try:
            hist = json.load(outfile)
            hist[model_name] = model_metadata
            outfile.seek(0)
            json.dump(hist, outfile, indent=4)
        except json.decoder.JSONDecodeError:
            json.dump({model_name: model_metadata}, outfile, indent=4)

def update_history_add_eval(
    models_hist_path, model_id=None, model_name=None, metrics=None
):
    """ADD EVALUATION METRICS THE HISTORY FILE FOR THE SPECIFIED MODEL

    Args:
        models_hist_path (str): path to the history file
        model_id (str, optional): the id of the model. Defaults to None.
        model_name (str, optional): the name of the model. Defaults to None.
        metrics (dict, optional): a dictionnary containing metadata related
                                    to the model evaluation. Defaults to None.
    """

    assert (
        model_id is not None or model_name is not None
    ), "At least the model id or name must be given"
    assert models_hist_path is not None, "You must specify the path to the history file"

    if not model_name:
        model_name = "model_" + model_id + ".pkl"

    eval_metadata = dict()
    eval_metadata["datetime"] = str(datetime.now())
    eval_metadata["metrics"] = metrics

    with open(models_hist_path, "r+") as outfile:
        try:
            hist = json.load(outfile)
            hist[model_name]["evaluation"] = eval_metadata
            outfile.seek(0)
            json.dump(hist, outfile, indent=4)
        except json.decoder.JSONDecodeError:
            print("cannot save evaluation metadata")

def generate_features_importance_plot(model, features, model_id):
    """GENERATES A PLOT DESCRIBING FEATURES IMPORTANCE FOR THE MODEL
    TO MAKE THE PREDICTION.

    Args:
        model (tree-based model): a tree based model (decision tree, random forest ...)
        features (pandas dataframe): a table of the features on which we trained the model
        model_id (str): the unique id of the model
    """
    mean_importances = model.feature_importances_
    importances_indices = np.argsort(mean_importances)[::-1]
    ordered_columns = [features.columns[i] for i in importances_indices]
    importances = pd.DataFrame(
        [tree.feature_importances_ for tree in model.estimators_],
        columns=features.columns,
    )
    importances = importances[ordered_columns]
    _, ax = plt.subplots(figsize=(12, 8))
    sns.boxplot(x="variable", y="value", ax=ax, data=pd.melt(importances))
    figure = ax.get_figure()
    figure.savefig(
        "models/models-training/run_" + model_id + "/features_importance.png"
    )

def plot_trees(rf, feature_names, target_names, model_id):
    """GENERATES A PLOT THAT SHOWS THE DECISION MAKING OF THE TREES

    Args:
        rf (model): a tree based model (random forest ...)
        feature_names (list): names of the columns of the training set
        target_names (str): name of the target columns
        model_id (str): unique id of the model
    """
    fn = feature_names
    cn = target_names
    fig, axes = plt.subplots(nrows=1, ncols=5, figsize=(10, 2), dpi=900)
    for index in range(0, 5):
        tree.plot_tree(
            rf.estimators_[index],
            feature_names=fn,
            class_names=cn,
            filled=True,
            ax=axes[index],
        )

        axes[index].set_title("Estimator: " + str(index), fontsize=11)
    fig.savefig("models/models-training/run_" + model_id + "/Trees.png")

def get_id_list(N=6):
    print (os.getcwd())
    print([x[0] for x in os.walk("../../models/models-training")])
    return [x[0][-N:] for x in os.walk("../../models/models-training")][1:]

max_stars_repo_name: Soufiane-Fartit/cars-prices

max_issues_repo_name: Soufiane-Fartit/cars-prices

max_forks_repo_name: Soufiane-Fartit/cars-prices


text:
'''
(*)~---------------------------------------------------------------------------
Pupil - eye tracking platform
Copyright (C) 2012-2017  Pupil Labs

Distributed under the terms of the GNU
Lesser General Public License (LGPL v3.0).
See COPYING and COPYING.LESSER for license details.
---------------------------------------------------------------------------~(*)
'''

import os
import cv2
import numpy as np
from plugin import Plugin
from calibration_routines.calibrate import calibrate_2d_polynomial
from file_methods import load_object

from pyglui import ui
from pyglui.cygl.utils import RGBA
from pyglui.cygl.utils import draw_polyline_norm
from OpenGL.GL import GL_LINES,GL_LINE_LOOP

#logging
import logging
logger = logging.getLogger(__name__)

class Show_Calibration(Plugin):
    """Calibration results visualization plugin"""
    def __init__(self,g_pool):
        super().__init__(g_pool)

        self.menu=None

        logger.error("This will be implemented as part of gaze mapper soon.")
        self.alive= False
        return

        width,height = self.g_pool.capture.frame_size

        if g_pool.app == 'capture':
            cal_pt_path =  os.path.join(g_pool.user_dir,"user_calibration_data")
        else:
            cal_pt_path =  os.path.join(g_pool.rec_dir,"user_calibration_data")

        try:
            user_calibration_data = load_object(cal_pt_path)
        except:
            logger.warning("Please calibrate first")
            self.close()
            return

        if self.g_pool.binocular:

            fn_input_eye1 = cal_pt_cloud[:,2:4].transpose()
            cal_pt_cloud[:,0:2] =  np.array(map_fn(fn_input_eye0, fn_input_eye1)).transpose()
            cal_pt_cloud[:,2:4] = cal_pt_cloud[:,4:6]
        else:
            fn_input = cal_pt_cloud[:,0:2].transpose()
            cal_pt_cloud[:,0:2] =  np.array(map_fn(fn_input)).transpose()

        ref_pts = cal_pt_cloud[inlier_map][:,np.newaxis,2:4]
        ref_pts = np.array(ref_pts,dtype=np.float32)

        logger.debug("calibration ref_pts %s"%ref_pts)
        if len(ref_pts)== 0:
            logger.warning("Calibration is bad. Please re-calibrate")
            self.close()
            return

        self.calib_bounds =  cv2.convexHull(ref_pts)
        # create a list [[px1,py1],[wx1,wy1],[px2,py2],[wx2,wy2]...] of outliers and inliers for gl_lines
        self.outliers = np.concatenate((cal_pt_cloud[~inlier_map][:,0:2],cal_pt_cloud[~inlier_map][:,2:4])).reshape(-1,2)
        self.inliers = np.concatenate((cal_pt_cloud[inlier_map][:,0:2],cal_pt_cloud[inlier_map][:,2:4]),axis=1).reshape(-1,2)

        self.inlier_ratio = cal_pt_cloud[inlier_map].shape[0]/float(cal_pt_cloud.shape[0])
        self.inlier_count = cal_pt_cloud[inlier_map].shape[0]
        # hull = cv2.approxPolyDP(self.calib_bounds, 0.001,closed=True)
        full_screen_area = 1.
        logger.debug("calibration bounds %s"%self.calib_bounds)
        self.calib_area_ratio = cv2.contourArea(self.calib_bounds)/full_screen_area

    def init_gui(self):
        self.menu = ui.Scrolling_Menu('Calibration Results',pos=(300,300),size=(300,300))
        self.menu.append(ui.Button('Close', self.close))
        self.info = ui.Info_Text("Yellow: calibration error; Red: discarded outliers; Outline: calibrated area.")
        self.menu.append(self.info)
        self.menu.append(ui.Text_Input('inlier_count',self, label='Number of used samples'))
        self.menu.elements[-1].read_only=True
        self.menu.append(ui.Text_Input('inlier_ratio',self, label='Fraction of used data points'))
        self.menu.elements[-1].read_only=True
        self.menu.append(ui.Text_Input('calib_area_ratio',self, label='Fraction of calibrated screen area'))
        self.menu.elements[-1].read_only=True

        self.g_pool.gui.append(self.menu)

    def deinit_gui(self):
        if self.menu:
            self.g_pool.gui.remove(self.menu)

    def gl_display(self):
        if self.inliers is not None:
            draw_polyline_norm(self.inliers,1,RGBA(1.,.5,0.,.5),line_type=GL_LINES)
            draw_polyline_norm(self.outliers,1,RGBA(1.,0.,0.,.5),line_type=GL_LINES)
            draw_polyline_norm(self.calib_bounds[:,0],1,RGBA(.0,1.,0,.5),line_type=GL_LINE_LOOP)

    def close(self):
        self.alive = False

    def cleanup(self):
        self.deinit_gui()

if __name__ == '__main__':
    cal_pt_cloud = np.load("cal_pt_cloud.npy")
    map_fn,inlier_map = calibrate_2d_polynomial(cal_pt_cloud,(1280,720),return_inlier_map=True)
    # print cal_pt_cloud[inlier_map][:,0:2].shape
    # print cal_pt_cloud[inlier_map][0,2:4]
    inlier = np.concatenate((cal_pt_cloud[inlier_map][:,0:2],cal_pt_cloud[inlier_map][:,2:4]),axis=1)
    print(inlier)
    print(inlier.reshape(-1,2))

max_stars_repo_name: fsxfreak/esys-pbi

max_issues_repo_name: fsxfreak/esys-pbi

max_forks_repo_name: fsxfreak/esys-pbi


text:
import numpy as np
import pandas as pd
import os

def funVar():

	HOME_FOLDER = os.getcwd()

	#df_phonon = pd.read_csv(HOME_FOLDER+'/data/phonon_dielectric/phonon_dielectric_mp.csv')
	df_common = pd.read_csv(HOME_FOLDER+'/data/Common_materials.tsv', sep='\t')
	df_mat_dens = pd.read_csv(HOME_FOLDER+'/data/materials_strength_density.tsv', sep='\t')

	df = pd.merge(df_common, df_mat_dens, on="Material")
	#df = df[df['Category_x']=='Metals']

	df_use = df[['Material',
		'Category_x',
		'Young Modulus low', 'Young Modulus high',
		'Density low_y', 'Density high_y',
		'Resistivity low', 'Resistivity high',
		'Yield Strength low', 'Yield Strength high']]

	df_use['Young Modulus low'] = df_use['Young Modulus low']*1e9
	df_use['Young Modulus high'] = df_use['Young Modulus high']*1e9
	df_use['Yield Strength low'] = df_use['Yield Strength low']*1e9
	df_use['Yield Strength high'] = df_use['Yield Strength high']*1e9

	#Discretization
	Nx = df_use.shape[0]
	Ny = df_use.shape[0]

	# Poisson's ratio  					PARAMETER
	ne = np.array([0.5, 0.5, 0.5, 0.1, 0.1, 0.1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.35,
	0.4, 0.25, 0.3, 0.001, 0.33, 0.31, 0.39, 0.35, 0.39, 0.33, 0.45, 0.43, 0.36,
	0.46, 0.38, 0.1, 0.1, 0.1, 0.1, 0.15, 0.15, 0.36, 0.2, 0.25, 0.35, 0.25,
	0.22, 0.44, 0.33, 0.27, 0.3, 0.34, 0.32, 0.29, 0.27, 0.25])

	# Young module [Pa]   				PARAMETER
	E = np.array((df_use['Young Modulus high'] - df_use['Young Modulus low'])/2)

	f = 10/1000
	h = np.arange(0.1/1000,f+f/Nx,f/Nx) # plate thickness [m] 				PARAMETER
	#f = 100/100
	#a = np.arange(10/100,f+f/Nx,f/Nx) # plate x-length [m] 				PARAMETER
	#b = np.arange(10/100,f+f/Nx,f/Nx) # plate y-length [m] 				PARAMETER
	a = 300/1000
	b = 300/1000

	# A = np.array( np.meshgrid(*[ne, E], h) ) # mix the variables
	# A = np.reshape(A, (A.shape[0], -1))

	return Nx, Ny, ne[45], E[45], h, a, b
	'''return: Nx, Ny, ne, E, h, a, b'''

	#==== Material index ====#
	# 0            Flexible Foam VLD
	# 1             Flexible Foam LD
	# 2             Flexible Foam MD
	# 3                Rigid Foam LD
	# 4                Rigid Foam MD
	# 5                Rigid Foam HD
	# 6                     Isoprene
	# 7                 Butyl rubber
	# 8                          EVA
	# 9                     Neoprene
	# 10                Polyurethane
	# 11         Silicone elastomers
	# 12                     Leather
	# 13             Wood (parallel)
	# 14        Wood (perpendicular)
	# 15                      Bamboo
	# 16                        Cork
	# 17                   Polyester
	# 18                     Epoxies
	# 19                        PMMA
	# 20                          PS
	# 21                          PA
	# 22                          PC
	# 23                          PE
	# 24                         PET
	# 25                          PP
	# 26                        PTFE
	# 27                        PEEK
	# 28                       Brick
	# 29                       Stone
	# 30                    Concrete
	# 31                       Glass
	# 32                        GFRP
	# 33                        CFRP
	# 34    Aluminun/silicon carbide
	# 35               Boron carbide
	# 36             Silicon nitride
	# 37             Silicon carbide
	# 38                     Alumina
	# 39            Tungsten carbide
	# 40                 Lead alloys
	# 41                   Cu alloys
	# 42                      Steels
	# 43                   Ni alloys
	# 44                   Ti Alloys
	# 45                   Al alloys
	# 46                   Mg alloys
	# 47                   Cast iron
	# 48                   Zn alloys


text:
import pandas as pd
import numpy as np
from utils import transform_datetime_features
from profiler import Profiler

ONEHOT_MAX_UNIQUE_VALUES = 20
BIG_DATASET_SIZE = 500 * 1024 * 1024

def get_mem(df):
    mem = df.memory_usage().sum() / 1000000
    return f'{mem:.2f}Mb'

def check_column_name(name):
    if name == 'line_id':
        return False
    if name.startswith('datetime'):
        return False
    if name.startswith('string'):
        return False
    if name.startswith('id'):
        return False

    return True

def load_test_label(path):
    y = pd.read_csv(path, low_memory=False).target
    return y

def load_data(path, mode='train', sample=None, used_cols=None, dtypes=None):
    with Profiler('read dataset'):
        if mode == 'train':
            cols = ['line_id', 'target'] + used_cols if used_cols is not None else used_cols
            df = pd.read_csv(path, low_memory=False, usecols=cols, dtype=dtypes, engine='c')
            shape_orig = df.shape
            if sample is not None and sample < df.shape[0]:
                df = df.sample(n=sample)
            df.set_index('line_id', inplace=True)
            y = df.target
            df = df.drop('target', axis=1)
        else:
            cols = ['line_id'] + used_cols if used_cols is not None else used_cols
            df = pd.read_csv(path, low_memory=False, usecols=cols, dtype=dtypes, engine='c')
            shape_orig = df.shape
            df.set_index('line_id', inplace=True)
            y = None

    print(f'Dataset read, orig: {shape_orig}, sampled: {df.shape}, memory: {get_mem(df)}, mode: {mode}')

    line_id = pd.DataFrame(df.index)

    return df, y, line_id

def initial_processing(df, mode):
    if df.memory_usage().sum() > BIG_DATASET_SIZE:
        is_big = True
    else:
        is_big = False

    with Profiler(' - features from datetime'):
        df, date_cols, orig_date_cols = transform_datetime_features(df)

    cat_cols = get_cat_freqs(df)

    numeric_cols = [c for c in df.columns if c.startswith('number')]

    with Profiler(' - reindex new cols'):
        used_cols = date_cols + list(cat_cols) + numeric_cols
        df = df.reindex(columns=used_cols)

    # if is_big:
    #     with Profiler(' - convert to float32'):
    #         df[numeric_cols] = df[numeric_cols].astype(np.float32)

    print(f' - Cat: {len(cat_cols)}, num: {len(numeric_cols)}, date: {len(date_cols)}, orig_dt: {len(orig_date_cols)}')
    print(f' - Used: {len(used_cols)}, memory: {get_mem(df)}')
    params = dict(
        cat_cols=cat_cols,
        numeric_cols=numeric_cols,
        date_cols=date_cols,
        used_cols=used_cols
    )
    return df, params

def get_cat_freqs(df):
    cat_cols = [col for col in df.columns.values if col.startswith('id') or col.startswith('string')]

    return cat_cols

max_stars_repo_name: kaplanov/sbcomp

max_issues_repo_name: kaplanov/sbcomp

max_forks_repo_name: kaplanov/sbcomp


text:
import os
import ants

from glob import glob

from dlaais.dlaais.atlas import get_atlas_nii_file
from dlaais.dlaais.ants.registration_antspy import rigid_reg_4d_nifti
from dlaais.dlaais.ctp.brain_mask import generate_brain_mask

#t1w_nii_file = get_atlas_nii_file('mni_icbm152_t1_tal_nlin_sym_09c')
#ref_image_ants = ants.image_read(t1w_nii_file)

import numpy as np
import nibabel as nib

#--------------------------------------------------------------------------------------------------------
#
def generate_brain_mask(ctp_nii_file, mask_nii_file):
    """
    """
    nib_ctp = nib.load(ctp_nii_file)
    np_ctp = nib_ctp.get_fdata()
    np_max = np.max(np_ctp, axis=-1)
    np_mean = np.mean(np_ctp, axis=-1)
    np_working = np_max - np_mean
    np_mask = np.zeros(np_working.shape, dtype=np.int8)
    np_mask[np_working>10] = 1
    nib_mask = nib.Nifti1Image(np_mask, affine=nib_ctp.affine)
    nib.save(nib_mask, mask_nii_file)

"""
working_root = '/data/public/data/003_STROKE_CTP/002_NIIX_SITE005'
CTP_nii_files = glob(os.path.join(working_root, 'PID*PNAME*', '*', 'CTP_1mm.nii.gz'))
failed_cases = []
for CTP_nii_file in CTP_nii_files:
    series_root = os.path.dirname(CTP_nii_file)
    date = os.path.basename(series_root)
    pname = os.path.basename(os.path.dirname(series_root))
    print('working on %s - %s'%(pname, date))
    try:
        rigid_reg_4d_nifti(os.path.dirname(CTP_nii_file), 'CTP_1mm', ref_nii_file='First')
    except:
        failed_cases.append(pname)
"""
#ctp_nii_file = '/data/public/data/003_STROKE_CTP/002_NIIX_SITE005/PID_3052429_PNAME_ZHANG_BING_LIANG/20210813/CTP_1mm_mc.nii.gz'
#mask_nii_file = '/data/public/data/003_STROKE_CTP/002_NIIX_SITE005/PID_3052429_PNAME_ZHANG_BING_LIANG/20210813/CTP_1mm_mc_mask.nii.gz'
#generate_brain_mask(ctp_nii_file, mask_nii_file)

max_stars_repo_name: devhliu/dlaais-data

max_issues_repo_name: devhliu/dlaais-data

max_forks_repo_name: devhliu/dlaais-data


text:
#!/usr/bin/env python

from argparse import ArgumentParser
from os.path import isfile, isdir, join
from sys import exit
import numpy as np
#from pleiopred import prior_generating, coord_trimmed, pre_sumstats
import pred_main_no_comparison

# Create the master argparser and returns the argparser object
def get_argparser():
  parser = ArgumentParser(prog="PleioPred",
                          description="Genetic Risk Prediction by joint modeling of multiple diseases and functional annotation.")
  ## Input Files

  ## Parameters
  parser.add_argument('--N1', required=True, type=int,
                      help="Sample size of the first disease GWAS")
  parser.add_argument('--N2', required=True, type=int,
                      help="Sample size of the second disease GWAS")
  parser.add_argument('--rho', required=True, type=float,
                      help="Tuning parameter in (-1,1)"
                           ", the genetic correlation between diseases")
  parser.add_argument('--local_ld_prefix', required=True,
                      help="A local LD file name prefix"
                           ", will be created if not present")
  parser.add_argument('--hfile', required=True,
                      help="per-SNP heritability estimation")
  parser.add_argument('--ld_radius', type=int,
                      help="If not provided, will use the number of SNPs in"
                           " common divided by 3000")
  parser.add_argument('--coord_D1', required=True,
                      help="Output H5 File for coord_genotypes of D1")
  parser.add_argument('--coord_D2', required=True,
                      help="Output H5 File for coord_genotypes of D2")
  parser.add_argument('--out', default="PleioPred_out",
                      help="Output filename prefix for AnnoPred")
  parser.add_argument('--user_h1', type=float,
                      help="User-provided heritability estimation for D1")
  parser.add_argument('--user_h2', type=float,
                      help="User-provided heritability estimation for D2")

  return parser

def process_args(args):
  pdict = {}

  pdict['coord_D1'] = args.coord_D1
  pdict['coord_D2'] = args.coord_D2
  pdict['N1'] = args.N1
  pdict['N2'] = args.N2

  if (args.rho>-1 and args.rho<1):
    pdict['rho'] = args.rho
  else:
    exit("Tuning parameter needs to be in (-1,1)!")

  pdict['ld_radius'] = args.ld_radius

  pdict['local_ld_prefix'] = args.local_ld_prefix
  pdict['hfile'] = args.hfile
  pdict['out'] = args.out
  pdict['user_h1'] = args.user_h1
  pdict['user_h2'] = args.user_h2
  return pdict

def main(pdict):
  print(pdict)
  pred_main_no_comparison.main(pdict)

if __name__ == '__main__':
  args = get_argparser().parse_args()
  main(process_args(args))

max_stars_repo_name: yiminghu/post-GWAS-pipeline

max_issues_repo_name: yiminghu/post-GWAS-pipeline

max_forks_repo_name: yiminghu/post-GWAS-pipeline


text:
from unittest import TestCase
import numpy as np

from Implementations.FastIntegersFromGit import FastIntegersFromGit
from Implementations.FastIntegersFromJournal import FastIntegersFromJournal
from Implementations.FastIntegersPersonal import FastIntegersPersonal
from Implementations.FasterSubsetSum.RandomizedBase import NearLinearBase
from benchmarks.test_distributions import Distributions

class Test(TestCase):

    @classmethod
    def setUp(self):
        self.fastIntegers = FastIntegersFromJournal('does not matter')
        self.fastIntegersFromGit = FastIntegersFromGit()
        self.RandomizedFasterSubsetSum = NearLinearBase(False, 'near_linear', 1)

    def test_find_bug(self):
        i = 5
        delta = 0.0001
        a, t = Distributions().HighEndDistribution(i)
        tested = self.fastIntegers.run(a, t)
        tested.sort()
        expected = self.fastIntegersFromGit.run(a, t)
        self.assertListEqual(tested, expected)

max_stars_repo_name: joakiti/Benchmark-SubsetSums

max_issues_repo_name: joakiti/Benchmark-SubsetSums

max_forks_repo_name: joakiti/Benchmark-SubsetSums


text:
#-*-coding=utf-8-*-

import tushare as ts
from pandas import Series
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pylab import mpl
#mpl.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams['font.sans-serif'] = ['Microsoft YaHei']    # 指定默认字体：解决plot不能显示中文问题
mpl.rcParams['axes.unicode_minus'] = False
# 每个月的解禁股与大盘指数的关系
def ban_share(code,name):
    conn =ts.get_apis()
    year_2017 = [2629.218,3970.902,2083.032,1720.327,1999.456,1771.074,2417.082,2904.992,2910.946,2971.483,2350.122,3874.328]
    df = ts.bar(code, conn=conn, freq='M', start_date='2016-12-30', end_date='2017-11-01', asset='INDEX')
    series = df['close']
    '''
    diff_series=[]
    print(series)
    l = len(series)
    for i in range(l-1):
        print(series[i])
        d= series[i]-series[i+1]
        diff_series.append(d)
    #print(len(diff_series))
    '''

    #s2=Series(series)
    s2=series[:len(series)-1]
    s3 = s2.sort_index(ascending=True)
    #print(s3)
    s1 = Series(year_2017[0:len(s3)])
    s3=s3.reset_index(drop=True)
    #print(s3)
    #print(s1)
    cor = s3.corr(s1)
    #print(len(s3))
    #print(len(s1))
    print(cor)
    plt.figure()
    plt.subplot(2,1,1)
    s1.plot()
    plt.subplot(2,1,2)
    s3.plot(title=name)
    plt.show()

    if abs(cor) >0.5:
        print('Great factor: ',code)

def read_index():
    df = pd.read_excel('data/index_data.xls')
    df['index_data']=df['index_data'].apply(lambda x:str(x).zfill(6))
    #print(df)
    #df['index_data'].apply(lambda x:ban_share(x))
    for i in range(len(df)):
        code = df.loc[i]['index_data']
        name =df.loc[i]['name']
        ban_share(code,name)
def main():
    read_index()
    #ban_share('000001')
    print('Done')
main()

max_issues_repo_name: sulongniao/stock

max_forks_repo_name: sulongniao/stock