text:
#!/usr/bin/env python

import io
import os
import subprocess
import linecache
import numpy as np
import soundfile as sf
import scipy.signal as ss
import random
import time
import librosa
import argparse

def get_line_context(file_path, line_number):
    return linecache.getline(file_path, line_number).strip()

def sfread(fname):
    y, fs = sf.read(fname)
    if fs != 16000:
        y = librosa.resample(y, fs, 16000)
    return y

def cutwav(wav, minlen, maxlen):
    if wav.shape[0] < 16000*maxlen:
        return wav
    else:
        duration = int(random.uniform(minlen,maxlen)*16000)
        start = random.randint(0, wav.shape[0]-duration)
        wav = wav[start:start+duration]
        return wav

def mixwav(fname1, fname2, fnoisename, frir, fisotropic, ratio, snr, sir, isosnr):
    samps1= cutwav(sfread(fname1), 3, 5)
    samps2 = cutwav(sfread(fname2), 3, 5)
    noise = cutwav(sfread(fnoisename), 5, 10)
    rir = sfread(frir)
    isotropic = sfread(fisotropic)
    if len(samps1.shape) > 1:
        samps1 = samps1[:,0]
    if len(samps2.shape) > 1:
        samps2 = samps2[:,0]
    if len(noise.shape) > 1:
        noise = noise[:,0]

    ratio = float(ratio)
    snr = float(snr)
    sir = float(sir)
    isosnr = float(isosnr)

    overlaplength = int(ratio*(samps1.shape[0] + samps2.shape[0])/2)

    padlength1 = samps2.shape[0] - overlaplength
    padlength2 = samps1.shape[0] - overlaplength

    if padlength1 > 0 and padlength2 > 0:
        samps1 = np.pad(samps1,(0,padlength1),'constant', constant_values=(0,0))
        samps2 = np.pad(samps2,(padlength2,0),'constant', constant_values=(0,0))
    samps, samps1, samps2 = add_noise(samps1, samps2, rir, sir)
    samps, _, _ = add_noise(samps, noise, rir, snr)
    samps, _, _ = add_noise(samps, isotropic, rir, isosnr)
    return samps, samps1, samps2

def mix_snr(clean, noise, snr):
    t = np.random.normal(loc = 0.9, scale = 0.1)
    if t < 0:
        t = 1e-1
    elif t > 1:
        t = 1
    scale = t

    clean_snr = snr
    noise_snr = -snr

    clean_weight = 10**(clean_snr/20)
    noise_weight = 10**(noise_snr/20)
    for i in range(clean.shape[1]):
        clean[:, i]  = activelev(clean[:, i]) * clean_weight
        noise[:, i]  = activelev(noise[:, i]) * noise_weight
    noisy = clean + noise

    max_amp = np.zeros(clean.shape[1])
    for i in range(clean.shape[1]):
        max_amp[i] = np.max(np.abs([clean[:,i], noise[:,i], noisy[:,i]]))
        if max_amp[i] == 0:
            max_amp[i] = 1
        max_amp[i] = 1 / max_amp[i] * scale

    for i in range(noisy.shape[1]):
        noisy[:, i]= noisy[:, i] * max_amp[i]
        clean[:, i]= clean[:, i] * max_amp[i]
        noise[:, i]= noise[:, i] * max_amp[i]

    return noisy, clean, noise

def add_reverb(cln_wav, rir_wav):
    """
    Args:
        :@param cln_wav: the clean wav
        :@param rir_wav: the rir wav
    Return:
        :@param wav_tgt: the reverberant signal
    """
    rir_wav = np.array(rir_wav)
    wav_tgt = np.zeros([cln_wav.shape[0]+7999, rir_wav.shape[1]])
    for i in range(rir_wav.shape[1]):
        wav_tgt[:, i] = ss.oaconvolve(cln_wav, rir_wav[:,i]/np.max(np.abs(rir_wav[:,i])))
    return wav_tgt

def activelev(data):
    # max_val = np.max(np.abs(data))
    max_val = np.std(data)
    if max_val == 0:
        return data
    else:
        return data / max_val

def add_noise(clean, noise, rir, snr):
    random.seed(time.clock())
    if len(noise.shape) == 1 and len(clean.shape) > 1:
        noise = add_reverb(noise, rir[:, 16:24])
        noise = noise[:-7999]
        snr = snr / 2
        flag = 'addnoise'
    elif len(noise.shape) == 1 and len(clean.shape) == 1:
        clean = add_reverb(clean, rir[:, 0:8])
        noise = add_reverb(noise, rir[:, 8:16])
        clean = clean[:-7999]
        noise = noise[:-7999]
        flag = 'twospk'
    else:
        snr = snr / 2
        flag = 'iso'

    clean_length = clean.shape[0]
    noise_length = noise.shape[0]

    if clean_length > noise_length:
        padlength = clean_length - noise_length
        padfront = random.randint(0, padlength)
        padend =  padlength - padfront
        noise = np.pad(noise, ((padfront, padend), (0, 0)),'constant', constant_values=(0,0))
        noise_selected = noise
        clean_selected = clean

    elif clean_length < noise_length and flag == 'twospk':
        padlength = noise_length - clean_length
        padfront = random.randint(0, padlength)
        padend =  padlength - padfront
        clean = np.pad(clean, ((padfront, padend), (0, 0)),'constant', constant_values=(0,0))
        noise_selected = noise
        clean_selected = clean

    elif clean_length < noise_length and (flag == 'addnoise' or flag == 'iso'):
        start = random.randint(0, noise_length - clean_length)
        noise = noise[start:start+clean_length]
        noise_selected = noise
        clean_selected = clean

    else:
        noise_selected = noise
        clean_selected = clean

    noisy, clean, noise = mix_snr(clean_selected, noise_selected, snr)
    return noisy, clean, noise

def run(args):
    wavlist1 = args.spk1_list
    wavlist2 = args.spk2_list
    noiselist = args.noise_list
    rirlist = args.rir_list
    isolist = args.isotropic_list
    datamode = args.mode
    output_dir = args.output_dir

    utt2dur = open(output_dir+'/'+datamode+'/utt2dur', 'w')
    if not os.path.exists(output_dir):
        os.mkdir(output_dir)
    if not os.path.exists(output_dir+'/'+datamode+'/mix'):
        os.makedirs(output_dir+'/'+datamode+'/mix')
        os.makedirs(output_dir+'/'+datamode+'/spk1')
        os.makedirs(output_dir+'/'+datamode+'/spk2')

    for i in range(args.wavnum):

        random.seed(time.clock())
        wav1idx = random.randint(0, len(open(wavlist1,'r').readlines())-1)
        wav2idx = random.randint(0, len(open(wavlist2,'r').readlines())-1)
        noiseidx = random.randint(0, len(open(noiselist,'r').readlines())-1)
        riridx = random.randint(0, len(open(rirlist,'r').readlines())-1)
        isotropicidx = random.randint(0, len(open(isolist,'r').readlines())-1)
        wav1_path = get_line_context(wavlist1, wav1idx+1)
        wav2_path = get_line_context(wavlist2, wav2idx+1)
        noise_path = get_line_context(noiselist, noiseidx+1)
        rir_path = get_line_context(rirlist, riridx+1)
        isotropic_path = get_line_context(isolist, isotropicidx+1)
        random.seed(time.clock())
        snr = random.uniform(5, 20)
        sir = random.uniform(-5, 5)
        isosnr = random.uniform(15,25)
        scenario = random.randint(0, 2)
        if scenario == 0:
            ratio = random.uniform(0, 0.2)
        elif scenario == 1:
            ratio = random.uniform(0.2, 0.8)
        elif scenario == 2:
            ratio = 0.0
        outname = str(i+1).rjust(5,'0')+'.wav'
        out, spk1, spk2 = mixwav(wav1_path, wav2_path, noise_path, rir_path, isotropic_path, ratio, snr, sir, isosnr)
        sf.write(output_dir+'/'+datamode+'/mix/'+outname, out, 16000)
        sf.write(output_dir+'/'+datamode+'/spk1/'+outname, spk1[:,0], 16000)
        sf.write(output_dir+'/'+datamode+'/spk2/'+outname, spk2[:,0], 16000)
        utt2dur.write(outname.split('.wav')[0]+' '+str(out.shape[0]) + '\n')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--spk1_list",
                        type=str,
                        help="spk1_list",
                        default="rawwav_list/train/librispeech_tr_spk1.txt")
    parser.add_argument("--spk2_list",
                        type=str,
                        help="spk2_list",
                        default="rawwav_list/train/librispeech_tr_spk2.txt")
    parser.add_argument("--noise_list",
                        type=str,
                        help="noise_list",
                        default="rawwav_list/train/noise_tr.txt")
    parser.add_argument("--rir_list",
                        type=str,
                        help="rir_list",
                        default="rawwav_list/train/rir_2-8s_1-5m_aishell4_tr.txt")
    parser.add_argument("--isotropic_list",
                        type=str,
                        help="isotropic_list",
                        default="rawwav_list/train/iso_tr.txt")
    parser.add_argument("--mode",
                        type=str,
                        help="train or dev",
                        default="train")
    parser.add_argument("--output_dir",
                        type=str,
                        help="output_dir for data",
                        default="data_frontend")
    parser.add_argument("--wavnum",
                        type=int,
                        help="total number of simulated wavs",
                        default=100)
    args = parser.parse_args()
    run(args)


text:
import logging, matplotlib, os, sys
import scanpy as sc
import numpy as np
import scipy as sp
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import rcParams
from matplotlib import colors
import seaborn as sb
from rpy2.robjects.packages import importr
#from gprofiler import gprofiler
plt.rcParams['figure.figsize']=(8,8) #rescale figures
sc.settings.verbosity = 3
sc.set_figure_params(dpi=200, dpi_save=300)

# Get back:
adata = sc.read('./filtered.h5ad')

size_factors = []

with open('size_factors.csv', 'rt') as oh:
    size_factors = [float(line.rstrip()) for line in oh if line and 'size_factors' not in line]

adata.obs['size_factors'] = size_factors

sc.pl.scatter(adata, 'size_factors', 'n_counts', show=False, save='size_factors_vs_ncounts.pdf')
sc.pl.scatter(adata, 'size_factors', 'n_genes', show=False, save='size_factors_vs_ncounts.pdf')
p = sb.distplot(size_factors, bins=50, kde=False)
p.get_figure().savefig('figures/size_factors.pdf')

#Normalize adata
adata.X /= adata.obs['size_factors'].values[:,None]
adata.X = np.clip(adata.X, 0, 1e10) # get rid of <0

sc.pp.log1p(adata)
adata.X = sp.sparse.csr_matrix(adata.X)

# Store the full data set in 'raw' as log-normalised data for statistical testing
adata.raw = adata

print('Combat')
sc.pp.combat(adata, key='replicate')

# resparsify:
print('Resparsify')
adata.X = np.clip(adata.X, 0, 1e10) # get rid of <0 to help sparsification:
adata.X = sp.sparse.csr_matrix(adata.X)

adata.write('./normed.h5ad')

ext: py

reason: import numpy,import scipy


text:
import numpy as np

from library.custos.cf import CF

class NPV:

    def __init__(self, municipio):
        self.municipio = municipio

        # 2608
        self.assinaturas_gov = np.zeros(self.municipio.tempo_analise)

        # 2608
        self.assinaturas_usuarios = np.zeros(self.municipio.tempo_analise)

        # 2608
        self.income = np.zeros(self.municipio.tempo_analise)

        # 2608
        self.arpu = np.zeros(self.municipio.tempo_analise)

        # 2608
        self.si = np.zeros(self.municipio.tempo_analise)

        self.tco = dict(Macro=np.zeros(self.municipio.tempo_analise), Hetnet=np.zeros(self.municipio.tempo_analise))
        self.cf = dict(Macro=np.zeros(self.municipio.tempo_analise), Hetnet=np.zeros(self.municipio.tempo_analise))
        self.npv = dict(Macro=np.zeros(self.municipio.tempo_analise), Hetnet=np.zeros(self.municipio.tempo_analise))

        self.payback = dict(Macro=0.0, Hetnet=0.0)

    # 2608
    def get_income(self):
        for ag in self.municipio.aglomerados:
            # Calcula o quantitativo de Termianais Gov
            self.assinaturas_gov += ag.total_terminais

            # Calcula o quantitativo de Termianais de Usuários
            self.assinaturas_usuarios += ag.densidade_usuarios * ag.area_aglomerado

        # 12.0 = 12 meses
        # 0.56 = 56% da população é considerada ativa
        self.income = self.assinaturas_usuarios * CF.TAXA_SUBSCRICAO_USUARIO.valor * 12.0 * 0.56
        self.income += self.assinaturas_gov * CF.TAXA_SUBSCRICAO_GOV.valor * 12.0
        self.income *= 0.5

    # 2608
    def get_arpu(self):
        self.arpu = self.income/(self.assinaturas_usuarios + self.assinaturas_gov)

    # 2608
    def get_si(self):
        # 0.00095 Renda Media Per Capita
        temp = self.arpu/(0.00095)
        self.si = temp

    def get_cf(self):
        #for ag in self.municipio.aglomerados:
        #    # Calcula o quantitativo de Termianais de Dados
        #    self.assinaturas_gov += ag.total_terminais
        #    print()

        # arreacadacao = self.assinaturas_gov * CF.TAXA_SUBSCRICAO_GOV.valor * 12.0
        # print('Valor de Arrecadação: ')
        # print(arreacadacao)
        # print(arreacadacao.sum())

        capex = np.zeros(self.municipio.tempo_analise)
        opex = np.zeros(self.municipio.tempo_analise)

        for ag in self.municipio.aglomerados:

            # Recupera CAPEX e OPEX do Aglomerado Macro
            capex += ag.capex_macro['Radio']['infraestrutura']
            capex += ag.capex_macro['Radio']['equipamentos']
            capex += ag.capex_macro['Radio']['instalacao']
            capex += ag.capex_macro['Transporte']['infraestrutura']
            capex += ag.capex_macro['Transporte']['equipamentos']
            capex += ag.capex_macro['Transporte']['instalacao']

            opex += ag.opex_macro['Radio']['energia']
            opex += ag.opex_macro['Radio']['manutencao']
            opex += ag.opex_macro['Radio']['aluguel']
            opex += ag.opex_macro['Radio']['falhas']
            opex += ag.opex_macro['Transporte']['energia']
            opex += ag.opex_macro['Transporte']['manutencao']
            opex += ag.opex_macro['Transporte']['aluguel']
            opex += ag.opex_macro['Transporte']['falhas']

        # 2806
        capex += self.municipio.capex_co['infraestrutura']
        capex += self.municipio.capex_co['equipamentos']
        capex += self.municipio.capex_co['instalacao']

        # 2806
        opex += self.municipio.opex_co['energia']
        opex += self.municipio.opex_co['manutencao']
        opex += self.municipio.opex_co['aluguel']
        opex += self.municipio.opex_co['falhas']

        self.cf['Macro'] = self.income - (capex + opex)
        self.tco['Macro'] += (capex + opex)

        capex = np.zeros(self.municipio.tempo_analise)
        opex = np.zeros(self.municipio.tempo_analise)

        for ag in self.municipio.aglomerados:

            # Recupera CAPEX e OPEX do Aglomerado Hetnet
            capex += ag.capex_hetnet['Radio']['infraestrutura']
            capex += ag.capex_hetnet['Radio']['equipamentos']
            capex += ag.capex_hetnet['Radio']['instalacao']
            capex += ag.capex_hetnet['Transporte']['infraestrutura']
            capex += ag.capex_hetnet['Transporte']['equipamentos']
            capex += ag.capex_hetnet['Transporte']['instalacao']

            opex += ag.opex_hetnet['Radio']['energia']
            opex += ag.opex_hetnet['Radio']['manutencao']
            opex += ag.opex_hetnet['Radio']['aluguel']
            opex += ag.opex_hetnet['Radio']['falhas']
            opex += ag.opex_hetnet['Transporte']['energia']
            opex += ag.opex_hetnet['Transporte']['aluguel']
            opex += ag.opex_hetnet['Transporte']['falhas']

        # 2806
        capex += self.municipio.capex_co['infraestrutura']
        capex += self.municipio.capex_co['equipamentos']
        capex += self.municipio.capex_co['instalacao']

        # 2806
        opex += self.municipio.opex_co['energia']
        opex += self.municipio.opex_co['manutencao']
        opex += self.municipio.opex_co['aluguel']
        opex += self.municipio.opex_co['falhas']

        self.cf['Hetnet'] = self.income - (capex + opex)
        self.tco['Hetnet'] += (capex + opex)

        print('TCO:')
        print('Macro:')
        print(self.tco['Macro'])
        print(self.tco['Macro'].sum())
        print('CF:')
        print('Macro:')
        print(self.cf['Macro'])

        print('TCO:')
        print('Hetnet:')
        print(self.tco['Hetnet'])
        print(self.tco['Hetnet'].sum())
        print('CF:')
        print('Hetnet:')
        print(self.cf['Hetnet'])

    def get_npv(self):
        print('NPV')
        for tipo in self.municipio.tipos_rede_radio:
            npv = np.zeros(self.municipio.tempo_analise)
            for ano, cf_ano in enumerate(self.cf[tipo]):
                tma = (1 + CF.TAXA_DESCONTO.valor) ** ano
                npv[ano] += (cf_ano / tma)

            for ano, npv_ano in enumerate(npv):
                if npv_ano > 0:
                    self.payback[tipo] = ano
                    break
            self.npv[tipo] = npv.sum()

            print('Implantação {}:'.format(tipo))
            print(self.npv[tipo])
            print()

max_stars_repo_name: hkuribayashi/PrevisaoTrafego

max_issues_repo_name: hkuribayashi/PrevisaoTrafego

max_forks_repo_name: hkuribayashi/PrevisaoTrafego

reason: import numpy


text:
from typing import NamedTuple, Union

import numpy as np
from scipy.sparse import coo_matrix, csr_matrix

# import numpy.typing as npt

FloatDType = np.float64
IntDType = np.intp

# Requires numpy 1.21, not on conda yet...
# FloatArray = np.ndarray[FloatDType]
# IntArray = np.ndarray[IntDType]
# BoolArray = np.ndarray[np.bool_]

FloatArray = np.ndarray
IntArray = np.ndarray
BoolArray = np.ndarray
# Pygeos collections:
PointArray = np.ndarray
LineArray = np.ndarray
PolygonArray = np.ndarray
SparseMatrix = Union[coo_matrix, csr_matrix]

class Point(NamedTuple):
    x: float
    y: float

class Vector(NamedTuple):
    x: float
    y: float

# Spatial coordinate epsilon for floating point comparison
# Assuming world coordinates in meters: 40 000 m along equator:
# 40 000 000 = 4e7 mm
# np.spacing(4e7) == 7.45E-9 ~= 1E-8
X_EPSILON = 1.0e-8
X_OFFSET = 1.0e-8
T_OFFSET = 1.0e-6

class MissingOptionalModule:
    """
    Presents a clear error for optional modules.
    """

    def __init__(self, name):
        self.name = name

    def __getattr__(self, name):
        raise ImportError(f"{self.name} is required for this functionality")

max_stars_repo_name: Deltares/xugrid

max_issues_repo_name: Deltares/xugrid

max_forks_repo_name: Deltares/xugrid

reason: import numpy,from scipy


text:
import numpy as np
import numpy.linalg as la
import errno
import os

import sys
from contextlib import contextmanager

# Plotting
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import matplotlib.pylab as plt

@contextmanager
def stdout_redirected(to=os.devnull):
    '''
    import os

    with stdout_redirected(to=filename):
        print("from Python")
        os.system("echo non-Python applications are also supported")
    '''
    fd = sys.stdout.fileno()

    # assert that Python and C stdio write using the same file descriptor
    # assert libc.fileno(ctypes.c_void_p.in_dll(libc, "stdout")) == fd == 1

    def _redirect_stdout(to):
        sys.stdout.close()  # + implicit flush()
        os.dup2(to.fileno(), fd)  # fd writes to 'to' file
        sys.stdout = os.fdopen(fd, 'w')  # Python writes to fd

    with os.fdopen(os.dup(fd), 'w') as old_stdout:
        with open(to, 'w') as file:
            _redirect_stdout(to=file)
        try:
            yield  # allow code to be run with the redirected stdout
        finally:
            # restore stdout.
            # buffering and flags such as
            # CLOEXEC may be different
            _redirect_stdout(to=old_stdout)

# Function to create directories
def make_sure_path_exists(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        # Catch exception if directory created in between
        if exception.errno != errno.EEXIST:
            raise

def gen_int_log_space(min_val, limit, n):
    result = [1]
    if n > 1:  # just a check to avoid ZeroDivisionError
        ratio = (float(limit)/result[-1]) ** (1.0/(n-len(result)))
    while len(result) < n:
        next_value = result[-1]*ratio
        if next_value - result[-1] >= 1:
            # safe zone. next_value will be a different integer
            result.append(next_value)
        else:
            # problem! same integer. we need to find next_value
            # by artificially incrementing previous value
            result.append(result[-1]+1)
            # recalculate the ratio so that the remaining values will scale
            # correctly
            ratio = (float(limit)/result[-1]) ** (1.0/(n-len(result)))
    # round, re-adjust to 0 indexing (i.e. minus 1) and return np.uint64 array
    return np.array(list(map(lambda x: round(x)-1 + min_val, result)),
                    dtype=int)

def is_qp_solution_optimal(qp_problem, x, y,
                           eps_abs=1e-03, eps_rel=1e-03):
    '''
    Check optimality condition of the QP given the
    primal-dual solution (x, y) and the tolerance eps
    '''

    # Get problem matrices
    P = qp_problem['P']
    q = qp_problem['q']
    A = qp_problem['A']
    l = qp_problem['l']
    u = qp_problem['u']

    # Check primal feasibility
    Ax = A.dot(x)
    eps_pri = eps_abs + eps_rel * la.norm(Ax, np.inf)
    pri_res = np.minimum(Ax - l, 0) + np.maximum(Ax - u, 0)

    if la.norm(pri_res, np.inf) > eps_pri:
        print("Error in primal residual: %.4e > %.4e" %
              (la.norm(pri_res, np.inf), eps_pri))
        return False

    # Check dual feasibility
    Px = P.dot(x)
    Aty = A.T.dot(y)
    eps_dua = eps_abs + eps_rel * np.max([la.norm(Px, np.inf),
                                          la.norm(q, np.inf),
                                          la.norm(Aty, np.inf)])
    dua_res = Px + q + Aty

    if la.norm(dua_res, np.inf) > eps_dua:
        print("Error in dual residual: %.4e > %.4e" %
              (la.norm(dua_res, np.inf), eps_dua))
        return False

    # Check complementary slackness
    y_plus = np.maximum(y, 0)
    y_minus = np.minimum(y, 0)

    eps_comp = eps_abs + eps_rel * np.max([la.norm(Ax, np.inf)])

    comp_res_u = np.minimum(y_plus, np.abs(u - Ax))
    comp_res_l = np.minimum(-y_minus, np.abs(Ax - l))

    if la.norm(comp_res_l, np.inf) > eps_comp:
        print("Error in complementary slackness residual l: %.4e > %.4e" %
              (la.norm(comp_res_l, np.inf), eps_comp))
        return False

    if la.norm(comp_res_u, np.inf) > eps_comp:
        print("Error in complementary slackness residual u: %.4e > %.4e" %
              (la.norm(comp_res_u, np.inf), eps_comp))
        return False
    # If we arrived until here, the solution is optimal
    return True

def plot_performance_profiles(problems, solvers):
    """
    Plot performance profiles in matplotlib for specified problems and solvers
    """
    df = pd.read_csv('./results/%s/performance_profiles.csv' % problems)
    plt.figure(0)
    for solver in solvers:
        plt.plot(df["tau"], df[solver], label=solver)
    plt.xlim(1., 10000.)
    plt.ylim(0., 1.)
    plt.xlabel(r'Performance ratio $\tau$')
    plt.ylabel('Ratio of problems solved')
    plt.xscale('log')
    plt.legend()
    plt.grid()
    plt.show(block=False)
    results_file = './results/%s/%s.png' % (problems, problems)
    print("Saving plots to %s" % results_file)
    plt.savefig(results_file)


text:
import numpy as np
from auxiliary.dataset import load_ucr
from scipy.interpolate import interp1d
import matplotlib.pyplot as plt
import pathlib as path

def transform(y: np.array, n: int):
    # print(y.shape)
    n1 = np.random.randint(int(n / 3))
    n2 = np.random.randint(n - n1)
    n3 = n - n1 - n2
    i1 = [np.random.randint(2)+1 for i in range(n1)]
    i2 = [np.random.randint(4)+1 for i in range(n2)]
    i3 = [np.random.randint(16)+1 for i in range(n3)]
    new_x = np.random.dirichlet(i1 + i2 + i3).cumsum()
    new_x = new_x / new_x.max() * (y.shape[0] - 1)
    x = np.arange(y.shape[0])
    f = interp1d(x, y)
    return f(new_x)

def gen_series(x):
    n = int(np.random.randn() * x.shape[0] / 5 + x.shape[0])
    x_t = transform(x, n)
    return x_t

def equal_interp(y, n):
    x = np.arange(y.shape[0])
    new_x = np.arange(n)
    new_x = new_x / new_x.max() * (y.shape[0] - 1)
    f = interp1d(x, y)
    return f(new_x)

def processs_data(data_name):
    X_tr, y_tr, X_te, y_te = load_ucr("../../data/ucr2015", data_name)

    # X_tr_p1 = X_tr[y_tr == 1]
    # X_te_p1 = X_te[y_te == 1]
    #
    # print(X_tr_p1.shape)
    # print(X_te_p1.shape)

    print(f"{data_name}, length: {X_tr.shape[1]} train: {X_tr.shape[0]} test:{X_te.shape[0]}")

    target = np.unique(y_tr)
    try:
        ucr_d_dir = path.Path(f"../../data/ucr2015_d/{data_name}")
        ucr_d_dir.mkdir(exist_ok=True)
    except:
        pass
    with open(f"../../data/ucr2015_d/{data_name}/{data_name}_TRAIN", "w") as f:
        for t in target:
            for x in X_tr[y_tr == t]:
                x_t = gen_series(x)
                line = ','.join(map(str, [t] + list(x_t)))
                f.write(line+'\n')

    with open(f"../../data/ucr2015_d/{data_name}/{data_name}_TEST", "w") as f:
        for t in target:
            for x in X_te[y_te == t]:
                x_t = gen_series(x)
                line = ','.join(map(str, [t] + list(x_t)))
                f.write(line+'\n')

    # x = X_tr_p1[1,:]
    # plt.plot(x)
    #
    # x2 = transform(x, 400)
    # plt.plot(x2)
    #
    # x3 = equal_interp(x2, 270)
    # plt.plot(x3)
    #
    # plt.show()

if __name__ == "__main__":
    ucr_dir = path.Path("../../data/ucr2015")
    # ctx = {"settings": SETTINGS, 'recorder': init_recorder(SETTINGS)}

    for data_name in sorted(ucr_dir.iterdir()):
        # print(data_name.name)
        # if data_name.name != 'ECGFiveDays':
        #     continue
        # if data_name.name == 'Adaic':
        processs_data(data_name.name)
        # break


text:
# ******************************************************************************
# Copyright 2017-2018 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
import numpy as np
import pytest
from contextlib import closing
from ngraph.testing import ExecutorFactory
import ngraph as ng
import ngraph.transformers as ngt
from ngraph.transformers.hetr.mpilauncher import MPILauncher
import time
import os
import subprocess
import tempfile
import random

pytestmark = pytest.mark.hetr_only
STARTUP_TIME = 3

ax_A = ng.make_axis(4)
ax_B = ng.make_axis(8)
ax_C = ng.make_axis(12)
ax_D = ng.make_axis(24)

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'device_id': ('0', '1'),
    },
    {
        'device_id': ('0', '1', '2', '3'),
    },
])
def test_broadcast_scalar(hetr_device, config):
    if hetr_device == 'gpu':
        pytest.skip('gpu communication broadcast op is not supported.')
    device_id = config['device_id']
    x = ng.placeholder(())
    y = ng.placeholder(())
    with ng.metadata(device_id=device_id, parallel=ax_A):
        x_plus_y = x + y

    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation(x_plus_y, x, y)
        res = computation(1, 2)
        np.testing.assert_array_equal(res, 3)

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'axes': ng.make_axes([ax_A]),
        'device_id': ('0', '1'),
        'parallel_axis': ax_A,
    },
    {
        'axes': ng.make_axes([ax_A, ax_B]),
        'device_id': ('0', '1', '2', '3'),
        'parallel_axis': ax_A,
    },
])
def test_distributed_plus_one(hetr_device, config):
    device_id = config['device_id']
    axes = config['axes']
    parallel_axis = config['parallel_axis']

    with ng.metadata(device=hetr_device):
        x = ng.placeholder(axes=axes)
        with ng.metadata(device_id=device_id, parallel=parallel_axis):
            x_plus_one = x + 1

    np_x = np.random.randint(100, size=axes.lengths)
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation(x_plus_one, x)
        res = computation(np_x)
        np.testing.assert_array_equal(res, np_x + 1)

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'axes_x': ng.make_axes([ax_B, ax_A]),
        'axes_w': ng.make_axes([ax_A, ax_C]),
        'device_id': ('0', '1'),
        'parallel_axis': ax_B,
    },
    {
        'axes_x': ng.make_axes([ax_B, ax_A]),
        'axes_w': ng.make_axes([ax_A, ax_C]),
        'device_id': ('0', '1', '2', '3'),
        'parallel_axis': ax_B,
    },
])
def test_distributed_dot(hetr_device, config):
    if hetr_device == 'gpu':
        pytest.xfail("Intermittent failure on jenkins for mgpu")
    device_id = config['device_id']
    axes_x = config['axes_x']
    axes_w = config['axes_w']
    parallel_axis = config['parallel_axis']

    np_weight = np.ones(axes_w.lengths)
    with ng.metadata(device=hetr_device):
        x = ng.placeholder(axes=axes_x)
        with ng.metadata(device_id=device_id, parallel=parallel_axis):
            w = ng.variable(axes=axes_w, initial_value=np_weight)
            dot = ng.dot(x, w)

    np_x = np.random.randint(100, size=axes_x.lengths)
    with closing(ngt.make_transformer_factory('hetr',
                 device=hetr_device)()) as transformer:
        computation = transformer.computation(dot, x)
        res = computation(np_x)
        np.testing.assert_array_equal(res, np.dot(np_x, np_weight))

@pytest.mark.multi_device
def test_multi_computations(hetr_device):
    if hetr_device == 'gpu':
        pytest.xfail("enable after gpu exgraph")
    axes_x = ng.make_axes([ax_A, ax_B])
    x = ng.placeholder(axes=axes_x)
    y = ng.placeholder(())
    with ng.metadata(device_id=('0', '1'), parallel=ax_A):
        f = x ** 2
        out = y - ng.mean(f, out_axes=())

    np_x = np.random.randint(10, size=axes_x.lengths)
    np_y = np.random.randint(10)
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as t:
        comp = t.computation(out, x, y)
        another_comp = t.computation(f, x)

        res_comp = comp(np_x, np_y)
        res_another_comp = another_comp(np_x)
        ref_comp = np_y - np.mean(np_x**2)
        np.testing.assert_array_equal(res_comp, ref_comp)
        np.testing.assert_array_equal(res_another_comp, np_x**2)

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'axes': ng.make_axes([ax_A]),
        'device_id': ('0', '1'),
        'parallel_axis': ax_A,
    },
])
def test_repeat_computation(hetr_device, config):
    if hetr_device == 'gpu':
        pytest.xfail("enable after gpu exgraph")
    device_id = config['device_id']
    axes = config['axes']
    parallel_axis = config['parallel_axis']

    with ng.metadata(device=hetr_device):
        x = ng.placeholder(axes=axes)
        with ng.metadata(device_id=device_id, parallel=parallel_axis):
            x_plus_one = x + 1

        np_x = np.random.randint(100, size=axes.lengths)
        with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
            comp = transformer.computation(x_plus_one, x)
            comp2 = transformer.computation(x_plus_one, x)

            res = comp(np_x)
            np.testing.assert_array_equal(res, np_x + 1)

            res2 = comp2(np_x)
            np.testing.assert_array_equal(res2, np_x + 1)

@pytest.mark.multi_device
def test_comm_broadcast_op(hetr_device):
    if hetr_device == 'gpu':
        pytest.skip('gpu communication broadcast op is not supported.')
    H = ng.make_axis(length=4, name='height')
    N = ng.make_axis(length=8, name='batch')
    weight = ng.make_axis(length=2, name='weight')
    x = ng.placeholder(axes=[N, H])
    # w will be broadcasted to devices
    w = ng.placeholder(axes=[H, weight])
    with ng.metadata(device=hetr_device, device_id=('0', '1'), parallel=N):
        dot = ng.dot(x, w)

    np_x = np.random.randint(100, size=[N.length, H.length])
    np_weight = np.random.randint(100, size=[H.length, weight.length])
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation(dot, x, w)
        res = computation(np_x, np_weight)
        np.testing.assert_array_equal(res, np.dot(np_x, np_weight))

@pytest.mark.multi_device
def test_reduce_scalar(hetr_device):
    """
    A scalar is produced by sum() on each worker
    in this case, should be mean reduced before being returned
    """
    if hetr_device == 'gpu':
        pytest.xfail("gather/reduce work-around for gpus does not choose between mean or sum,\
        it uses only the value on the first device and ignores the values on other devices")

    N = ng.make_axis(length=8, name='batch')
    x = ng.placeholder(axes=[N])
    with ng.metadata(device=hetr_device, device_id=('0', '1'), parallel=N):
        out = ng.sum(x)

    np_x = np.random.randint(100, size=[N.length])
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation(out, x)
        res = computation(np_x)

        # gather returns one element per worker
        np.testing.assert_array_equal(res, np.sum(np_x) / 2.)

@pytest.mark.multi_device
def test_reduce_vector(hetr_device):
    """
    A whole vector is produced on each worker and should be reduced
    before being returned, but not along its axes since it
    does not have the parallel axis in its axes
    """
    if hetr_device == 'gpu':
        pytest.xfail("broadcast communication ops not yet supported on gpus")

    H = ng.make_axis(length=4, name='height')
    N = ng.make_axis(length=8, name='batch')
    weight = ng.make_axis(length=2, name='weight')
    x = ng.placeholder(axes=[N, H])
    w = ng.placeholder(axes=[H, weight])
    with ng.metadata(device=hetr_device, device_id=('0', '1'), parallel=N):
        dot = ng.dot(x, w)
        out = ng.sum(dot, N)

    np_x = np.random.randint(100, size=[N.length, H.length])
    np_weight = np.random.randint(100, size=[H.length, weight.length])
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation(out, x, w)
        res = computation(np_x, np_weight)
        # TODO should the reduce infer a sum or mean?
        expected = np.sum(np.dot(np_x, np_weight), 0) / 2.
        np.testing.assert_array_equal(res, expected)

@pytest.mark.multi_device
def test_distributed_dot_parallel_second_axis(hetr_device):
    if hetr_device == 'gpu':
        pytest.xfail("Axes Layout needs to be fixed for GPUs after changes to make\
        parallel_axis the least contiguous axis for scatter/gather communication ops")

    H = ng.make_axis(length=6, name='height')
    N = ng.make_axis(length=8, name='batch')
    W1 = ng.make_axis(length=2, name='W1')
    W2 = ng.make_axis(length=4, name='W2')
    x = ng.placeholder(axes=[H, N])
    w2 = ng.placeholder(axes=[W2, W1])
    with ng.metadata(device=hetr_device, device_id=('0', '1'), parallel=N):
        w1 = ng.placeholder(axes=[W1, H])
        dot1 = ng.dot(w1, x).named("dot1")
    dot2 = ng.dot(w2, dot1).named("dot2")

    np_x = np.random.randint(100, size=[H.length, N.length])
    np_w1 = np.random.randint(100, size=[W1.length, H.length])
    np_w2 = np.random.randint(100, size=[W2.length, W1.length])
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation([dot2, dot1], x, w1, w2)
        res2, res1 = computation(np_x, np_w1, np_w2)
        np.testing.assert_array_equal(res1, np.dot(np_w1, np_x))
        np.testing.assert_array_equal(res2, np.dot(np_w2, np.dot(np_w1, np_x)))

        computation2 = transformer.computation([dot1, dot2], x, w1, w2)
        res1, res2 = computation2(np_x, np_w1, np_w2)
        np.testing.assert_array_equal(res1, np.dot(np_w1, np_x))
        np.testing.assert_array_equal(res2, np.dot(np_w2, np.dot(np_w1, np_x)))

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'axes': ng.make_axes([ax_A]),
        'device_id': ('0', '1'),
        'parallel_axis': ax_A,
    },
    {
        'axes': ng.make_axes([ax_A, ax_B]),
        'device_id': ('0', '1', '2', '3'),
        'parallel_axis': ax_A,
    },
])
def test_distributed_plus_two(hetr_device, config):
    device_id = config['device_id']
    axes = config['axes']
    parallel_axis = config['parallel_axis']

    with ng.metadata(device=hetr_device):
        x = ng.placeholder(axes=axes)
        with ng.metadata(device_id=device_id, parallel=parallel_axis):
            x_plus_one = x + 1
        x_plus_two = x_plus_one + 1

    np_x = np.random.randint(100, size=axes.lengths)
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation(x_plus_two, x)
        res = computation(np_x)
        np.testing.assert_array_equal(res, np_x + 2)

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'axes': None,
    },
    {
        'axes': ng.make_axes([ax_A]),
    },
    {
        'axes': ng.make_axes([ax_A, ax_B]),
    },
])
def test_to_and_from_device(hetr_device, config):
    axes = config['axes']
    with ng.metadata(device=hetr_device):
        x = ng.placeholder(axes=axes) if axes else ng.placeholder(())
        with ng.metadata(device_id='1'):
            x_plus_one = x + 1
        x_plus_two = x_plus_one * 2

    np_x = np.random.randint(100, size=axes.lengths) if axes else random.random()
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation([x_plus_one, x_plus_two], x)
        res = computation(np_x)
        np.testing.assert_allclose(res[0], np_x + 1.0)
        np.testing.assert_allclose(res[1], (np_x + 1.0) * 2.0)

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'axes': None,
    },
    {
        'axes': ng.make_axes([ax_A]),
    },
    {
        'axes': ng.make_axes([ax_A, ax_B]),
    },
])
def test_computation_return_list(hetr_device, config):
    axes = config['axes']
    with ng.metadata(device=hetr_device):
        x = ng.placeholder(axes=axes) if axes else ng.placeholder(())

        with ng.metadata(device_id='1'):
            x_plus_one = x + 1
        with ng.metadata(device_id='2'):
            x_plus_two = x_plus_one + 1
        with ng.metadata(device_id='3'):
            x_mul_two = x_plus_two * 2

    np_x = np.random.randint(100, size=axes.lengths) if axes else random.random()
    with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as transformer:
        computation = transformer.computation([x_plus_one, x_plus_two, x_mul_two], x)
        res = computation(np_x)
        np.testing.assert_allclose(res[0], np_x + 1)
        np.testing.assert_allclose(res[1], np_x + 2)
        np.testing.assert_allclose(res[2], (np_x + 2) * 2)

@pytest.mark.hetr_gpu_only
def test_gpu_send_and_recv(hetr_device):
    pytest.xfail("GitHub issue: #2007, Unknown error - investigation is needed")
    # put x+1 on cpu numpy
    with ng.metadata(device='cpu'):
        x = ng.placeholder(())
        x_plus_one = x + 1
    # put x+2 on gpu numpy
    with ng.metadata(device='gpu'):
        x_plus_two = x_plus_one + 1

    with ExecutorFactory() as ex:
        computation = ex.executor(x_plus_two, x)
        for i in [10, 20, 30]:
            assert computation(i) == i + 2

    # put x+1 on gpu numpy
    with ng.metadata(device='gpu'):
        x = ng.placeholder(())
        x_plus_one = x + 1
    # put x+2 on cpu numpy
    with ng.metadata(device='cpu'):
        x_plus_two = x_plus_one + 1

    with ExecutorFactory() as ex:
        computation = ex.executor(x_plus_two, x)
        for i in [10, 20, 30]:
            assert computation(i) == i + 2

def test_recvop_axes_using_dot():
    x_value = np.array([[1],
                        [2]])
    w_value = np.array([[-1, 1]])

    A1 = ng.make_axis(length=1)
    A2 = ng.make_axis(length=2)
    A3 = ng.make_axis(length=2)

    x = ng.placeholder([A2, A1])
    w = ng.variable([A1, A3], initial_value=w_value)

    with ng.metadata(device_id='1'):
        result = ng.dot(x, w)

    with ExecutorFactory() as ex:
        computation = ex.executor(result, x, w)
        val_ng = computation(x_value, w_value)
        val_np = np.dot(x_value, w_value)
        ng.testing.assert_allclose(val_ng, val_np)

def test_recvop_tensorupdate():
    """
    The tensor (RecvOp_#_#) associated with the following conv op has two views:
    1) Non-flat view (e.g. RecvOp_#_#_1_1_1_1_4.shape=(1,1,1,1,4))
    2) Flat view (e.g. RecvOp_#_#_1_4.shape = (1,4))
    This test ensures that inside RecvOp code generation, the generated code
    should make sure both views get updated (e.g. by using update_RecvOp_#_# API)
    In this test, ng.dot operation tends to use the flat view (i.e. RecvOp_#_#_1_4)
    And previously RecvOp with RecvOp_#_#_1_1_1_1_4 = recv_from_send(send_id) failed
    to update both two views (i.e. flat and non-flat view of the same buffer/tensor)
    """
    class ConvParams(object):

        def __init__(self, C=1, N=1, K=1, D=1, H=1, W=1, T=1, R=1, S=1,
                     pad_d=0, pad_h=0, pad_w=0,
                     str_d=1, str_h=1, str_w=1):

            from ngraph.frontends.common.utils import conv_output_dim
            M = conv_output_dim(D, T, pad_d, str_d)
            P = conv_output_dim(H, R, pad_h, str_h)
            Q = conv_output_dim(W, S, pad_w, str_w)

            self.dimO = (K, M, P, Q, N)
            self.dimI = (C, D, H, W, N)
            self.dimF = (C, T, R, S, K)

            self.conv_params = dict(
                pad_d=pad_d, pad_h=pad_h, pad_w=pad_w,
                str_d=str_d, str_h=str_h, str_w=str_w,
                dil_d=1, dil_h=1, dil_w=1
            )

            self.batch_axis = ng.make_axis(name='N', length=N)

            self.ax_i = ng.make_axes([
                ng.make_axis(name='C', length=C),
                ng.make_axis(name='D', length=D),
                ng.make_axis(name='H', length=H),
                ng.make_axis(name='W', length=W),
                self.batch_axis
            ])

            self.ax_f = ng.make_axes([
                ng.make_axis(name='C', length=C),
                ng.make_axis(name='D', length=T),
                ng.make_axis(name='H', length=R),
                ng.make_axis(name='W', length=S),
                ng.make_axis(name='K', length=K),
            ])

            self.ax_o = ng.make_axes([
                ng.make_axis(name='C', length=K),
                ng.make_axis(name='D', length=M),
                ng.make_axis(name='H', length=P),
                ng.make_axis(name='W', length=Q),
                self.batch_axis
            ])

    # Layer 1, using convolutation introduces multi/flatten view of tensors
    cf = ConvParams(C=2, N=4, K=1, H=2, W=2, R=2, S=2)

    inputs = ng.placeholder(axes=cf.ax_i)
    filters = ng.placeholder(axes=cf.ax_f)

    # randomly initialize
    from ngraph.testing import RandomTensorGenerator
    rng = RandomTensorGenerator(0, np.float32)
    # put value 1 into inputs/filters for conv
    input_value = rng.uniform(1, 1, cf.ax_i)
    filter_value = rng.uniform(1, 1, cf.ax_f)

    conv = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o)

    # Layer 2, using dot to ensure recv_op.axes == send_op.axes
    from ngraph.frontends.neon import UniformInit
    # put value 1 into weights for dot
    init_uni = UniformInit(1, 1)
    W_A = ng.make_axis(length=2)
    w_axes = ng.make_axes(W_A) + conv.axes.feature_axes()
    w = ng.variable(axes=w_axes, initial_value=init_uni)

    with ng.metadata(device_id='1'):
        dot = ng.dot(w, conv)

    with ExecutorFactory() as ex:
        dot_comp = ex.executor(dot, filters, inputs)
        dot_val = dot_comp(filter_value, input_value)

    np.testing.assert_array_equal(dot_val, [[8., 8., 8., 8.],
                                            [8., 8., 8., 8.]])

class ClosingHetrServers():
    def __init__(self, ports):
        self.tmpfile = tempfile.NamedTemporaryFile(dir=os.path.dirname(os.path.realpath(__file__)),
                                                   delete=True)
        self.processes = []
        for p in ports:
            hetr_server = os.path.dirname(os.path.realpath(__file__)) +\
                "/../../ngraph/transformers/hetr/hetr_server.py"
            command = ["python", hetr_server, "-tf", self.tmpfile.name, "-p", p]
            try:
                proc = subprocess.Popen(command)
                self.processes.append(proc)
            except Exception as e:
                print(e)
        time.sleep(STARTUP_TIME)

    def close(self):
        for p in self.processes:
            p.terminate()
        for p in self.processes:
            p.kill()
        for p in self.processes:
            p.wait()
        self.tmpfile.close()

def test_rpc_transformer():
    pytest.xfail("Needs investigation-STARTUP_TIME is too large, needs to be over 5 seconds.")
    from ngraph.transformers.hetr.rpc_client import RPCTransformerClient
    rpc_client_list = list()
    port_list = ['50111', '50112']
    num_procs = len(port_list)

    with closing(ClosingHetrServers(port_list)):
        for p in range(num_procs):
            rpc_client_list.append(RPCTransformerClient('cpu' + str(p),
                                                        'localhost:' + port_list[p]))
            np.testing.assert_equal(rpc_client_list[p].is_trans_built, False)
            np.testing.assert_equal(rpc_client_list[p].transformer_type, 'cpu' + str(p))
            np.testing.assert_equal(rpc_client_list[p].server_address, 'localhost:' + port_list[p])
        for p in range(num_procs):
            rpc_client_list[p].build_transformer()
            np.testing.assert_equal(rpc_client_list[p].is_trans_built, True)
        for p in range(num_procs):
            rpc_client_list[p].close_transformer()
        for p in range(num_procs):
            rpc_client_list[p].close()
            np.testing.assert_equal(rpc_client_list[p].is_trans_built, False)

def test_mpilauncher():
    os.environ["HETR_SERVER_PORTS"] = "51111, 51112"
    mpilauncher = MPILauncher()
    mpilauncher.launch(2, 1)

    # Check if process has launched
    assert mpilauncher.mpirun_proc.poll() is None

    mpilauncher.close()

    # Check if process has completed
    assert mpilauncher.mpirun_proc is None

@pytest.mark.multi_device
@pytest.mark.parametrize('config', [
    {
        'dataset': 'cifar10',
        'iter_count': 1,
        'batch_size': 64,
        'device_id': ('0', '1'),
        'bprop': True,
        'batch_norm': False,
    }
])
def test_hetr_benchmark(hetr_device, config):
    pytest.skip('Possible issue only on jenkins, disable until figured out.')
    """
    Description:
        Test to ensure benchmarks are working.
        Benchmark used for test is mini_resnet
    """
    from examples.benchmarks.mini_resnet import run_resnet_benchmark
    c = config
    run_resnet_benchmark(dataset=c['dataset'],
                         num_iterations=c['iter_count'],
                         n_skip=1,
                         batch_size=c['batch_size'],
                         device_id=c['device_id'],
                         transformer_type='hetr',
                         device=hetr_device,
                         bprop=c['bprop'],
                         batch_norm=c['batch_norm'],
                         visualize=False)

max_stars_repo_name: NervanaSystems/ngraph-python


text:
import matplotlib
import matplotlib.colors as colors
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
import networkx as nx
import numpy as np
import sklearn.metrics as metrics
import torch
import torch.nn as nn
from torch.autograd import Variable
import tensorboardX
from tensorboardX import SummaryWriter

import argparse
import os
import pickle
import random
import shutil
import time

import cross_val
import mxPoolSUM
import mxPoolCAT
import encoders
import gen.feat as featgen
import gen.data as datagen
from graph_sampler import GraphSampler
import load_data
import util

def evaluates(dataset, model, args, name='Validation', max_num_examples=None):
    model.eval()

    labels = []
    preds = []
    for batch_idx, data in enumerate(dataset):
        adj = Variable(data['adj'].float(), requires_grad=False).cuda()
        h0 = Variable(data['feats'].float()).cuda()
        labels.append(data['label'].long().numpy())
        batch_num_nodes = data['num_nodes'].int().numpy()
        assign_input = Variable(data['assign_feats'].float(), requires_grad=False).cuda()
        efeat = Variable(data['feature'].float(), requires_grad=False).cuda()
        ypred = model(h0, adj, batch_num_nodes, features=efeat, assign_x=assign_input)
        _, indices = torch.max(ypred, 1)
        preds.append(indices.cpu().data.numpy())
        if max_num_examples is not None:
            if (batch_idx+1)*args.batch_size > max_num_examples:
                break

    labels = np.hstack(labels)
    preds = np.hstack(preds)

    result = {'prec': metrics.precision_score(labels, preds, average='macro'),
              'recall': metrics.recall_score(labels, preds, average='macro'),
              'acc': metrics.accuracy_score(labels, preds),
              'F1': metrics.f1_score(labels, preds, average="micro")}
    print(name, " accuracy:", result['acc'])
    return result

def gen_prefix(args):
    if args.bmname is not None:
        name = args.bmname
    else:
        name = args.dataset
    name += '_' + args.method

    if args.method == 'MxPool':
        name += '_l' + str(args.num_gc_layers) + 'x' + str(args.num_pool)
        print("args.assign_ratio*100",args.assign_ratio)
        if args.multi_pool==1:
            name += '_ar_['
            for i in range(args.num_aspect-1):
                name += str(int(args.assign_ratio[i]*args.max_nodes)) + ','
            name += str(int(args.assign_ratio[args.num_aspect-1]*args.max_nodes)) + ']'
        else:
            name +=  '_ar_'+str(int(args.assign_ratio[0]*args.max_nodes))
        if args.linkpred:
            name += '_lp'
        if args.multi_conv == 1:
            name += '_h' + str(args.hidden_dim) + '_o' + str(args.output_dim)
        else:
            name += '_h' + str(args.hidden_dim[0]) + '_o' + str(args.output_dim[0])
    elif args.method == 'diffpool':
        name += '_l' + str(args.num_gc_layers) + 'x' + str(args.num_pool)
        name += '_ar' + str(int(args.assign_ratio * 100))
        if args.linkpred:
            name += '_lp'
        name += '_h' + str(args.hidden_dim) + '_o' + str(args.output_dim)
    else:
        name += '_l' + str(args.num_gc_layers)

    if not args.bias:
        name += '_nobias'
    if len(args.name_suffix) > 0:
        name += '_' + args.name_suffix
    return name

def gen_train_plt_name(args):
    return './' + gen_prefix(args) + '.png'

def log_assignment(assign_tensor, writer, epoch, batch_idx):
    plt.switch_backend('agg')
    fig = plt.figure(figsize=(8,6), dpi=300)

    # has to be smaller than args.batch_size
    for i in range(len(batch_idx)):
        plt.subplot(2, 2, i+1)
        plt.imshow(assign_tensor.cpu().data.numpy()[batch_idx[i]], cmap=plt.get_cmap('BuPu'))
        cbar = plt.colorbar()
        cbar.solids.set_edgecolor("face")
    plt.tight_layout()
    fig.canvas.draw()

    #data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    #data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
    data = tensorboardX.utils.figure_to_image(fig)
    writer.add_image('assignment', data, epoch)

def log_graph(adj, batch_num_nodes, writer, epoch, batch_idx, assign_tensor=None):
    plt.switch_backend('agg')
    fig = plt.figure(figsize=(8,6), dpi=300)

    for i in range(len(batch_idx)):
        ax = plt.subplot(2, 2, i+1)
        num_nodes = batch_num_nodes[batch_idx[i]]
        adj_matrix = adj[batch_idx[i], :num_nodes, :num_nodes].cpu().data.numpy()
        G = nx.from_numpy_matrix(adj_matrix)
        nx.draw(G, pos=nx.spring_layout(G), with_labels=True, node_color='#336699',
                edge_color='grey', width=0.5, node_size=300,
                alpha=0.7)
        ax.xaxis.set_visible(False)

    plt.tight_layout()
    fig.canvas.draw()
    data = tensorboardX.utils.figure_to_image(fig)
    writer.add_image('graphs', data, epoch)

    assignment = assign_tensor.cpu().data.numpy()
    fig = plt.figure(figsize=(8,6), dpi=300)

    num_clusters = assignment.shape[2]
    all_colors = np.array(range(num_clusters))

    for i in range(len(batch_idx)):
        ax = plt.subplot(2, 2, i+1)
        num_nodes = batch_num_nodes[batch_idx[i]]
        adj_matrix = adj[batch_idx[i], :num_nodes, :num_nodes].cpu().data.numpy()

        label = np.argmax(assignment[batch_idx[i]], axis=1).astype(int)
        label = label[: batch_num_nodes[batch_idx[i]]]
        node_colors = all_colors[label]

        G = nx.from_numpy_matrix(adj_matrix)
        nx.draw(G, pos=nx.spring_layout(G), with_labels=False, node_color=node_colors,
                edge_color='grey', width=0.4, node_size=50, cmap=plt.get_cmap('Set1'),
                vmin=0, vmax=num_clusters-1,
                alpha=0.8)

    plt.tight_layout()
    fig.canvas.draw()

    #data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    #data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
    data = tensorboardX.utils.figure_to_image(fig)
    writer.add_image('graphs_colored', data, epoch)

def train(dataset, model, args, same_feat=True, val_dataset=None, test_dataset=None, writer=None,
        mask_nodes = True):
    writer_batch_idx = [0, 3, 6, 9]

    optimizer = torch.optim.Adam(filter(lambda p : p.requires_grad, model.parameters()), lr=0.001)
    iter = 0
    best_val_result = {
            'epoch': 0,
            'loss': 0,
            'acc': 0}
    test_result = {
            'epoch': 0,
            'loss': 0,
            'acc': 0}
    train_accs = []
    train_epochs = []
    best_val_accs = []
    best_val_epochs = []
    test_accs = []
    test_epochs = []
    val_accs = []

    for epoch in range(args.num_epochs):
        total_time = 0
        avg_loss = 0.0
        model.train()
        print('Epoch: ', epoch)
        for batch_idx, data in enumerate(dataset):
            #print('batch_idx=',batch_idx)
            begin_time = time.time()
            model.zero_grad()
            adj = Variable(data['adj'].float(), requires_grad=False).cuda()
            h0 = Variable(data['feats'].float(), requires_grad=False).cuda()
            label = Variable(data['label'].long()).cuda()
            batch_num_nodes = data['num_nodes'].int().numpy() if mask_nodes else None
            assign_input = Variable(data['assign_feats'].float(), requires_grad=False).cuda()
            gfeat = Variable(data['feature'].float(), requires_grad=False).cuda()
            ypred = model(h0, adj, batch_num_nodes, features=gfeat, assign_x=assign_input) #

            if not (args.method == 'MxPool' or args.method == 'diffpool') or not args.linkpred:
                loss = model.loss(ypred, label)
            else:
                loss = model.loss(ypred, label, adj, batch_num_nodes)
            #optimizer.zero_grad() 这一句是自己加的，不是原文中有的
            loss.backward()
            nn.utils.clip_grad_norm(model.parameters(), args.clip)
            optimizer.step()
            iter = iter + 1
            avg_loss = avg_loss + loss
            #if iter % 20 == 0:
            #    print('Iter: ', iter, ', loss: ', loss.data[0])
            elapsed = time.time() - begin_time
            total_time = total_time + elapsed

            # log once per XX epochs
            if epoch % 10 == 0 and batch_idx == len(dataset) // 2 and args.method == 'MxPool' and writer is not None:
                log_assignment(model.assign_tensor, writer, epoch, writer_batch_idx)
                if args.log_graph:
                    log_graph(adj, batch_num_nodes, writer, epoch, writer_batch_idx, model.assign_tensor)

        avg_loss =avg_loss / (batch_idx + 1)
        if writer is not None:
            writer.add_scalar('loss/avg_loss', avg_loss, epoch)
            if args.linkpred:
                writer.add_scalar('loss/linkpred_loss', model.link_loss, epoch)
        print('Avg loss: ', avg_loss, '; epoch time: ', total_time)
        result = evaluates(dataset, model, args, name='Train', max_num_examples=100)
        train_accs.append(result['acc'])
        train_epochs.append(epoch)
        if val_dataset is not None:
            val_result = evaluates(val_dataset, model, args, name='Validation')
            val_accs.append(val_result['acc'])
        if val_result['acc'] > best_val_result['acc'] - 1e-7:
            best_val_result['acc'] = val_result['acc']
            best_val_result['epoch'] = epoch
            best_val_result['loss'] = avg_loss
        if test_dataset is not None:
            test_result = evaluates(test_dataset, model, args, name='Test')
            test_result['epoch'] = epoch
        if writer is not None:
            writer.add_scalar('acc/train_acc', result['acc'], epoch)
            writer.add_scalar('acc/val_acc', val_result['acc'], epoch)
            writer.add_scalar('loss/best_val_loss', best_val_result['loss'], epoch)
            if test_dataset is not None:
                writer.add_scalar('acc/test_acc', test_result['acc'], epoch)

        print('Best val result: ', best_val_result)
        best_val_epochs.append(best_val_result['epoch'])
        best_val_accs.append(best_val_result['acc'])
        if test_dataset is not None:
            print('Test result: ', test_result)
            test_epochs.append(test_result['epoch'])
            test_accs.append(test_result['acc'])

    matplotlib.style.use('seaborn')
    plt.switch_backend('agg')
    plt.figure()
    plt.plot(train_epochs, util.exp_moving_avg(train_accs, 0.85), '-', lw=1)
    if test_dataset is not None:
        plt.plot(best_val_epochs, best_val_accs, 'bo', test_epochs, test_accs, 'go')
        plt.legend(['train', 'val', 'test'])
    else:
        plt.plot(best_val_epochs, best_val_accs, 'bo')
        plt.legend(['train', 'val'])
    plt.savefig(gen_train_plt_name(args), dpi=600)
    plt.close()
    matplotlib.style.use('default')

    return model, val_accs

def prepare_data(graphs, args, test_graphs=None, max_nodes=0):

    random.shuffle(graphs)
    if test_graphs is None:
        train_idx = int(len(graphs) * args.train_ratio)
        test_idx = int(len(graphs) * (1-args.test_ratio))
        train_graphs = graphs[:train_idx]
        val_graphs = graphs[train_idx: test_idx]
        test_graphs = graphs[test_idx:]
    else:
        train_idx = int(len(graphs) * args.train_ratio)
        train_graphs = graphs[:train_idx]
        val_graphs = graphs[train_idx:]
    print('Num training graphs: ', len(train_graphs),
          '; Num validation graphs: ', len(val_graphs),
          '; Num testing graphs: ', len(test_graphs))

    print('Number of graphs: ', len(graphs))
    print('Number of edges: ', sum([G.number_of_edges() for G in graphs]))
    print('Max, avg, std of graph size: ',
            max([G.number_of_nodes() for G in graphs]), ', '
            "{0:.2f}".format(np.mean([G.number_of_nodes() for G in graphs])), ', '
            "{0:.2f}".format(np.std([G.number_of_nodes() for G in graphs])))

    # minibatch
    dataset_sampler = GraphSampler(train_graphs, normalize=False, max_num_nodes=max_nodes,
            features=args.feature_type)
    train_dataset_loader = torch.utils.data.DataLoader(
            dataset_sampler,
            batch_size=args.batch_size,
            shuffle=True,
            num_workers=args.num_workers)

    dataset_sampler = GraphSampler(val_graphs, normalize=False, max_num_nodes=max_nodes,
            features=args.feature_type)
    val_dataset_loader = torch.utils.data.DataLoader(
            dataset_sampler,
            batch_size=args.batch_size,
            shuffle=False,
            num_workers=args.num_workers)

    dataset_sampler = GraphSampler(test_graphs, normalize=False, max_num_nodes=max_nodes,
            features=args.feature_type)
    test_dataset_loader = torch.utils.data.DataLoader(
            dataset_sampler,
            batch_size=args.batch_size,
            shuffle=False,
            num_workers=args.num_workers)

    return train_dataset_loader, val_dataset_loader, test_dataset_loader, \
            dataset_sampler.max_num_nodes, dataset_sampler.feat_dim, dataset_sampler.assign_feat_dim

def benchmark_task_val(args, writer=None, feat='node-label'):
    all_vals = []
    graphs= load_data.read_graphfile(args.datadir, args.bmname, max_nodes=args.max_nodes)
    #args.max_nodes = 600, maxnodes
    #print("maxnodes=",maxnodes),features
    #compute_matching_degree(features)
    if feat == 'node-feat' and 'feat_dim' in graphs[0].graph:
        print('Using node features')
        input_dim = graphs[0].graph['feat_dim']
    elif feat == 'node-label' and 'label' in graphs[0].nodes[0]:
        print('Using node labels')
        for G in graphs:
            for u in G.nodes():
                G.nodes[u]['feat'] = np.array(G.nodes[u]['label'])
    else:
        print('Using constant labels')
        featgen_const = featgen.ConstFeatureGen(np.ones(args.input_dim, dtype=float))
        for G in graphs:
            featgen_const.gen_node_features(G)

    for i in range(10):
        print("****************", i)
        train_dataset, val_dataset, max_num_nodes, input_dim, assign_input_dim = \
                cross_val.prepare_val_data(graphs, args, i, max_nodes=args.max_nodes)
        temp_input_dim = []
        temp_assign_input_dim = []
        for i in range(args.num_aspect):
            temp_input_dim.append(input_dim)
            temp_assign_input_dim.append(assign_input_dim)
        input_dim_aspect = temp_input_dim
        assign_input_dim_aspect= temp_assign_input_dim
        #assign_input_di, max_graph_node_numm_aspect = [assign_input_dim, assign_input_dim, assign_input_dim]
        if args.method == 'MxPool':
            print('Method: MxPool')
            if args.merge_method == 'cat':
                model = mxPoolCAT.SoftPoolingGcnEncoder(
                    max_num_nodes, args.num_aspect, args.multi_conv, args.multi_pool,
                    input_dim_aspect, args.hidden_dim, args.output_dim, args.num_classes, args.num_gc_layers,
                    args.hidden_dim, assign_ratio=args.assign_ratio, num_pooling=args.num_pool,
                    bn=args.bn, dropout=args.dropout, linkpred=args.linkpred, args=args,
                    assign_input_dim=assign_input_dim_aspect).cuda()
            else:
                model = mxPoolSUM.SoftPoolingGcnEncoder(
                    max_num_nodes, args.num_aspect, args.multi_conv, args.multi_pool,
                    input_dim_aspect, args.hidden_dim, args.output_dim, args.num_classes, args.num_gc_layers,
                    args.hidden_dim, assign_ratio=args.assign_ratio, num_pooling=args.num_pool,
                    bn=args.bn, dropout=args.dropout, linkpred=args.linkpred, args=args,
                    assign_input_dim=assign_input_dim_aspect).cuda()
        elif args.method == 'diffpool':
            print('Method: diffpool')
            model = encoders.SoftPoolingGcnEncoder(
                max_num_nodes,
                input_dim, args.hidden_dim, args.output_dim, args.num_classes, args.num_gc_layers,
                args.hidden_dim, assign_ratio=args.assign_ratio, num_pooling=args.num_pool,
                bn=args.bn, dropout=args.dropout, linkpred=args.linkpred, args=args,
                assign_input_dim=assign_input_dim).cuda()
        else:
            print('Method: base')
            model = encoders.GcnEncoderGraph(
                    input_dim, args.hidden_dim, args.output_dim, args.num_classes,
                    args.num_gc_layers, bn=args.bn, dropout=args.dropout, args=args).cuda()

        _, val_accs = train(train_dataset, model, args, val_dataset=val_dataset, test_dataset=None,
                            writer=writer)
        all_vals.append(np.array(val_accs))

    all_vals = np.vstack(all_vals)
    all_vals = np.mean(all_vals, axis=0)
    print(all_vals)
    print(np.max(all_vals))
    print(np.argmax(all_vals))
    result = "./result.txt"
    f = open(result, 'a+')
    f.write(str(np.max(all_vals)))
    f.write("\n")
    f.close()

def arg_parse():
    parser = argparse.ArgumentParser(description='GraphPool arguments.')
    io_parser = parser.add_mutually_exclusive_group(required=False)
    io_parser.add_argument('--dataset', dest='dataset',
                           help='Input dataset.')
    benchmark_parser = io_parser.add_argument_group()
    benchmark_parser.add_argument('--bmname', dest='bmname',
                                  help='Name of the benchmark dataset')
    io_parser.add_argument('--pkl', dest='pkl_fname',
                           help='Name of the pkl data file')

    softpool_parser = parser.add_argument_group()
    softpool_parser.add_argument('--assign-ratio', dest='assign_ratio', nargs="*",type=float,
                                 help='ratio of number of nodes in consecutive layers')
    softpool_parser.add_argument('--num-pool', dest='num_pool', type=int,
                                 help='number of pooling layers')
    parser.add_argument('--linkpred', dest='linkpred', action='store_const',
                        const=True, default=False,
                        help='Whether link prediction side objective is used, can only be used in diffpool or base')
    parser.add_argument('--datadir', dest='datadir',
                        help='Directory where benchmark is located')
    parser.add_argument('--logdir', dest='logdir',
                        help='Tensorboard log directory')
    parser.add_argument('--cuda', dest='cuda',
                        help='CUDA.')
    parser.add_argument('--max-nodes', dest='max_nodes', type=int,
                        help='Maximum number of nodes (ignore graghs with nodes exceeding the number.')
    parser.add_argument('--lr', dest='lr', type=float,
                        help='Learning rate.')
    parser.add_argument('--clip', dest='clip', type=float,
                        help='Gradient clipping.')
    parser.add_argument('--batch-size', dest='batch_size', type=int,
                        help='Batch size.')
    parser.add_argument('--epochs', dest='num_epochs', type=int,
                        help='Number of epochs to train.')
    parser.add_argument('--train-ratio', dest='train_ratio', type=float,
                        help='Ratio of number of graphs training set to all graphs.')
    parser.add_argument('--num_workers', dest='num_workers', type=int,
                        help='Number of workers to load data.')
    parser.add_argument('--feature', dest='feature_type',
                        help='Feature used for encoder. Can be: id, deg')
    parser.add_argument('--input-dim', dest='input_dim', type=int,
                        help='Input feature dimension')
    parser.add_argument('--hidden-dim', dest='hidden_dim', nargs="*",type=int,
                        help='Hidden dimension')
    parser.add_argument('--output-dim', dest='output_dim', nargs="*",type=int,
                        help='Output dimension')
    parser.add_argument('--num-classes', dest='num_classes', type=int,
                        help='Number of label classes')
    parser.add_argument('--num-gc-layers', dest='num_gc_layers', type=int,
                        help='Number of graph convolution layers before each pooling')
    parser.add_argument('--nobn', dest='bn', action='store_const',
                        const=False, default=True,
                        help='Whether batch normalization is used')
    parser.add_argument('--dropout', dest='dropout', type=float,
                        help='Dropout rate.')
    parser.add_argument('--nobias', dest='bias', action='store_const',
                        const=False, default=True,
                        help='Whether to add bias. Default to True.')
    parser.add_argument('--no-log-graph', dest='log_graph', action='store_const',
                        const=False, default=True,
                        help='Whether disable log graph')

    parser.add_argument('--method', dest='method',
                        help='Method. Possible values: base, diffpool, MxPool')
    parser.add_argument('--name-suffix', dest='name_suffix',
                        help='suffix added to the output filename')
    parser.add_argument('--num-aspect', dest='num_aspect', type=int,
                        help='Number of angles for convolution or pooling')
    parser.add_argument('--multi-conv', dest='multi_conv', type=int,
                        help='Multiplex convolution, 0 or 1')
    parser.add_argument('--multi-pool', dest='multi_pool', type=int,
                        help='Multiplex pool, 0 or 1')
    parser.add_argument('--merge-method', dest='merge_method', help='merge_method :cat or sum. ')
    parser.set_defaults(datadir='data',
                        logdir='log',
                        dataset='syn1v2',
                        max_nodes=100,
                        cuda='0',
                        feature_type='default',
                        lr=0.001,
                        clip=2.0,
                        batch_size=20,
                        num_epochs=1000,
                        train_ratio=0.8,
                        test_ratio=0.1,
                        num_workers=1,
                        input_dim=10,
                        hidden_dim=[30, 50, 80],
                        output_dim=[30, 50, 80],
                        num_classes=6,
                        num_gc_layers=3,
                        dropout=0.0,
                        method='MxPool',
                        name_suffix='',
                        assign_ratio=[0.1, 0.15, 0.2],
                        num_pool=1,
                        bmname='ENZYMES',
                        num_aspect=3,
                        multi_conv=1,
                        multi_pool=1,
                        merge_method='sum',
                       )
    return parser.parse_args()

def main():
    prog_args = arg_parse()
    if prog_args.method=="diffpool" or prog_args.method=="base":
        prog_args.assign_ratio=float(prog_args.assign_ratio[0])
        prog_args.hidden_dim=int(prog_args.hidden_dim[0])
        prog_args.output_dim=int(prog_args.output_dim[0])
    # export scalar data to JSON for external processing
    path = os.path.join(prog_args.logdir, gen_prefix(prog_args))
    print(prog_args.bmname)
    if os.path.isdir(path):
        print(',,,Remove existing log dir: ', path)
        shutil.rmtree(path)
    writer = SummaryWriter(path)
    #writer = None
    os.environ['CUDA_VISIBLE_DEVICES'] = prog_args.cuda
    print('CUDA', prog_args.cuda)

    benchmark_task_val(prog_args, writer=writer)

    writer.close()

if __name__ == "__main__":
    main()

max_stars_repo_name: JucatL/MxPool

reason: import numpy,import networkx


text:
import numpy as np
from matplotlib.dates import date2num

def polyfit(dates, levels, p):
    """
    Returns a np.poly1d object that best fits the dates and relative water levels

    INPUTS
    ------
    dates: a list of datetime objects
    levels: a list of water levels corresponding to the dates
    p: int, the order of best-fit polynomial

    RETURNS
    ------
    poly: np.poly1d object, a pth order order polynomial that best fits the dates and relative water levels
    d0: float, the number of days since the year 0001
    """

    x = np.array(date2num(dates))
    y = levels

    d0 = x[0]

    p_coeff = np.polyfit(x - d0, y, p)
    poly = np.poly1d(p_coeff)

    return poly, d0


text:
import argparse
import os
import datetime
import logging
import time
import math
import numpy as np
from tqdm import tqdm
from collections import OrderedDict
import re

import torch
import torch.nn.functional as F
import torch.backends.cudnn

from core.configs import cfg
from core.datasets import build_dataset
from core.models import build_feature_extractor, build_classifier
from core.utils.misc import mkdir, AverageMeter, intersectionAndUnionGPU, get_color_pallete
from core.utils.logger import setup_logger

def strip_prefix_if_present(state_dict, prefix):
    keys = sorted(state_dict.keys())
    if not all(key.startswith(prefix) for key in keys):
        return state_dict
    stripped_state_dict = OrderedDict()
    for key, value in state_dict.items():
        stripped_state_dict[key.replace(prefix, "")] = value
    return stripped_state_dict

def inference(feature_extractor, classifier, image, label, flip=False):
    size = label.shape[-2:]
    if flip:
        image = torch.cat([image, torch.flip(image, [3])], 0)
    with torch.no_grad():
        output = classifier(feature_extractor(image))
    output = F.interpolate(output, size=size, mode='bilinear', align_corners=True)
    output = F.softmax(output, dim=1)
    if flip:
        output = (output[0] + output[1].flip(2)) / 2
    else:
        output = output[0]
    return output.unsqueeze(dim=0)

def transform_color(pred):
    synthia_to_city = {
        0: 0,
        1: 1,
        2: 2,
        3: 3,
        4: 4,
        5: 5,
        6: 6,
        7: 7,
        8: 8,
        9: 10,
        10: 11,
        11: 12,
        12: 13,
        13: 15,
        14: 17,
        15: 18,
    }
    label_copy = 255 * np.ones(pred.shape, dtype=np.float32)
    for k, v in synthia_to_city.items():
        label_copy[pred == k] = v
    return label_copy.copy()

def test(cfg):
    logger = logging.getLogger("AL-RIPU.tester")
    logger.info("Start testing")
    device = torch.device(cfg.MODEL.DEVICE)

    feature_extractor = build_feature_extractor(cfg)
    feature_extractor.to(device)

    classifier = build_classifier(cfg)
    classifier.to(device)

    if cfg.resume:
        logger.info("Loading checkpoint from {}".format(cfg.resume))
        checkpoint = torch.load(cfg.resume, map_location=torch.device('cpu'))
        feature_extractor_weights = strip_prefix_if_present(checkpoint['feature_extractor'], 'module.')
        feature_extractor.load_state_dict(feature_extractor_weights)
        classifier_weights = strip_prefix_if_present(checkpoint['classifier'], 'module.')
        classifier.load_state_dict(classifier_weights)

    feature_extractor.eval()
    classifier.eval()

    intersection_meter = AverageMeter()
    union_meter = AverageMeter()
    target_meter = AverageMeter()

    torch.cuda.empty_cache()
    dataset_name = cfg.DATASETS.TEST
    output_folder = '.'
    if cfg.OUTPUT_DIR:
        output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
        mkdir(output_folder)

    test_data = build_dataset(cfg, mode='test', is_source=False)

    assert cfg.TEST.BATCH_SIZE == 1, "Test batch size should be 1!"
    test_loader = torch.utils.data.DataLoader(test_data, batch_size=cfg.TEST.BATCH_SIZE, shuffle=False, num_workers=4,
                                              pin_memory=True, sampler=None)

    for batch in tqdm(test_loader):
        x, y, name = batch['img'], batch['label'], batch['name']
        name = name[0]
        x = x.cuda(non_blocking=True)
        y = y.cuda(non_blocking=True).long()

        pred = inference(feature_extractor, classifier, x, y, True)

        output = pred.max(1)[1]
        intersection, union, target = intersectionAndUnionGPU(output, y, cfg.MODEL.NUM_CLASSES, cfg.INPUT.IGNORE_LABEL)
        intersection, union, target = intersection.cpu().numpy(), union.cpu().numpy(), target.cpu().numpy()
        intersection_meter.update(intersection), union_meter.update(union), target_meter.update(target)

        accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10)

        # save the result
        pred = pred.cpu().numpy().squeeze().argmax(0)
        if cfg.MODEL.NUM_CLASSES == 16:
            pred = transform_color(pred)
        mask = get_color_pallete(pred, "city")
        mask_filename = name if len(name.split("/")) < 2 else name.split("/")[1]
        if mask.mode == 'P':
            mask = mask.convert('RGB')
        mask.save(os.path.join(output_folder, mask_filename))

    iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
    accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
    mIoU = np.mean(iou_class)
    mAcc = np.mean(accuracy_class)
    allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)

    logger.info('Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc))
    for i in range(cfg.MODEL.NUM_CLASSES):
        logger.info(
            '{} {} iou/accuracy: {:.4f}/{:.4f}.'.format(i, test_data.trainid2name[i], iou_class[i], accuracy_class[i]))

def main():
    parser = argparse.ArgumentParser(description="Active Domain Adaptive Semantic Segmentation Testing")
    parser.add_argument("-cfg",
                        "--config-file",
                        default="",
                        metavar="FILE",
                        help="path to config file",
                        type=str,
                        )
    parser.add_argument("--proctitle",
                        type=str,
                        default="AL-RIPU",
                        help="allow a process to change its title", )
    parser.add_argument(
        "opts",
        help="Modify config options using the command-line",
        default=None,
        nargs=argparse.REMAINDER,
    )

    args = parser.parse_args()

    torch.backends.cudnn.benchmark = True

    cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)
    cfg.freeze()

    save_dir = ""
    logger = setup_logger("AL-RIPU", save_dir, 0)
    logger.info(cfg)

    logger.info("Loaded configuration file {}".format(args.config_file))
    logger.info("Running with config:\n{}".format(cfg))

    test(cfg)

if __name__ == "__main__":
    main()

max_stars_repo_name: BIT-DA/RIPU

max_issues_repo_name: BinhuiXie/RIPU

max_forks_repo_name: BinhuiXie/RIPU


text:
import cv2
import tensorflow as tf
import sys
import numpy as np
from subprocess import check_output

sys.path.append("..")
from utils.utils import tensor_to_image

def preserve_colors(stylized_images, yuvs):
    """
    Method for preserving color of a video using YUV
    """
    sty_images = []

    for idx in range(len(yuvs)):
        sty = stylized_images[idx]
        content_yuv = yuvs[idx]

        sty_yuv = cv2.cvtColor(np.float32(sty[0].numpy()), cv2.COLOR_RGB2YUV)

        sty_yuv[:, :, 1:3] = content_yuv[:, :, 1:3]

        img = cv2.cvtColor(sty_yuv, cv2.COLOR_YUV2RGB)
        img = tf.convert_to_tensor([img], dtype=tf.float32)

        sty_images.append(img)
    return sty_images

def convert_to_video(images, frame_interval, name="video"):
    """
    Converts array of images to a video
    """
    frame_array = []
    path_out = "../stylized_videos/" + name + ".mp4"
    size = 0

    for i in images:
        file_name = "temp.jpg"
        tensor_to_image(i).save(file_name)
        img = cv2.imread(file_name)
        check_output(f"rm -rf {file_name}".split())
        height, width, layers = img.shape
        size = (width, height)

        frame_array.append(img)
    out = cv2.VideoWriter(
        path_out, cv2.VideoWriter_fourcc(*"DIVX"), 1 / frame_interval, size
    )

    for i in range(len(frame_array)):
        # writing to a image array
        out.write(frame_array[i])
    out.release()

max_stars_repo_name: dilawarm/video-style-transfer

max_issues_repo_name: multitalentloes/video-style-transfer

max_forks_repo_name: multitalentloes/video-style-transfer


text:
# Imports
from torch.optim import optimizer
import os
import copy
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import TensorDataset, DataLoader
from tqdm import tqdm
import numpy as np

# Custom imports
from libs.core.core_base import model as base_model
from libs.utils.utils import *
from libs.utils.logger import Logger
import libs.utils.globals as g
if g.wandb_log:
    import wandb

class model(base_model):
    def batch_forward(self, inputs, labels=None, phase="train", retrain= False):
        """Batch Forward

        Args:
            inputs (float Tensor): batch_size x image_size
            labels (int, optional): Labels. Defaults to None.
            phase (str, optional): Train or Test?. Defaults to "train".
            retrain (bool, optional): When retraining, only the classifier is forward propagated. Defaults to False.
        """

        # Calculate Features and outputs
        if not(retrain):
            self.features = self.networks["feat_model"](inputs)
        else:
            self.features = inputs

        self.logits = self.networks["classifier"](self.features, labels)

    def accumulate(self, phase):
        """Accumulates features of all the datapoints in a particular split

        Args:
            phase ([type]): Which split of dataset should be accumulated?
        """
        print_str = ['Accumulating features: %s' % (phase)]
        print_write(print_str, self.log_file)
        time.sleep(0.25)

        torch.cuda.empty_cache()

        # In validation or testing mode, set model to eval() and initialize running loss/correct
        for model in self.networks.values():
            model.eval()

        # Iterate over dataset
        self.feat = {}
        self.labs = {}

        accum_features = []
        accum_labels = []

        for inputs, labels, paths in tqdm(self.data[phase]):
            inputs, labels = inputs.cuda(), labels.cuda()
            # If on training phase, enable gradients
            with torch.set_grad_enabled(False):

                # In validation or testing
                self.batch_forward(inputs, labels, phase=phase)

                accum_features.append(self.features)
                accum_labels.append(labels)
                torch.cuda.empty_cache()

        # Appending and stacking is same as just concatenating. In this you just dont have to declare a torch.empty of particular size.
        accum_features = torch.vstack(accum_features)
        accum_labels = torch.hstack(accum_labels)

        for i in accum_labels.unique().cpu().numpy():
            self.feat[i] = accum_features[accum_labels == i]
            self.labs[i] = torch.full((self.feat[i].size()[0],), i).cuda()

    def generate_points(self):
        """Generate new datapoints
        """

        if not os.path.isdir(self.config["training_opt"]["log_generate"]):
            os.makedirs(self.config["training_opt"]["log_generate"])
        else:
            raise Exception("Generation Directory already exists!!")

        # Class statistics
        self.base_std = []
        for i in self.feat.keys():
            self.base_std.append(torch.std(self.feat[i], dim=0))

        self.base_std = torch.vstack(self.base_std)

        if self.config["pg"]["generate"]:
            sample_from_each = self.get_sample_count(self.config["training_opt"]["data_count"],self.feat.keys())

            self.generated_points = {}

            for i in tqdm(self.feat.keys()):
                if np.sum(sample_from_each[i]) == 0 and self.config["pg"]["extra_points"] == 0 :
                    continue

                self.generated_points[i] = []
                for k, x_ij in zip(sample_from_each[i], self.feat[i]):
                    if k == 0:
                        continue
                    # Adding a small gaussian noise to the existing train datapoints
                    gen = x_ij + self.config["pg"]["lambda"]*torch.randn((int(k), 128), device='cuda')*self.base_std[i] #FIXME should be standard deviation across samples or dims or dim_samples?
                    self.generated_points[i].append(gen)

                self.generated_points[i] = torch.vstack(self.generated_points[i])

            torch.save(self.generated_points, self.config["training_opt"]["log_generate"] + "/generated_points.pt")
            print("\nPoint Generation Completed!\n")

        else:
            self.generated_points ={}
            print("\nPoint Generation is False!\n")

    def retrain(self,):
        """Creates a new dataloader, reinits everything and trains just the classifier part.
        """
        # Prepare a dataloader for all splits which includes the generated points and is also tukey transformed.
        self.prepare_updated_dataset(include_generated_points = self.config["pg"]["generate"])

        # Change log_dir so it wont change the weights of the parent model
        self.config["training_opt"]["log_dir"] = self.config["training_opt"]["log_retrain"]

        # Create retrain directory
        if not os.path.isdir(self.config["training_opt"]["log_dir"]):
            os.makedirs(self.config["training_opt"]["log_dir"])
        else:
            raise Exception("Retrained Directory already exists!!")

        g.log_dir  = self.config["training_opt"]["log_dir"]
        if g.log_offline:
            if not os.path.isdir(f"{g.log_dir}/metrics"):
                os.makedirs(f"{g.log_dir}/metrics")

        # Reinitialize everything
        print("Using steps for training.")
        self.training_data_num = len(self.my_dataloader["train"].dataset)
        self.epoch_steps = int(
            self.training_data_num / self.training_opt["batch_size"]
        )

        # Init logger
        self.logger = Logger(self.training_opt["log_dir"])
        self.log_file = os.path.join(self.training_opt["log_dir"], "log.txt")
        self.logger.log_cfg(self.config)

        # Initialize loss
        self.init_criterions()

        # Initialize model
        self.init_models()

        # Initialize model optimizer and scheduler
        print("Initializing model optimizer.")
        self.init_optimizers(self.model_optim_params_dict)

        self.train(retrain=True)

    def prepare_updated_dataset(self, include_generated_points = True):
        """Prepares a dataloader for all splits which includes the generated points and is also tukey transformed.

        Args:
            include_generated_points (bool, optional): Do you wanna include the newly generated points. Defaults to True.
        """
        self.my_dataloader = {}
        self.for_distance_analysis = {}
        for phase in ["train", "val", "test"]:

            self.accumulate(phase=phase)

            feat_all = []
            labs_all = []

            if self.config["pg"]["tukey"]:
                for i in self.labs.keys():
                    self.feat[i] = self.tukey_transform(self.feat[i], lam=self.config["pg"]["tukey_value"])

            for i in self.labs.keys():
                feat_all.append(self.feat[i])
                labs_all.append(self.labs[i])

            feat_all = torch.vstack(feat_all)
            labs_all = torch.hstack(labs_all).cuda()

            if include_generated_points and phase == "train":
                generated_points = torch.load(self.config["training_opt"]["log_generate"] + "/generated_points.pt")
                for i in generated_points.keys():
                    feat_all = torch.cat((feat_all, generated_points[i].cuda()))
                    labs_all = torch.cat((labs_all, torch.full((generated_points[i].size()[0],), int(i)).cuda()))

            # Create dataloader
            my_dataset = TensorDataset(feat_all, labs_all, labs_all)
            self.my_dataloader[phase] = DataLoader(my_dataset, batch_size=self.config["training_opt"]["batch_size"], shuffle=True)

# This is there so that we can use source_import from the utils to import model
def get_core(*args):
    return model(*args)

max_stars_repo_name: rahulvigneswaran/TailCalibX

max_issues_repo_name: rahulvigneswaran/TailCalibX

max_forks_repo_name: rahulvigneswaran/TailCalibX


text:
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import csv
import numpy as np
import os
import sys

from observations.util import maybe_download_and_extract

def nwtco(path):
  """Data from the National Wilm's Tumor Study

  Measurement error example. Tumor histology predicts survival, but
  prediction is stronger with central lab histology than with the local
  institution determination.

  A data frame with 4028 observations on the following 9 variables.

  `seqno`
      id number

  `instit`
      Histology from local institution

  `histol`
      Histology from central lab

  `stage`
      Disease stage

  `study`
      study

  `rel`
      indicator for relapse

  `edrel`
      time to relapse

  `age`
      age in months

  `in.subcohort`
      Included in the subcohort for the example in the paper

  Args:

    path: str.
      Path to directory which either stores file or otherwise file will
      be downloaded and extracted there.
      Filename is `nwtco.csv`.

  Returns:

    Tuple of np.ndarray `x_train` with 4028 rows and 9 columns and
    dictionary `metadata` of column headers (feature names).
  """
  import pandas as pd
  path = os.path.expanduser(path)
  filename = 'nwtco.csv'
  if not os.path.exists(os.path.join(path, filename)):
    url = 'http://dustintran.com/data/r/survival/nwtco.csv'
    maybe_download_and_extract(path, url,
                               save_file_name='nwtco.csv',
                               resume=False)

  data = pd.read_csv(os.path.join(path, filename), index_col=0,
                     parse_dates=True)
  x_train = data.values
  metadata = {'columns': data.columns}
  return x_train, metadata


text:
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
MPC training.
"""
import numpy as np
import os
import sys
import time

import paddle.fluid as fluid
import paddle_fl.mpc as pfl_mpc
from paddle_fl.mpc.data_utils.data_utils import get_datautils

sys.path.append('..')
import network
import process_data

mpc_protocol_name = 'aby3'
mpc_du = get_datautils(mpc_protocol_name)

def encrypt_model_and_train(role, ip, server, port, model_save_dir, model_filename):
    """
    Load uci network and train MPC model.

    """
    place = fluid.CPUPlace()
    exe = fluid.Executor(place)

    # Step 1. Initialize MPC environment and load paddle model network and initialize parameter.
    pfl_mpc.init(mpc_protocol_name, role, ip, server, port)
    [_, _, _, loss] = network.uci_network()
    exe.run(fluid.default_startup_program())

    # Step 2. TRANSPILE: encrypt default_main_program into MPC program
    mpc_du.transpile()

    # Step 3. MPC-TRAINING: model training based on MPC program.
    mpc_data_dir = "../mpc_data/"
    feature_file = mpc_data_dir + "house_feature"
    feature_shape = (13,)
    label_file = mpc_data_dir + "house_label"
    label_shape = (1,)
    if not os.path.exists('./tmp'):
        os.makedirs('./tmp')
    loss_file = "./tmp/uci_mpc_loss.part{}".format(role)
    if os.path.exists(loss_file):
        os.remove(loss_file)
    batch_size = network.UCI_BATCH_SIZE
    epoch_num = network.TRAIN_EPOCH
    feature_name = 'x'
    label_name = 'y'
    loader = process_data.get_mpc_dataloader(feature_file, label_file, feature_shape, label_shape,
                               feature_name, label_name, role, batch_size)
    start_time = time.time()
    for epoch_id in range(epoch_num):
        step = 0
        for sample in loader():
            mpc_loss = exe.run(feed=sample, fetch_list=[loss.name])
            if step % 50 == 0:
                print('Epoch={}, Step={}, Loss={}'.format(epoch_id, step, mpc_loss))
                with open(loss_file, 'ab') as f:
                    f.write(np.array(mpc_loss).tostring())
                step += 1
    end_time = time.time()
    print('Mpc Training of Epoch={} Batch_size={}, cost time in seconds:{}'
          .format(epoch_num, batch_size, (end_time - start_time)))

    # Step 4. SAVE trained MPC model as a trainable model.
    mpc_du.save_trainable_model(exe=exe,
                              model_dir=model_save_dir,
                              model_filename=model_filename)
    print('Successfully save mpc trained model into:{}'.format(model_save_dir))

role, server, port = int(sys.argv[1]), sys.argv[2], int(sys.argv[3])
model_save_dir = './tmp/mpc_models_trained/trained_model_share_{}'.format(role)
trained_model_name = 'mpc_trained_model'
encrypt_model_and_train(role=role,
                        ip='localhost',
                        server=server,
                        port=port,
                        model_save_dir=model_save_dir,
                        model_filename=trained_model_name)

max_stars_repo_name: barrierye/PaddleFL

max_issues_repo_name: Sprate/PaddleFL

max_forks_repo_name: Sprate/PaddleFL


text:
import os
from functools import partial

import matplotlib.pyplot as plt
import numpy as np
import pyswarms as ps
from pyswarms.utils.search import RandomSearch

from erinn.python.FW2_5D.fw2_5d_ext import get_forward_para, forward_simulation
from erinn.python.utils.io_utils import read_pkl

def _forward_simulation(pred_log_rho_swarm, obs_V, config):
    l1 = np.array([], dtype=np.float64)
    # l2 = np.array([], dtype=np.float64)
    smooth = np.array([], dtype=np.float64)
    for pred_log_rho in pred_log_rho_swarm:
        tmp = np.power(10, pred_log_rho).reshape(config['nz'], config['nx']).T
        sigma = 1 / tmp
        # l2_tmp = np.array([np.sum(np.power(forward_simulation(sigma, config) - obs_V, 2))])
        l1_tmp = np.array([np.sum(np.abs(forward_simulation(sigma, config) - obs_V))])
        smooth_tmp = np.array([np.sum(np.abs(np.gradient(tmp)))])
        # smooth_tmp = np.array([np.sum(np.power(np.gradient(tmp), 2))])
        # smooth_tmp = np.array([np.sum(np.power(np.diff(F, 0).flatten(), 2))
        #                        + np.sum(np.power(np.diff(tmp, 1).flatten(), 2))])
        if smooth.size:
            l1 = np.concatenate((l1, l1_tmp))
            # l2 = np.concatenate((l2, l2_tmp))
            smooth = np.concatenate((smooth, smooth_tmp))
        else:
            l1 = l1_tmp.copy()
            # l2 = l2_tmp.copy()
            smooth = smooth_tmp.copy()
    return l1 + 0.2 * smooth

if __name__ == '__main__':

    config_file = os.path.join('..', 'config', 'config.yml')
    prediction_dir = os.path.join('..', 'models', 'no_preprocess', 'predictions', 'raw_data')
    pkl_file = os.path.join(prediction_dir, 'result_4750.pkl')
    data = read_pkl(pkl_file)

    # # Random search hyper-parameter
    # options = {'c1': [0.1, 5],
    #            'c2': [0.1, 5],
    #            'w': [0.1, 5],
    #            'k': [4, 32],
    #            'p': 1}
    # num_particle = 64
    # iters = 30
    # n_selection_iters = 10
    # dimensions = data['pred_log_rho'].size
    # max_bound = 5 * np.ones((dimensions, ))
    # min_bound = -2 * np.ones((dimensions, ))
    # bounds = (min_bound, max_bound)
    # velocity_clamp = (-0.3, 0.3)
    #
    # obs_V = data['synth_V'].flatten()
    # config = get_forward_para(config_file)
    # par = partial(_forward_simulation, obs_V=obs_V, config=config)
    # g = RandomSearch(ps.single.LocalBestPSO, n_particles=num_particle,
    #                  dimensions=dimensions, options=options, objective_func=par,
    #                  iters=iters, n_selection_iters=n_selection_iters,
    #                  bounds=bounds, velocity_clamp=velocity_clamp)
    #
    # best_score, best_options = g.search()
    # # >>> best_options = {'c1': 3.5322163151126724,
    # #                     'c2': 2.518708436292546,
    # #                     'w': 1.4265179791119178,
    # #                     'k': 21,
    # #                     'p': 1}

    # Set up hyper-parameters
    options = {'c1': 0.5,
               'c2': 0.4,
               'w': 0.9,
               'k': 16,
               'p': 1}
    num_particle = 64
    iters = 50
    num_processes = os.cpu_count()
    dimensions = data['pred_log_rho'].size
    max_bound = 5 * np.ones((dimensions,))
    min_bound = -2 * np.ones((dimensions,))
    bounds = (min_bound, max_bound)
    bh_strategy = 'random'  # {'nearest', 'random', 'shrink', 'reflective', 'intermediate', 'periodic'}
    velocity_clamp = (-0.2, 0.2)
    vh_strategy = 'unmodified'  # {'unmodified', 'adjust', 'invert', 'zero'}
    ftol = -np.inf
    init_pos = np.tile(data['pred_log_rho'].flatten(), (num_particle, 1))
    init_pos[1:, :] = init_pos[1:, :] + ((1 - (-1)) * np.random.random(init_pos[1:, :].shape) + (-1))

    optimizer = ps.single.LocalBestPSO(n_particles=num_particle,
                                       dimensions=dimensions,
                                       options=options,
                                       bounds=bounds,
                                       bh_strategy=bh_strategy,
                                       velocity_clamp=velocity_clamp,
                                       vh_strategy=vh_strategy,
                                       ftol=ftol,
                                       init_pos=init_pos)

    # Perform optimization
    obs_V = data['synth_V'].flatten()
    config = get_forward_para(config_file)
    # par = partial(_forward_simulation, obs_V=obs_V, config=config)
    # cost, pos = optimizer.optimize(par, iters=iters)
    cost, pos = optimizer.optimize(_forward_simulation, iters=iters,
                                   n_processes=num_processes, obs_V=obs_V, config=config)

    plt.imshow(data['synth_log_rho'])
    plt.colorbar()
    plt.show()
    plt.imshow(data['pred_log_rho'])
    plt.colorbar()
    plt.show()
    plt.imshow(pos.reshape(30, 140))
    plt.colorbar()
    plt.show()


text:
import numpy as np

class Stochastic(object):
    """
    A stochastic (random) model
    """

    def __init__(self, env):
        self.act_size = env.act_size
        self.n_actions = env.n_actions

    def step(self, obs):
        return np.random.choice(self.act_size, self.n_actions, replace=False), 0

    def train(self, obs, d_rewards, d_actions, d_values):
        pass

    def output(self, info):
        pass

    def save(self, save_path):
        pass

    def load(self, load_path):
        pass