text:
import os
import pathlib
import math

import pandas as pd
import numpy as np

from utils.split_pruning import getTrainTimes

# Load the data
PATH = pathlib.Path(__file__).resolve().parent / "1_processed data"
df = pd.read_csv(
    os.path.join(PATH, "4_merged2.csv"),
    index_col=[0, 1],
    parse_dates=["Date", "Label_t1"],
)
# Split stocks in sets
TRAIN_SIZE = 0.6
VAL_SIZE = 0.28

def set_split(df, t, v):
    idx1 = math.ceil(t * df.shape[0])
    idx2 = math.ceil((t + v) * df.shape[0])
    df.loc[:, "Set"] = None
    df.loc[:, "Set"].iloc[:idx1] = "Train"
    df.loc[:, "Set"].iloc[idx1:idx2] = "Validation"
    df.loc[:, "Set"].iloc[idx2:] = "Test"
    return df

df = df.groupby(by="Stock").apply(set_split, t=TRAIN_SIZE, v=VAL_SIZE)
# 'Prune' sets whose features or labels overlap with other sets
def prune(df):
    train = df.loc[df["Set"] == "Train"].droplevel("Stock")
    val = df.loc[df["Set"] == "Validation"].droplevel("Stock")
    test = df.loc[df["Set"] == "Test"].droplevel("Stock")
    train_timestamps = pd.Series(
        train["Label_t1"].values, index=train.reset_index()["Date"].values
    )
    val_timestamps = pd.Series(
        val["Label_t1"].values, index=val.reset_index()["Date"].values
    )
    test_timestamps = pd.Series(
        test["Label_t1"].values, index=test.reset_index()["Date"].values
    )
    train_timestamps_pruned = pd.to_datetime(
        getTrainTimes(train_timestamps, val_timestamps).index, utc=True
    )
    val_timestamps_pruned = pd.to_datetime(
        getTrainTimes(val_timestamps, test_timestamps).index, utc=True
    )
    test_timestamps = pd.to_datetime(test.reset_index()["Date"].values, utc=True)
    timestamps_pruned = list(
        train_timestamps_pruned.union(val_timestamps_pruned).union(test_timestamps)
    )
    return df.droplevel("Stock").loc[timestamps_pruned]

tr_val = df.copy(deep=True)
# Did not prune the train+val set so that the labels don't overlap with the test set, but this is ok for now
tr_val = tr_val.loc[(tr_val["Set"] == "Train") | (tr_val["Set"] == "Validation")].drop(
    ["Label_t1", "Set"], axis=1
)
df = df.groupby(by="Stock").apply(prune)
train = df.loc[df["Set"] == "Train"].drop(["Label_t1", "Set"], axis=1)
val = df.loc[df["Set"] == "Validation"].drop(["Label_t1", "Set"], axis=1)
test = df.loc[df["Set"] == "Test"].drop(["Label_t1", "Set"], axis=1)
# After splitting, trim/prune sets that have instances that are standarized on rolling windows that overlap with other sets?
# Balance classes by undersampling
print("********* Before undersampling")
print("Train set:", train.value_counts("Label"))
print("Val set:", val.value_counts("Label"))
print("Test set:", test.value_counts("Label"))
print("Train and Val set:", tr_val.value_counts("Label"))

def downsample(df: pd.DataFrame, label_col_name: str) -> pd.DataFrame:
    nmin = df[label_col_name].value_counts().min()
    return df.groupby(label_col_name).apply(lambda x: x.sample(nmin)).droplevel(0)

train = downsample(train, "Label")
val = downsample(val, "Label")
test = downsample(test, "Label")
print("********* After undersampling")
print("Train set:", train.value_counts("Label"))
print("Val set:", val.value_counts("Label"))
print("Test set:", test.value_counts("Label"))
# Sort by date in case the algos used need it later on
train.sort_index(sort_remaining=True, inplace=True)
val.sort_index(sort_remaining=True, inplace=True)
test.sort_index(sort_remaining=True, inplace=True)
tr_val.sort_index(sort_remaining=True, inplace=True)
# Split in features and labels
train_features = train.loc[:, train.columns != "Label"]
train_labels = train.loc[:, "Label"]
val_features = val.loc[:, val.columns != "Label"]
val_labels = val.loc[:, "Label"]
test_features = test.loc[:, test.columns != "Label"]
test_labels = test.loc[:, "Label"]
tr_val_features = tr_val.loc[:, tr_val.columns != "Label"]
tr_val_labels = tr_val.loc[:, "Label"]
# Replace -1 labels for 0s so tf.one_hot() works properly later on
train_labels.replace(-1, 0, inplace=True)
val_labels.replace(-1, 0, inplace=True)
test_labels.replace(-1, 0, inplace=True)
tr_val_labels.replace(-1, 0, inplace=True)
# Convert to numpy
train_features = train_features.to_numpy()
train_labels = train_labels.to_numpy()
val_features = val_features.to_numpy()
val_labels = val_labels.to_numpy()
test_features = test_features.to_numpy()
test_labels = test_labels.to_numpy()
tr_val_features = tr_val_features.to_numpy()
tr_val_labels = tr_val_labels.to_numpy()
# Save the data
PATH = pathlib.Path(__file__).resolve().parent / "2_final data"
PATH.mkdir(parents=True, exist_ok=True)
np.save(os.path.join(PATH, "train_features.npy"), train_features)
np.save(os.path.join(PATH, "train_labels.npy"), train_labels)
np.save(os.path.join(PATH, "val_features.npy"), val_features)
np.save(os.path.join(PATH, "val_labels.npy"), val_labels)
np.save(os.path.join(PATH, "test_features.npy"), test_features)
np.save(os.path.join(PATH, "test_labels.npy"), test_labels)
np.save(os.path.join(PATH, "train_val_features.npy"), tr_val_features)
np.save(os.path.join(PATH, "train_val_labels.npy"), tr_val_labels)


text:
########################################################
# Rodrigo Leite - drigols                              #
# Last update: 17/12/2021                              #
########################################################

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as stats

# Cria uma distribuição normal padrão aleatória com np.random.randn() e salva no DataFrame - df:
# - Os argumentos para a função random.randn() são as dimensões: (100000, 1) = 100.000 linhas por 1 coluna.
# - columns=['Grade'] é onde armazenar os dados no df. Se não específicar o df não vai ter uma label específico.
df = pd.DataFrame(np.random.randn(100000, 1), columns=['Grade'])

grade = df['Grade'] # Pega a distribuição criada no label/coluna "Grade".
density = stats.gaussian_kde(grade) # Pega a densidade da distribuição.

n, x, _ = plt.hist(grade, color='lightgrey', density=True, bins=100) # Cria o Histograma.
plt.plot(x, density(x)) # Adiciona a densidade/linha de densidade no plot/gráfico.

s = df['Grade'].std() # Pega o Desvio Padrão da distribuição normal padrão criada.
m = df['Grade'].mean() # Pega a média da distribuição normal padrão criada.

# Annotate 1 stdev
x1 = [m-s, m+s]
y1 = [0.25, 0.25]
plt.plot(x1,y1, color='magenta')
plt.annotate('1s (68.26%)', (x1[1],y1[1]))

# Annotate 2 stdevs
x2 = [m-(s*2), m+(s*2)]
y2 = [0.05, 0.05]
plt.plot(x2,y2, color='green')
plt.annotate('2s (95.45%)', (x2[1],y2[1]))

# Annotate 3 stdevs
x3 = [m-(s*3), m+(s*3)]
y3 = [0.005, 0.005]
plt.plot(x3,y3, color='orange')
plt.annotate('3s (99.73%)', (x3[1],y3[1]))

# Adiciona a média(mean) no plot/gráfico.
plt.axvline(grade.mean(), color='grey', linestyle='dashed', linewidth=1)
plt.show()

ext: py

max_stars_repo_name: drigols/Studies

max_issues_repo_name: drigols/Studies

max_forks_repo_name: drigols/Studies

reason: import numpy,import scipy


text:
# -*- coding: utf-8 -*-
from __future__ import division
import math
import numpy as np
from logger import Logger
from rest_framework.exceptions import APIException
import warnings
warnings.filterwarnings("ignore")

def get_energy(dataTraining, energy_type):
    try:
        feats = []
        for subject in dataTraining:
            energyValues = []
            for channel in subject:
                energyValues += eval(energy_type)(channel)
            feats.append(energyValues)
        return feats
    except Exception as e:
        Logger().log(Logger.ERROR, ERROR=""" {0}""".format(str(e)), ROUTE="get_energy", log_now=True)
        raise APIException

def instantaneous(c_data):
    try:
        IE = []
        for data in c_data:
            IE.append(np.log10((float(1) / float(len(data))) * float(sum(i ** 2 for i in data))))
        return IE
    except Exception as e:
        Logger().log(Logger.ERROR, ERROR=""" {0}""".format(str(e)), log_now=True)
        return e

def teager(c_data):
    try:
        TE = []
        for data in c_data:
            sum_values = sum(abs(data[x]**2) if x == 0
                             else abs(data[x]**2 - data[x - 1] * data[x + 1])
                             for x in xrange(0, len(data) - 1))
            TE.append(np.log10((float(1) / float(len(data))) * float(sum_values)))
        return TE
    except Exception as e:
        Logger().log(Logger.ERROR, ERROR=""" {0}""".format(str(e)), log_now=True)
        return e

reason: import numpy


text:
import matplotlib.pyplot as plt
from matplotlib import pyplot
import numpy as np
np.set_printoptions(suppress=True)
import os
import pickle
import sys
import json
from importlib import reload
import tensorflow as tf
import tensorflow_probability as tfp
tfd = tfp.distributions
sys.path.insert(0, './src')
# %%
hf_usc_indexs = {}
col_names = [
         'episode_id', 'time_step',
         'e_veh_id', 'f_veh_id', 'm_veh_id',
         'm_veh_exists', 'e_veh_att',
         'e_veh_speed', 'f_veh_speed', 'm_veh_speed',
         'e_veh_action', 'f_veh_action', 'm_veh_action',
         'aggressiveness',
         'desired_v','desired_tgap', 'min_jamx', 'max_act', 'min_act',
         'el_delta_v', 'el_delta_x', 'em_delta_v', 'em_delta_x',
         'em_delta_y', 'delta_x_to_merge']

for i, item_name in enumerate(col_names):
    hf_usc_indexs[item_name] = i

# %%
"""
Needed methods
"""
from mpl_toolkits.axes_grid1.inset_locator import inset_axes

def latent_samples(model, sample_index):
    """
    Unlike CVAE, here the latent represents the posterior distribution
    """
    h_seq = history_sca[sample_index, :, 2:]
    enc_h = model.h_seq_encoder(h_seq)
    pos_params = model.belief_net(enc_h)
    sampled_z = model.belief_net.sample_z(pos_params)
    return sampled_z

def latent_vis(zsamples_n):
    fig = pyplot.figure(figsize=(4, 6))
    examples_to_vis = np.random.choice(val_samples, zsamples_n, replace=False)
    #===============
    #  First subplot
    #===============
    # set up the axes for the first plot
    ax = fig.add_subplot(1, 2, 1, projection='3d')

    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_zticks([])
    sampled_z = latent_samples(model, examples_to_vis)
    aggressiveness = history_future_usc[examples_to_vis, 0, -7]
    color_shade = aggressiveness
    att_sc = ax.scatter(sampled_z[:, 0], sampled_z[:, 1], sampled_z[:, 2],
                  s=5, c=color_shade, cmap='rainbow', edgecolors='black', linewidth=0.2)

    ax.tick_params(pad=1)
    ax.grid(False)
    # ax.view_init(30, 50)
    #===============
    #  Second subplot
    #===============
    # set up the axes for the second plot
    ax = fig.add_subplot(1, 2, 2, projection='3d')
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_zticks([])
    att_sc = ax.scatter(sampled_z[:, 0], sampled_z[:, 1], sampled_z[:, 2],
                  s=5, c=color_shade, cmap='rainbow', edgecolors='black', linewidth=0.2)

    axins = inset_axes(ax,
                        width="5%",
                        height="90%",
                        loc='right',
                        borderpad=-2
                       )

    fig.colorbar(att_sc, cax=axins)
    cbar = fig.colorbar(att_sc, cax=axins)
    ax.tick_params(pad=1)
    ax.grid(False)
    ax.view_init(30, 50)
    # ax.set_xlabel('$z_{1}$', labelpad=1)
    # ax.set_ylabel('$z_{2}$', labelpad=1)
    # ax.set_zlabel('$z_{3}$', labelpad=1)
    plt.subplots_adjust(wspace=0.2, hspace=None)

def vectorise(step_row, traces_n):
    return np.repeat(step_row, traces_n, axis=0)

def fetch_traj(data, sample_index, colum_index):
    """ Returns the state sequence. It also deletes the middle index, which is
        the transition point from history to future.
    """
    # data shape: [sample_index, time, feature]
    traj = np.delete(data[sample_index, :, colum_index:colum_index+1], 19, axis=1)
    return traj.flatten()
# %%
"""
Load data
"""
history_len = 20 # steps
rollout_len = 30
data_id = '049'
dataset_name = 'sim_data_'+data_id
data_arr_name = 'data_arrays_h{history_len}_f{rollout_len}'.format(\
                                history_len=history_len, rollout_len=rollout_len)

data_files_dir = './src/datasets/'+dataset_name+'/'
with open(data_files_dir+'latent_mlp/'+data_arr_name+'.pickle', 'rb') as handle:
    data_arrays = pickle.load(handle)
history_future_usc, history_sca, \
                future_idm_s, future_m_veh_c, history_e_veh_a = data_arrays
history_sca = np.float32(history_sca)
future_m_veh_c = np.float32(future_m_veh_c)
future_idm_s = np.float32(future_idm_s)

all_epis = np.unique(history_sca[:, 0, 0])
np.random.seed(2021)
np.random.shuffle(all_epis)
train_epis = all_epis[:int(len(all_epis)*0.7)]
val_epis = np.setdiff1d(all_epis, train_epis)
# np.where(train_epis == 64)
train_samples = np.where(history_future_usc[:, 0:1, 0] == train_epis)[0]
val_samples = np.where(history_future_usc[:, 0:1, 0] == val_epis)[0]
# history_sca.shape
train_samples.shape
# %%
"""
Load model (with config file)
"""
model_name = 'latent_mlp_12'
epoch_count = '15'
exp_path = './src/models/experiments/'+model_name+'/model_epo'+epoch_count
exp_dir = os.path.dirname(exp_path)
with open(exp_dir+'/'+'config.json', 'rb') as handle:
    config = json.load(handle)
    print(json.dumps(config, ensure_ascii=False, indent=4))

from models.core import latent_mlp
reload(latent_mlp)
from models.core.latent_mlp import  LatentMLP
model = LatentMLP(config)
model.load_weights(exp_path).expect_partial()

with open(data_files_dir+'env_scaler.pickle', 'rb') as handle:
    model.forward_sim.env_scaler = pickle.load(handle)

with open(data_files_dir+'dummy_value_set.pickle', 'rb') as handle:
    model.forward_sim.dummy_value_set = pickle.load(handle)
# model.forward_sim.attention_temp = 10
# %%
with open(exp_dir+'/'+'losses.pickle', 'rb') as handle:
    losses = pickle.load(handle)
plt.figure()
plt.plot(losses['test_llloss'], label='test_llloss')
plt.plot(losses['train_llloss'], label='train_llloss')
plt.grid()
plt.legend()
plt.figure()
plt.plot(losses['test_klloss'], label='test_klloss')
plt.plot(losses['train_klloss'], label='train_klloss')
plt.legend()
plt.grid()

# %%
"""
Latent visualisation - aggressiveness used for color coding the latent samples
"""
latent_vis(zsamples_n=1000)
# %%
"""
Visualisation of model predictions. Use this for debugging.
"""
Example_pred = 0
i = 0
covered_episodes = []
traces_n = 2
sepcific_samples = []

"""
Posterior is used, as encoder here is an inference network
that estimates the latent vehicle state.
"""
distribution_name = 'posterior'
# distribution_name = 'prior'

# for i in bad_samples[0]:
# for i in sepcific_samples:
# for i in bad_zs:
# for i in bad_samples[0]:
while Example_pred < 10:
    "ENSURE ONLY VAL SAMPLES CONSIDERED"
    sample_index = [val_samples[i]]
    # sample_index = [train_samples[i]]
    # sample_index = [i]
    i += 1
    e_veh_att = fetch_traj(history_future_usc, sample_index, hf_usc_indexs['e_veh_att'])
    m_veh_exists = fetch_traj(history_future_usc, sample_index, hf_usc_indexs['m_veh_exists'])
    aggressiveness = history_future_usc[sample_index, 0, hf_usc_indexs['aggressiveness']][0]
    em_delta_y = fetch_traj(history_future_usc, sample_index, hf_usc_indexs['em_delta_y'])
    episode = future_idm_s[sample_index, 0, 0][0]
    if episode not in covered_episodes:
    # if 4 == 4:
    # if episode not in covered_episodes and e_veh_att[55:65].mean() > 0:
        covered_episodes.append(episode)
        future_idm_ss = vectorise(future_idm_s[sample_index, :, 2:], traces_n)
        merger_cs = vectorise(future_m_veh_c[sample_index, :, 2:], traces_n)
        if distribution_name == 'posterior':
            h_seq = vectorise(history_sca[sample_index, :, 2:], traces_n)
            enc_h = model.h_seq_encoder(h_seq)
            pos_params = model.belief_net(enc_h)
            sampled_z = model.belief_net.sample_z(pos_params)

        elif distribution_name == 'prior':
            prior = tfd.Normal(loc=tf.zeros([traces_n, model.belief_net.latent_dim]), scale=1)
            sampled_z = prior.sample()

        sampled_z = tf.reshape(\
                        sampled_z, [traces_n, 1, model.belief_net.latent_dim])
        _, _, act_seq = model.forward_sim.rollout([sampled_z, \
                                future_idm_ss, merger_cs], training=False)
        future_idm_ss.shape
        act_seq = act_seq.numpy()
        plt.figure(figsize=(5, 3))
        episode_id = history_future_usc[sample_index, 0, hf_usc_indexs['episode_id']][0]
        e_veh_id = history_future_usc[sample_index, 0, hf_usc_indexs['e_veh_id']][0]
        time_0 = int(history_future_usc[sample_index, 0, hf_usc_indexs['time_step']][0])
        time_steps = range(time_0, time_0+49)
        info = [str(item)+' '+'\n' for item in [episode_id, time_0, e_veh_id, aggressiveness]]
        plt.text(0.1, 0.5,
                        'episode_id: '+ info[0] +
                        'time_0: '+ info[1] +
                        'e_veh_id: '+ info[2] +
                        'aggressiveness: '+ info[3] +
                        'speed: '+ str(future_idm_ss[0, 0, 0])
                            , fontsize=10)

        plt.figure(figsize=(5, 3))
        traj = fetch_traj(history_future_usc, sample_index, hf_usc_indexs['f_veh_action'])
        plt.plot(time_steps, traj, color='purple')
        traj = fetch_traj(history_future_usc, sample_index, hf_usc_indexs['e_veh_action'])
        plt.plot(time_steps, traj, color='black', linewidth=2)
        traj = fetch_traj(history_future_usc, sample_index, hf_usc_indexs['m_veh_action'])
        plt.plot(time_steps, traj, color='red')
        plt.legend(['f_veh_action', 'e_veh_action', 'm_veh_action'])

        for sample_trace_i in range(traces_n):
           plt.plot(time_steps[19:], act_seq[sample_trace_i, :, :].flatten(),
                                        color='grey', alpha=0.4)
           # plt.plot(range(29, 59), act_seq[sample_trace_i, :, :].flatten(), color='grey')

        # plt.ylim(-3, 3)
        plt.title(str(sample_index[0]) + ' -- Action')
        plt.grid()

        plt.figure(figsize=(5, 3))
        Example_pred += 1

max_stars_repo_name: saArbabi/DriverActionEstimators

max_issues_repo_name: saArbabi/DriverActionEstimators

max_forks_repo_name: saArbabi/DriverActionEstimators


text:
import numpy as np

def highligthroi(data, roi, verobse=False):
    data[:,:,2] = data[:,:,2] - np.multiply(data[:,:,2],roi)
    return data


text:
# Copyright (C) 2018 Seoul National University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import numpy as np
import tensorflow as tf
from tensorflow.contrib.training.python.training import device_setter
import horovod.tensorflow as hvd

from parallax.core.python.common.graph_transform_lib import *
from parallax.core.python.common.lib import *
from parallax.core.python.hybrid.in_graph_parallel import in_graph_auto_parallel_compute
from parallax.core.python.hybrid.between_graph_parallel import between_graph_auto_parallel_compute
from parallax.core.python.mpi.graph_transform import _add_aggregation_ops

def _place_post_grad_agg_ops_hybrid(ps_device,
                                    var_op_to_agg_grad,
                                    var_op_to_apply_grad_op):
    def _find_agg_grad_descendant_ops(agg_grad_ops, apply_grad_ops):
        agg_grad_descendant_ops = set()
        queue = []
        queue.extend(agg_grad_ops)

        while len(queue) > 0:
            curr_op = queue.pop()
            if curr_op in agg_grad_descendant_ops:
                continue
            agg_grad_descendant_ops.add(curr_op)
            if curr_op in apply_grad_ops:
                continue
            curr_op_consumers = get_consumers(curr_op)
            queue.extend(curr_op_consumers)
        return agg_grad_descendant_ops

    SHARED = -1

    def _assign(op_to_task, agg_grad_ops, apply_grad_ops,
                apply_grad_ancestor_ops, ancestors_diff_descendants,
                is_parent_to_child):
        queue = []
        stop = set()
        if is_parent_to_child:
            queue.extend(agg_grad_ops)
            stop.update(apply_grad_ops)
        else:
            queue.extend(apply_grad_ops)
            stop.update(agg_grad_ops)

        visited = set()
        while len(queue) > 0:
            curr_op = queue.pop(0)
            if curr_op in visited:
                continue
            visited.add(curr_op)
            # already assigned to a task,
            # so skip computing placement and then add next ops to the queue
            if curr_op in op_to_task and curr_op not in stop:
                if is_parent_to_child:
                    # do not care about ops not required for applying gradients
                    queue.extend(
                        [consumer for consumer in get_consumers(curr_op)
                         if consumer in apply_grad_ancestor_ops])
                else:
                    queue.extend([input.op for input in curr_op.inputs])
                continue

            if is_parent_to_child:
                placement_reference_ops = \
                    set([input.op for input in curr_op.inputs])
                placement_reference_ops = placement_reference_ops.difference(
                    ancestors_diff_descendants)
            else:
                placement_reference_ops = set(get_consumers(curr_op))
                placement_reference_ops = placement_reference_ops.intersection(
                    apply_grad_ancestor_ops)

            is_ready = True
            for ref_op in placement_reference_ops:
                if ref_op not in op_to_task:
                    is_ready = False
                    break

            if is_ready:
                placement_reference_tasks = \
                    [op_to_task[ref_op] for ref_op in placement_reference_ops]
            else:
                # requeue and wait for references
                queue.append(curr_op)
                continue

            unique_tasks = set(placement_reference_tasks)
            curr_op_task = None
            if len(unique_tasks) == 0:
                raise RuntimeError(
                    "Should have placement reference for operation %s"
                    % curr_op.name)
            elif len(unique_tasks) == 1:
                curr_op_task = unique_tasks.pop()
                op_to_task[curr_op] = curr_op_task
            else:
                # priority: assigned placement > SHARED
                if SHARED in unique_tasks:
                    unique_tasks.remove(SHARED)
                if len(unique_tasks) == 1:
                    curr_op_task = unique_tasks.pop()
                    op_to_task[curr_op] = curr_op_task
                else:
                    # multiple device placement -> SHARED
                    assert len(unique_tasks) > 1
                    curr_op_task = SHARED
                    op_to_task[curr_op] = SHARED
                    parallax_log.debug(unique_tasks)
            if curr_op_task != SHARED:
                parallax_log.debug('post_grad_agg_op %s is assigned to %s task %d'
                              % (curr_op.name, curr_op_task[0], curr_op_task[1]))

            if curr_op_task == SHARED:
                # TODO: do not assign all SHARED ops to task 0
                # - we can do better
                curr_op_task = 0
                ps_device.task = curr_op_task
                if tf.DeviceSpec.from_string(curr_op.device).job != ps_device.job or tf.DeviceSpec.from_string(curr_op.device).task != ps_device.task:
                  parallax_log.debug('shared_op : %s - %s -> %s' %(curr_op.name, curr_op.device, ps_device.to_string()))
                curr_op._set_device(ps_device)
                assert curr_op.device == ps_device.to_string()
            else:
                d = tf.DeviceSpec(job=curr_op_task[0], task=curr_op_task[1])
                if tf.DeviceSpec.from_string(curr_op.device).job != d.job or tf.DeviceSpec.from_string(curr_op.device).task != d.task:
                  parallax_log.debug('local_op : %s - %s -> %s' %(curr_op.name, curr_op.device, d.to_string()))
                curr_op._set_device(d)
                assert curr_op.device == d.to_string()

            if curr_op not in stop:
                if is_parent_to_child:
                    # do not care about ops not required for applying gradients
                    queue.extend(
                        [consumer for consumer in get_consumers(curr_op)
                         if consumer in apply_grad_ancestor_ops])
                else:
                    queue.extend([input.op for input in curr_op.inputs])

    op_to_task = {}
    agg_grad_ops = []
    for var_op, agg_grad in var_op_to_agg_grad.items():
        var_device = tf.DeviceSpec.from_string(var_op.device)
        if agg_grad[0] != None:
            agg_grad_ops.append(agg_grad[0].op)
            op_to_task[agg_grad[0].op] = (var_device.job, var_device.task)
        agg_grad_ops.append(agg_grad[1].op)
        op_to_task[agg_grad[1].op] = (var_device.job, var_device.task)

    apply_grad_ops = []
    for var_op, apply_grad_op in var_op_to_apply_grad_op.items():
        var_device = tf.DeviceSpec.from_string(var_op.device)
        apply_grad_ops.append(apply_grad_op)
        # colocate apply_grad and variable
        apply_grad_op._set_device(var_device)
        op_to_task[apply_grad_op] = (var_device.job, var_device.task)

    # Note(gyeongin): Need to include control dependency ops in ancestors and
    # descendants or not?
    apply_grad_ancestor_ops = get_ancestors(apply_grad_ops, agg_grad_ops)
    agg_grad_descendant_ops = _find_agg_grad_descendant_ops(agg_grad_ops,
                                                            apply_grad_ops)
    ancestors_diff_descendants = \
        apply_grad_ancestor_ops.difference(agg_grad_descendant_ops)
    parallax_log.debug(
        "apply_grad_ancestor_ops: %d" % len(apply_grad_ancestor_ops))
    parallax_log.debug(
        "agg_grad_descendant_ops: %d" % len(agg_grad_descendant_ops))
    parallax_log.debug(
        "ancestors diff descendants: %d" % len(ancestors_diff_descendants))
    parallax_log.debug(
        "descendants diff ancestors: %d"
        % len(agg_grad_descendant_ops.difference(apply_grad_ancestor_ops)))

    parallax_log.debug('boundary_between_servers called')

    before = {}
    for op in tf.get_default_graph().get_operations():
      before[op.name] = tf.DeviceSpec.from_string(op.device)

    _assign(op_to_task, agg_grad_ops, apply_grad_ops, apply_grad_ancestor_ops,
            ancestors_diff_descendants, is_parent_to_child=True)
    _assign(op_to_task, agg_grad_ops, apply_grad_ops, apply_grad_ancestor_ops,
            ancestors_diff_descendants, is_parent_to_child=False)

    for op in tf.get_default_graph().get_operations():
      if before[op.name].job != tf.DeviceSpec.from_string(op.device).job or before[op.name].task != tf.DeviceSpec.from_string(op.device).task:
        parallax_log.debug('boundary between servers: %s, %s -> %s' % (op.name, before[op.name].to_string(), op.device))

def _add_broadcast_ops(target, worker_id):
    bcast_global_variables_ops = []
    with tf.device('/job:worker/task:%d' % worker_id):
        for var in target:
            bcast_global_variables_ops.append(
                tf.assign(var, hvd.broadcast(var, 0)))
        with tf.control_dependencies(bcast_global_variables_ops):
            tf.no_op(name='auto_parallel_bcast_global_vars')

def graph_transform_dense_mpi(worker_id,
                              meta_graph_def,
                              op_library_path,
                              config):
    with tf.Graph().as_default() as graph:
        tf.train.import_meta_graph(meta_graph_def)

        num_workers = hvd.size()

        op_to_control_consumer_ops = get_all_control_consumers(graph)
        trainable_variable_ops = [var.op for var in tf.get_collection(
            tf.GraphKeys.TRAINABLE_VARIABLES)]

        global_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
        local_variables = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES)

        sparse_var_ops = \
            set([gradients_info._target.op
             for gradients_info in tf.get_collection(tf.GraphKeys.GRADIENTS_INFO)
             if not isinstance(gradients_info._grad, tf.Tensor)])

        for op in tf.get_default_graph().get_operations():
            if op.type in sparse_var_update_op_types.keys():
                sparse_var_ops.add(op.inputs[UPDATE_OP_VAR_POS].op)

        global_grad_ops = [var.op
                           for var in tf.get_collection(PARALLAX_GLOBAL_GRADS)]

        for var in global_variables + local_variables:
            op = var.op
            if op not in sparse_var_ops and op not in global_grad_ops:
                op._set_device('/job:worker/task:%d' % worker_id)

        var_op_to_agg_grad = {}
        with tf.device('/job:worker/task:%d' % worker_id):
            for gradients_info in tf.get_collection(tf.GraphKeys.GRADIENTS_INFO):
                target_tensor = gradients_info._target
                grad = gradients_info._grad
                if target_tensor.op not in trainable_variable_ops:
                    parallax_log.debug(
                        "Gradient for non-trainable variable %s is created, ignore"
                        % target_tensor.op.name)
                    continue
                if isinstance(grad, tf.Tensor):
                    _add_aggregation_ops(gradients_info, op_to_control_consumer_ops,
                                         config)
                    # Now, gradients_info._grad = aggregated gradient
                    var_op_to_agg_grad[target_tensor.op.name] = gradients_info._grad.name

    return tf.train.export_meta_graph(graph=graph), var_op_to_agg_grad

def _init_for_worker_vars(meta_graph_def, worker_id):
    with tf.Graph().as_default() as graph:
        tf.train.import_meta_graph(meta_graph_def)
        worker_vars = []
        global_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
        local_variables = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES)
        for var in global_variables + local_variables:
            device = tf.DeviceSpec.from_string(var.device)
            if device.job == 'worker' and device.task == worker_id:
                worker_vars.append(var)
        _add_broadcast_ops(worker_vars, worker_id)
        meta_graph_def = tf.train.export_meta_graph(graph=graph)
        return meta_graph_def

def graph_transform_hybrid(single_gpu_meta_graph_def,
                           worker_id,
                           local_worker_id,
                           machine_id,
                           hostname,
                           config):
    cluster_info = config.resource_info
    this_worker = None
    for w in cluster_info['worker']:
        if w['hostname'] == hostname:
            this_worker = w
    num_worker_machines = len(cluster_info['worker'])
    num_local_workers = max(len(this_worker['gpus']), 1)
    ps_device = '/job:ps' if 'ps' in cluster_info else '/job:worker/cpu:0'
    ps_job_name = tf.DeviceSpec.from_string(ps_device).job
    cluster_spec = get_tf_clusterspec(cluster_info).as_dict()
    num_pss = len(cluster_spec[ps_job_name])

    cluster_spec = get_tf_clusterspec(cluster_info)

    parallax_log.debug(
        "Starting graph transformation for PS for worker %d" % worker_id)

    # TODO: handle op_library_path
    tensor_or_op_name_to_replica_names = TensorOrOpNameToReplicaNames(
        single_gpu_meta_graph_def.meta_info_def.stripped_op_list)

    multi_gpu_meta_graph_def = \
        in_graph_auto_parallel_compute(
            single_gpu_meta_graph_def, worker_id, local_worker_id,
            num_local_workers, config, None,
            tensor_or_op_name_to_replica_names)

    local_aggregation = config.communication_config.ps_config.local_aggregation
    ps_meta_graph_def, var_op_to_agg_grad, trainable_var_op_to_update_op = \
        between_graph_auto_parallel_compute(
            multi_gpu_meta_graph_def,
            worker_id=worker_id,
            local_worker_id=local_worker_id,
            machine_id=machine_id,
            num_local_workers=num_local_workers,
            num_worker_machines=num_worker_machines,
            ps_device=ps_device,
            worker_device='/job:worker/task:%d' % worker_id,
            merge_devices=True,
            cluster_spec=cluster_spec,
            sync=True,
            op_library_path=None,
            num_replicas_per_worker=1,
            average_sparse=config.average_sparse,
            tensor_or_op_name_to_replica_names=tensor_or_op_name_to_replica_names,
            local_aggregation=local_aggregation)

    meta_graph_def, dense_var_op_to_agg_grad = \
        graph_transform_dense_mpi(worker_id,
                                  ps_meta_graph_def,
                                  op_library_path=None,
                                  config=config)

    with tf.Graph().as_default() as graph:
        tf.train.import_meta_graph(meta_graph_def)
        var_op_to_agg_grad_ = {}
        for var_op_name in var_op_to_agg_grad:
          var_op = graph.get_operation_by_name(var_op_name)
          indices = graph.get_tensor_by_name(var_op_to_agg_grad[var_op_name][0])
          values = graph.get_tensor_by_name(var_op_to_agg_grad[var_op_name][1])
          var_op_to_agg_grad_[var_op] = (indices, values)

        trainable_var_op_to_update_op_ = {}
        for var_op_name in trainable_var_op_to_update_op:
          var_op = graph.get_operation_by_name(var_op_name)
          update_op = graph.get_operation_by_name(trainable_var_op_to_update_op[var_op_name])
          trainable_var_op_to_update_op_[var_op] = update_op

        # TODO If we do MPI transform first, we can bring this block back to
        # between-graph transform.
        for var_op_name in dense_var_op_to_agg_grad.keys():
           var_op = graph.get_operation_by_name(var_op_name)
           grad =  graph.get_tensor_by_name(dense_var_op_to_agg_grad[var_op_name])
           if var_op in var_op_to_agg_grad_:
                raise RuntimeError('A dense variable is handled by between-graph.')
           else:
                var_op_to_agg_grad_[var_op] = \
                    (None, grad)

        if config.communication_config.ps_config.boundary_among_servers:
            _place_post_grad_agg_ops_hybrid(tf.DeviceSpec.from_string(ps_device),
                                            var_op_to_agg_grad_, trainable_var_op_to_update_op_)
        if config.communication_config.ps_config.boundary_between_workers_and_servers:
            set_boundary_between_workers_and_servers()
        meta_graph_def = tf.train.export_meta_graph(graph=graph)

    meta_graph_def = _init_for_worker_vars(meta_graph_def, worker_id)

    parallax_log.debug(
        "Finished graph transformation for PS for worker %d" % worker_id)

    return meta_graph_def, tensor_or_op_name_to_replica_names.export()

max_stars_repo_name: snuspl/parallax

max_issues_repo_name: nikkkkhil/auto-parallal-deeplearning

max_forks_repo_name: nikkkkhil/auto-parallal-deeplearning


text:
import numpy
from skimage import io
import tensorflow as tf
from PIL import Image
import os, sys, glob
from itertools import combinations
from functools import reduce

def log10(x):
    numerator = tf.log(x)
    denominator = tf.log(tf.constant(10, dtype=numerator.dtype))
    return numerator / denominator

def psnr(im1, im2):
    im1 = Image.open(im1)
    im2 = Image.open(im2)
    img_arr1 = numpy.array(im1).astype('double')
    img_arr2 = numpy.array(im2).astype('double')
    mse = tf.reduce_mean(tf.squared_difference(img_arr1, img_arr2))
    psnr = tf.constant(255**2, dtype=tf.double)/mse
    result = tf.constant(10, dtype=tf.double)*log10(psnr)
    with tf.Session():
        result = result.eval()
    return result

def msssim(im1, im2):
    im1 = Image.open(im1)
    im2 = Image.open(im2)
    img_arr1 = numpy.array(im1).astype('double')
    img_arr2 = numpy.array(im2).astype('double')
    img_arr1 = tf.convert_to_tensor(img_arr1)
    img_arr2 = tf.convert_to_tensor(img_arr2)
    with tf.Session():
        result = tf.image.ssim_multiscale(img_arr1, img_arr2, max_val=255)
        result = result.eval()
    return result

def Average(lst):
    return reduce(lambda a, b: a + b, lst) / len(lst)

def calculate_metrics():
    mydir = os.getcwd()
    os.chdir(str(mydir)+'/samples/' + 'relu/test_processed')
    idx=0
    m_list = []
    filelist =  glob.glob('*.png')
    combs = combinations(filelist, 2)
    for combination in combinations(filelist, 2):
        print(combination)
        m_list.append(psnr(str(combination[0]),str(combination[1])))
        print(m_list)
        # print('average ms-ssim score:')
        # print(Average(m_list))
        # print('iter:')
        # print(len(m_list))
    return m_list

calculate_metrics()

max_issues_repo_name: chinokenochkan/bs-dcgan

max_forks_repo_name: chinokenochkan/bs-dcgan


text:
"""
File Name: patch_generator.py

Authors: Kyle Seidenthal

Date: 05-10-2020

Description: Generate a patch dataset for training and testing.

"""

import skimage.io as io
import numpy as np
import os
import csv
import argparse
import sys

from skimage.util.shape import view_as_blocks
from math import floor
from tqdm import tqdm

def create_train_patch(img_path, mask_path, percent_patches, out_dir,
                       crop_padding, patch_shape=(256, 256)):
    """Create patches from the training image by:
        1. Cropping the image to remove extra background.
        2. Selecting n random patches from the cropped image.

    Args:
        img_path (string): Path to the image to make patches from
        mask_path (string): Path to the mask for the image
        percent_patches (float): The percentage of the image to cover with
                                 patches.  This makes sure we cover a majority
                                 of the image proportional to its size.
        out_dir (string): Path to the output directory for training data.
                          If an images/ and masks/ directory are available,
                          patches will be sorted.

        crop_padding (int): Number of pixels padding to use when cropping
        patch_shape (tuple): (x, y) size of each patch

    Returns: None, the patches are saved to out_dir

    """
    dirs = os.listdir(out_dir)

    out_dir_img = None
    out_dir_mask = None

    for d in dirs:
        if os.path.isdir(os.path.join(out_dir, d)):
            if "images" in d:
                out_dir_img = os.path.join(out_dir, d)
            elif "masks" in d:
                out_dir_mask = os.path.join(out_dir, d)

    mask = io.imread(mask_path)
    img = io.imread(img_path, plugin='pil')

    img_name = os.path.split(os.path.splitext(img_path)[0])[-1]

    cropped_img, cropped_mask = crop_and_pad(img, mask, crop_padding)

    num_patches_x = cropped_mask.shape[0] // patch_shape[0]
    num_patches_y = cropped_mask.shape[1] // patch_shape[1]

    num_patches = num_patches_x * num_patches_y
    num_patches = int(percent_patches * num_patches)

    for i in range(num_patches):
        x = cropped_mask.shape[0]
        y = cropped_mask.shape[1]

        while (x + patch_shape[0]) >= cropped_mask.shape[0]:
            x = np.random.randint(0, cropped_mask.shape[0])

        while (y + patch_shape[1]) >= cropped_mask.shape[1]:
            y = np.random.randint(0, cropped_mask.shape[1])

        patch_img = cropped_img[x:x+patch_shape[0], y:y+patch_shape[1]]
        patch_mask = cropped_mask[x:x+patch_shape[0], y:y+patch_shape[1]]

        out_name = img_name + "_" + str(i) + ".png"
        out_name_mask = img_name + "_" + str(i) + "_mask.png"

        if out_dir_img is None:
            out_name = os.path.join(out_dir, out_name)
        else:
            out_name = os.path.join(out_dir_img, out_name)

        if out_dir_mask is None:
            out_name_mask = os.path.join(out_dir, out_name_mask)
        else:
            out_name_mask = os.path.join(out_dir_mask, out_name_mask)
        try:
            io.imsave(out_name, patch_img, check_contrast=False)
            io.imsave(out_name_mask, patch_mask, check_contrast=False)
        except ValueError as e:
            print(e)
            print(out_name, patch_img.shape)

def create_test_patch(img_path, mask_path, out_dir, crop_padding,
                      patch_shape=(256, 256)):
    """Create patches from the test image by splitting the entire image into
       non-overlapping patches.

    Args:
        img_path (string): Path to the image to make patches from.
        mask_path (string): Path to the mask for the image.
        out_dir (string): Path to the output directory
        crop_padding (int): The amount of padding around the cropped
                            foreground.
        patch_shape (tuple): The shape (x, y) of the patches

    Returns: None, patches are saved to out_dir

    """

    dirs = os.listdir(out_dir)

    out_dir_img = None
    out_dir_mask = None

    for d in dirs:
        if os.path.isdir(os.path.join(out_dir, d)):
            if "images" in d:
                out_dir_img = os.path.join(out_dir, d)
            elif "masks" in d:
                out_dir_mask = os.path.join(out_dir, d)

    img_name = os.path.split(os.path.splitext(img_path)[0])[-1]

    img = io.imread(img_path, plugin='pil')
    mask = io.imread(mask_path)

    img, mask = crop_and_pad(img, mask, crop_padding)

    num_patches_x = floor(img.shape[0] / patch_shape[0])
    num_patches_y = floor(img.shape[1] / patch_shape[1])

    patch_excess_x = img.shape[0] - (patch_shape[0] * num_patches_x)
    patch_excess_y = img.shape[1] - (patch_shape[1] * num_patches_y)

    pad_x = (0, patch_shape[0] - patch_excess_x)
    pad_y = (0, patch_shape[1] - patch_excess_y)

    img = np.pad(img, (pad_x, pad_y, (0, 0)), 'constant',
                 constant_values=(0, 0))
    mask = np.pad(mask, (pad_x, pad_y), 'constant', constant_values=(0, 0))

    # Make the blocks
    img_block_shape = (patch_shape[0], patch_shape[1], img.shape[2])

    img_blocks = view_as_blocks(img, block_shape=img_block_shape)
    mask_blocks = view_as_blocks(mask, block_shape=patch_shape)

    patch_num = 0
    for i in range(img_blocks.shape[0]):
        for j in range(img_blocks.shape[1]):
            patch_img = img_blocks[i, j][0]
            patch_mask = mask_blocks[i, j]

            out_name = img_name + "_" + str(patch_num) + ".png"
            out_name_mask = img_name + "_" + str(patch_num) + "_mask.png"

            patch_num += 1

            if out_dir_img is None:
                out_name = os.path.join(out_dir, out_name)
            else:
                out_name = os.path.join(out_dir_img, out_name)

            if out_dir_mask is None:
                out_name_mask = os.path.join(out_dir, out_name_mask)
            else:
                out_name_mask = os.path.join(out_dir_mask, out_name_mask)

            io.imsave(out_name, patch_img, check_contrast=False)
            io.imsave(out_name_mask, patch_mask, check_contrast=False)

def crop_and_pad(img, mask, padding):
    """Crop the given image around the foreground.

    Args:
        img (numpy array): The image to crop and pad.
        mask (numpy array): The mask to use to crop with.
        padding (int): The amount of padding around the foreground to include.

    Returns: The cropped image and mask.

    """
    rows, columns = np.where(mask)

    min_row = min(rows) - padding
    max_row = max(rows) + padding

    if min_row < 0:
        min_row = 0

    if max_row > mask.shape[0]:
        max_row = mask.shape[0]

    min_col = min(columns) - padding
    max_col = max(columns) + padding

    if min_col < 0:
        min_col = 0

    if max_col > mask.shape[1]:
        max_col = mask.shape[1]

    cropped_mask = mask[min_row:max_row, min_col:max_col]
    cropped_img = img[min_row:max_row, min_col:max_col]

    return cropped_img, cropped_mask

def load_mask_paths(csv_file):
    """Load paths from the given csv file.

    Args:
        csv_file (string): The path to the csv file to read.

    Returns: A list of path names.

    """

    with open(csv_file) as csvfile:
        reader = csv.reader(csvfile)

        paths = []

        for row in reader:
            paths.append(row[0])

    return paths

def get_image_path(mask_path):
    """Get the image path from the given mask path.

    Args:
        mask_path (string): The path to the mask.

    Returns: The path to the image.

    """
    parts = os.path.split(mask_path)

    file_name = parts[-1]
    mask_path = parts[0]

    image_dir = os.path.split(mask_path)[0]
    image_dir = os.path.join(image_dir, "images")

    image_name = os.path.splitext(file_name)[0]
    image_code = image_name.split("_")[0]

    image_tif = image_code + ".tif"

    out_path = os.path.join(image_dir, image_tif)

    return out_path

def run(csv_file, out_dir, train, percent_patches=0.75, crop_padding=256,
        patch_shape=(256, 256)):

    patch_shape = (patch_shape[0], patch_shape[1])

    images = load_mask_paths(csv_file)

    for mask_path in tqdm(images, desc="Patching images..."):
        img_path = get_image_path(mask_path)

        if train:
            create_train_patch(img_path, mask_path, percent_patches, out_dir,
                               crop_padding, patch_shape=patch_shape)
        else:
            create_test_patch(img_path, mask_path, out_dir, crop_padding,
                              patch_shape=patch_shape)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Create a set of patches from"
                                     " the paths given in csv_file."
                                     "Patches will"
                                     "be stored in out_dir.")

    parser.add_argument("csv_file", help="The path to the csv_file containing"
                        " mask paths to be turned into patches.")

    parser.add_argument("out_dir", help="The path to the output directory.  If"
                        " the directory contains an images and masks dir, "
                        "patches will be stored in their respective "
                        "directories, otherwise all patches will be stored "
                        "together in the out_dir.")

    parser.add_argument("--train", action="store_true", help="Use this flag to"
                        "generate patches for training sets.")

    parser.add_argument("--test", action="store_true", help="Use this flag to"
                        "generate patches for testing and validation sets.")

    parser.add_argument("-p", "--percent_patches", type=float, help="Spceifies"
                        " the pecentage of the ratio of patches to make "
                        "relative to the total possible number of patches "
                        "for training sets.  Basically, we need a specific "
                        "number of patches to create, but after cropping "
                        "each image is a different size, so we can't always "
                        "say make 100 patches.  This is a floating point "
                        "number that says, make x percent of "
                        "the total possible number of patches we could make."
                        "Default is 0.75",
                        default=0.75)

    parser.add_argument("-c", "--crop_padding", type=int, help="The number of "
                        "pixels to add to each side of the image when cropping"
                        " around the foreground.  Default is 256",
                        default=256)

    parser.add_argument("-s", "--patch_shape", type=int, help="The x, y "
                        "dimensions of each patch.  Default is (256, 256)",
                        nargs="+",
                        default=(256, 256))

    args = parser.parse_args()

    # Error checking

    if not os.path.exists(args.csv_file):
        print("The path {} does not exist.".format(args.csv_file))
        sys.exit(1)

    if not os.path.exists(args.out_dir):
        print("The path {} does not exist.".format(args.out_dir))
        sys.exit(1)

    if not args.train and not args.test:
        print("One of --test or --train must be set.")
        sys.exit(1)

    if args.train and args.test:
        print("Only one of --test or --train can be set at a time.")
        sys.exit(1)

    if args.percent_patches > 1 or args.percent_patches <= 0:
        print("Percent patches must be between 0 and 1, and cannot be zero.")
        sys.exit(1)

    if args.crop_padding < 0:
        print("Crop padding cannot be negative.")
        sys.exit(1)

    if len(args.patch_shape) != 2:
        print("Patch shapes must be two dimensional (x, y)")
        sys.exit(1)

    if args.patch_shape[0] <= 0 or args.patch_shape[1] <= 0:
        print("Patch shapes must be positive integers.")
        sys.exit(1)

    # Call it
    if args.train:
        run(args.csv_file, args.out_dir, True,
            percent_patches=args.percent_patches,
            crop_padding=args.crop_padding, patch_shape=args.patch_shape)

    elif args.test:
        run(args.csv_file, args.out_dir, False,
            percent_patches=args.percent_patches,
            crop_padding=args.crop_padding, patch_shape=args.patch_shape)


text:
from __future__ import print_function
from __future__ import division
import numpy as np
import os.path
import codecs
from LanguageModel import LanguageModel

def softmax(mat):
	"calc softmax such that labels per time-step form probability distribution"
	# dim0=t, dim1=c
	maxT,_=mat.shape
	res=np.zeros(mat.shape)
	for t in range(maxT):
		y=mat[t,:]
		e=np.exp(y)
		s=np.sum(e)
		res[t,:]=e/s

	return res

def loadFromCSV(fn):
	"load matrix from csv file. Last entry in row terminated by semicolon."
	mat=np.genfromtxt(fn, delimiter=';')[:,:-1]
	mat=softmax(mat)
	return mat

class Data:
	"holds matrix, ground truth and filenames of a sample"
	def __init__(self):
		self.gt=''
		self.mat=None
		self.fn=''

class DataLoader:
	"load data from a given directory"
	def __init__(self, dataset, sampleEach=1):
		self.path='../data/'+dataset+'/'
		self.chars=codecs.open(self.path+'chars.txt', 'r', 'utf8').read()
		self.wordChars=codecs.open(self.path+'wordChars.txt', 'r', 'utf8').read()
		self.lm=LanguageModel(codecs.open(self.path+'corpus.txt', 'r', 'utf8').read(), self.chars, self.wordChars)
		self.mats=[]
		self.gts=[]
		self.fns=[]

		i=0
		while True:
			fnMat=self.path+'mat_'+str(i)+'.csv'
			fnGT=self.path+'gt_'+str(i)+'.txt'
			i+=1

			# file not found
			if (not os.path.isfile(fnMat)) or (not os.path.isfile(fnGT)):
				break

			# ignore this sample
			if (i-1)%sampleEach!=0:
				continue

			# put into result
			self.mats.append(fnMat)
			self.gts.append(fnGT)
			self.fns.append(fnMat+'|'+fnGT)

		self.currIdx=0

	def getNumSamples(self):
		return len(self.mats)

	def __next__(self):
		if self.currIdx>=len(self.mats):
			raise StopIteration()

		data=Data()
		data.mat=loadFromCSV(self.mats[self.currIdx])
		data.gt=codecs.open(self.gts[self.currIdx], 'r', 'utf8').read()
		data.fn=self.fns[self.currIdx]

		self.currIdx+=1
		return data

	# python2 needs next, not __next__
	next=__next__

	def __iter__(self):
		return self

max_issues_repo_name: chaitusvk/CTCWordBeamSearch

max_forks_repo_name: chaitusvk/CTCWordBeamSearch


text:
import pygame, sys
import numpy as np
import atexit
import random
import time
import math
from math import fabs

black = [0, 0, 0]
white = [255, 255, 255]
grey = [180, 180, 180]
dgrey = [120, 120, 120]
orange = [180, 100, 20]
red = [200, 0, 0]
pink = [250, 150, 150]
green = [0, 200, 0]
lgreen = [60, 250, 60]
dgreen = [0, 100, 0]
blue = [0, 0, 250]
lblue = [80, 200, 200]
brown = [140, 100, 40]
dbrown = [100, 80, 0]
gold = [230, 215, 80]
yellow = [210, 250, 80]

class TaskExecutor(object):

    def __init__(self, rows=5, cols=5, trainsessionname='test'):
        self.agent = None
        self.isAuto = True
        self.gui_visible = False
        self.userquit = False
        self.optimalPolicyUser = False  # optimal policy set by user
        self.trainsessionname = trainsessionname
        self.rows = rows
        self.cols = cols

        # Configuration
        self.pause = False  # game is paused
        self.debug = False

        self.differential = False
        self.initial_pos_x = 0
        self.initial_pos_y = 0
        self.initial_pos_th = 90

        self.sleeptime = 0.0
        self.command = 0
        self.iteration = 0
        self.score = 0
        self.numactions = 0
        self.cumreward = 0
        self.cumreward100 = 0  # cum reward for statistics
        self.cumscore100 = 0
        self.ngoalreached = 0

        self.nactionlimit = 1000
        self.ntaskactionslimit = 1000
        self.turnslimit = 10  # max consecutive turns allowed
        self.useslimit = 100  # max consecutive uses allowed

        self.hiscore = 0
        self.hireward = -1000000
        self.resfile = open("data/" + self.trainsessionname + ".dat", "a+")
        self.elapsedtime = 0  # elapsed time for this experiment

        self.win_width = 480
        self.win_height = 520

        self.size_square = 40
        self.offx = 40
        self.offy = 100
        self.radius = 5

        self.RA_exploration_enabled = False  # enable automatic options
        self.RA_visits = {}  # number of visits for each RA state
        self.RA_success = {}  # number of good transitions for each RA state

        if self.cols > 10:
            self.win_width += self.size_square * (self.cols - 10)
        if self.rows > 10:
            self.win_height += self.size_square * (self.rows - 10)

        pygame.init()

        # allows for holding of key
        # pygame.key.set_repeat(1,0)

        # self.reset()  called by game engine

        self.screen = pygame.display.set_mode([self.win_width, self.win_height])
        self.myfont = pygame.font.SysFont("Arial", 30)

        # to be set by sub-classes
        # self.locations = LOCATIONS
        # self.action_names = ACTION_NAMES
        # self.tasks = TASKS
        # self.reward_states = REWARD_STATES
        self.maxitemsheld = 1  # max number of items agent can hold

    def ntaskstates(self):
        r = 1
        for t in self.tasks.keys():
            tl = self.tasks[t]
            for l in tl:
                r *= len(l) + 1
        return r

    def init(self, agent):  # init after creation (uses args set from cli)
        if not self.gui_visible:
            pygame.display.iconify()

        # number of states
        ns = self.rows * self.cols
        if self.differential:
            ns *= 4
            self.nactionlimit *= 5
            # self.ntaskactionslimit *= 4

        # number of actions
        self.nactions = len(self.action_names)

        self.agent = agent
        ns *= self.ntaskstates()
        print('Number of states: %d' % ns)
        print('Number of actions: %d' % self.nactions)
        self.agent.init(ns, self.nactions)  # 1 for RA not used here
        self.agent.set_action_names(self.action_names)

    def setRandomSeed(self, seed):
        random.seed(seed)
        np.random.seed(seed)

    def reset(self):
        self.pos_x = self.initial_pos_x
        self.pos_y = self.initial_pos_y
        self.pos_th = self.initial_pos_th
        self.consecutive_turns = 0
        self.consecutive_uses = 0

        self.reset_tasks()
        self.has = []

        self.score = 0
        self.gamman = 1.0  # cumulative gamma over time
        self.cumreward = 0
        self.cumscore = 0

        self.current_reward = 0  # accumulate reward over all events happened during this action until next different state

        self.prev_state = None  # previous state
        self.firstAction = True  # first action of the episode
        self.finished = False  # episode finished
        self.newstate = True  # new state reached
        self.numactions = 0  # number of actions in this episode
        self.ntaskactions = 0  # number of actions for this task-part of episode
        self.iteration += 1

        self.agent.optimal = self.optimalPolicyUser or (
                self.iteration % 100) == 0  # False #(random.random() < 0.5)  # choose greedy action selection for the entire episode

        self.current_RA_state = 0
        self.last_RA_state = -1
        self.state_changed = False

        # RA exploration
        self.RA_exploration()

    def reset_tasks(self):
        # RA state of each sub-task
        self.task_state = {}
        for t in self.tasks.keys():
            tl = self.tasks[t]
            i = 0
            for l in tl:
                self.task_state[(t, i)] = 0
                i += 1
        self.actionlocation = []
        self.ntaskactions = 0
        self.taskscompleted = []

    def reset_partial_tasks(self):
        # reset state of each sub-task
        for t in self.tasks.keys():
            ltl = self.tasks[t]
            i = 0
            for tl in ltl:
                if t in self.taskscompleted:
                    self.task_state[(t, i)] = len(tl)
                elif self.task_state[(t, i)] < len(tl):
                    # print('reset task %s' %t)
                    self.task_state[(t, i)] = 0
                i += 1
        self.actionlocation = []

    def encode_task_state(self):
        r = 0
        b = 1
        for t in self.tasks.keys():
            tl = self.tasks[t]
            i = 0
            for l in tl:
                r += b * self.task_state[(t, i)]
                b *= len(l) + 1
                i += 1
        #            print('    ---  encode task state  ',t , self.task_state[t])
        #        print('    ---  encode task state final: ', r)
        return r

    def getstate(self):
        x = self.pos_x + self.cols * self.pos_y
        #        print '-----'
        #        print (self.pos_x,self.pos_y,self.pos_th/90,self.encode_task_state())
        #        print ' +++ state: ',x
        n = (self.rows * self.cols)
        if self.differential:
            x += (self.pos_th / 90) * n
            n *= 4
        x += n * self.encode_task_state()
        #        print ' +++ state: ',x
        #        print ' === state: ',x,'\n'
        return x

    def goal_reached(self):
        r = self.score == len(self.tasks.keys())
        #        print ' --- goal reached - score ', self.score
        #        if r:
        #            print ' --- goal reached!!!'
        return r

    def savedata(self):
        return [self.iteration, self.hiscore, self.hireward, self.elapsedtime,
                self.RA_visits, self.RA_success]

    def loaddata(self, data):
        self.iteration = data[0]
        self.hiscore = data[1]
        self.hireward = data[2]
        self.elapsedtime = data[3]
        self.RA_visits = data[4]
        self.RA_success = data[5]

    def itemat(self, x, y):  # which item is in this location
        r = None
        for t in self.locations:
            if t[2] == x and t[3] == y:
                r = t[0]
                break
        return r

    # check if this action progresses any task
    # check if any task finished and resets other sub-tasks
    def check_action_task(self, a, what=None):
        r = 0  # reward to return
        self.state_changed = False  # if RA state is changed
        if what != None:
            act = a + "_" + what
        else:
            act = a
        if not self.isAuto:
            print("checking action %s" % act)
        for t in self.tasks.keys():
            ltl = self.tasks[t]
            i = 0
            for tl in ltl:  # action list for this task
                ts = self.task_state[(t, i)]
                if not self.isAuto:
                    print('  -- task list: %r status: %d' % (tl, ts))
                if ts < len(tl) and act == tl[ts]:
                    if not self.isAuto:
                        print('*** good action for %s ***' % t)
                    self.actionlocation.append((self.pos_x, self.pos_y))
                    self.task_state[(t, i)] += 1  # go to next task state
                    r += self.reward_states['TaskProgress']
                    # print('self.task_state[(t, i)] = ', self.task_state[(t, i)], 'len(tl) = ', len(tl))
                    if self.task_state[(t, i)] == len(tl):
                        if not self.isAuto:
                            print("!!!Task %s completed!!!" % t)
                        self.taskscompleted.append(t)
                        r += self.reward_states['TaskComplete']
                        self.state_changed = True
                        # print("state changed")
                        self.score += 1
                        self.reset_partial_tasks()
                i += 1
        # print('   ... reward %d' %r)
        return r

    def current_successrate(self):
        s = 0.0
        v = 1.0
        if self.current_RA_state in self.RA_success:
            s = float(self.RA_success[self.current_RA_state])
        if self.current_RA_state in self.RA_visits:
            v = float(self.RA_visits[self.current_RA_state])
        # print "   -- state %d - success rate: %f / %f" %(self.current_RA_state,s,v)
        return s / v

    def RA_exploration(self):
        if not self.RA_exploration_enabled:
            return
        # update success/visit
        # print("update success/visit")
        self.current_RA_state = self.encode_task_state()
        if self.current_RA_state in self.RA_visits:
            self.RA_visits[self.current_RA_state] += 1
        else:
            self.RA_visits[self.current_RA_state] = 1

        if self.last_RA_state >= 0 and self.last_RA_state in self.RA_success:
            self.RA_success[self.last_RA_state] += 1
        else:
            self.RA_success[self.last_RA_state] = 1
        self.last_RA_state = self.current_RA_state

        # print "RA state: ",self.current_RA_state
        success_rate = max(min(self.current_successrate(), 0.9), 0.1)
        # print "RA exploration policy: current state success rate ",success_rate
        er = random.random()
        self.agent.option_enabled = (er < success_rate)
        # print "RA exploration policy: optimal ",self.agent.partialoptimal, "\n"

    def update(self, a):

        self.command = a

        self.prev_state = self.getstate()  # remember previous state

        # print " == Update start ",self.prev_state," action",self.command

        self.current_reward = 0  # accumulate reward over all events happened during this action until next different state
        # print('self.current_reward = 0')
        self.numactions += 1  # total number of actions axecuted in this episode
        self.ntaskactions += 1
        # while (self.prev_state == self.getstate()):

        if self.firstAction:
            self.firstAction = False
            self.current_reward += self.reward_states['Init']

        newposx = self.pos_x
        newposy = self.pos_y

        if not self.differential:
            if self.command == 0:  # moving left
                newposx = self.pos_x - 1
            elif self.command == 1:  # moving right
                newposx = self.pos_x + 1
            elif self.command == 2:  # moving up
                newposy = self.pos_y + 1
            elif self.command == 3:  # moving down
                newposy = self.pos_y - 1

        else:
            # differential motion
            if self.command == 0:  # turn left
                self.pos_th += 90
                if self.pos_th >= 360:
                    self.pos_th -= 360
                # print ("left")
                self.consecutive_turns += 1
                self.current_reward += self.reward_states['Turn']
            elif self.command == 1:  # turn right
                self.pos_th -= 90
                if self.pos_th < 0:
                    self.pos_th += 360
                    # print ("right")
                self.consecutive_turns += 1
                self.current_reward += self.reward_states['Turn']
            elif self.command == 2 or self.command == 3:
                dx = 0
                dy = 0
                if self.pos_th == 0:  # right
                    dx = 1
                elif self.pos_th == 90:  # up
                    dy = 1
                elif self.pos_th == 180:  # left
                    dx = -1
                elif self.pos_th == 270:  # down
                    dy = -1
                if self.command == 3:  # backward
                    dx = -dx
                    dy = -dy
                    # print ("backward")
                else:
                    # print ("forward")
                    self.current_reward += self.reward_states['Forward']
                self.consecutive_turns = 0
                self.consecutive_uses = 0
                newposx = self.pos_x + dx
                newposy = self.pos_y + dy

        if newposx < 0:
            newposx = 0
            self.current_reward += self.reward_states['Hit']
        if newposx >= self.cols:
            newposx = self.cols - 1
            self.current_reward += self.reward_states['Hit']
        if newposy >= self.rows:
            newposy = self.rows - 1
            self.current_reward += self.reward_states['Hit']
        if newposy < 0:
            newposy = 0
            self.current_reward += self.reward_states['Hit']

        if self.itemat(newposx, newposy) == 'obstacle':
            self.current_reward += self.reward_states['Hit']
        else:
            self.pos_x = newposx
            self.pos_y = newposy

        if self.command >= 4:
            r = 0
            if self.command in self.map_actionfns:
                # exec action function
                r = self.map_actionfns[self.command]()
            else:
                print('ERROR: action command %d unknown!!!' % self.command)
            self.current_reward += r
            self.consecutive_uses += 1

        self.current_reward += self.reward_states['Alive']

        # RA exploration
        if self.state_changed:  # when task completed
            self.RA_exploration()
            self.state_changed = False

        # check if episode terminated
        if self.goal_reached():
            self.current_reward += self.reward_states['Score']
            self.ngoalreached += 1
            self.finished = True

        if self.numactions > self.nactionlimit:
            self.current_reward += self.reward_states['Dead']
            self.finished = True

        # too many consecutive actions
        if (
                self.consecutive_turns > self.turnslimit or self.consecutive_uses > self.useslimit or self.ntaskactions > self.ntaskactionslimit):
            self.finished = True
            # if (self.agent.optimal):
            #    #self.finished = True
            #    pass
            # elif (self.agent.partialoptimal):
            #    self.agent.partialoptimal = False

        # print " ** Update end ",self.getstate(), " prev ",self.prev_state

    def input(self):

        self.usercommand = -1
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                return False

            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    self.usercommand = 0
                elif event.key == pygame.K_RIGHT:
                    self.usercommand = 1
                elif event.key == pygame.K_UP:
                    self.usercommand = 2
                elif event.key == pygame.K_DOWN:
                    self.usercommand = 3
                elif event.key == pygame.K_4:  # user action
                    self.usercommand = 4
                elif event.key == pygame.K_5:  # user action
                    self.usercommand = 5
                elif event.key == pygame.K_6:  # user action
                    self.usercommand = 6
                elif event.key == pygame.K_7:  # user action
                    self.usercommand = 7
                elif event.key == pygame.K_8:  # user action
                    self.usercommand = 8
                elif event.key == pygame.K_9:  # user action
                    self.usercommand = 9
                elif event.key == pygame.K_SPACE:
                    self.pause = not self.pause
                    print("Game paused: %s" % self.pause)
                elif event.key == pygame.K_a:
                    self.isAuto = not self.isAuto
                elif event.key == pygame.K_s:
                    self.sleeptime = 1.0
                elif event.key == pygame.K_d:
                    self.sleeptime = 0.05
                elif event.key == pygame.K_f:
                    self.sleeptime = 0.0
                    self.agent.debug = False
                elif event.key == pygame.K_o:
                    self.optimalPolicyUser = not self.optimalPolicyUser
                    print("Best policy: %s" % self.optimalPolicyUser)
                elif event.key == pygame.K_q:
                    self.userquit = True
                    print("User quit !!!")

        if self.usercommand >= self.nactions:
            self.usercommand = -1

        return not self.userquit

    def getUserAction(self):
        while self.usercommand < 0 and not self.isAuto and not self.userquit:
            self.input()
            time.sleep(0.2)
        if not self.isAuto:
            self.command = self.usercommand
        return self.command

    def getreward(self):
        r = self.current_reward
        self.cumreward += self.gamman * r
        self.gamman *= self.agent.gamma
        return r

    def print_report(self, printall=False):
        toprint = printall
        ch = ' '
        if self.agent.optimal:
            ch = '*'
            toprint = True

        s = 'Iter %6d, sc: %3d, na: %4d, r: %5d %c' % (self.iteration, self.score, self.numactions, self.cumreward, ch)

        if self.score > self.hiscore:
            self.hiscore = self.score
            s += ' HISCORE '
            toprint = True
        if self.cumreward > self.hireward:
            self.hireward = self.cumreward
            s += ' HIREWARD '
            toprint = True

        if toprint:
            print(s)

        self.cumreward100 += self.cumreward
        self.cumscore100 += self.score
        numiter = 100
        if self.iteration % numiter == 0:
            # self.doSave()
            pgoal = float(self.ngoalreached * 100) / numiter
            print('-----------------------------------------------------------------------')
            print("%s %6d/%4d avg last 100: reward %d | score %.2f | p goals %.1f %%" % (
                self.trainsessionname, self.iteration, self.elapsedtime, float(self.cumreward100) / 100,
                float(self.cumscore100) / 100, pgoal))
            print('-----------------------------------------------------------------------')
            self.cumreward100 = 0
            self.cumscore100 = 0
            self.ngoalreached = 0

        sys.stdout.flush()

        self.resfile.write("%d,%d,%d,%d,%d,%d,%d\n" % (
            self.iteration, self.elapsedtime, self.score, self.cumreward, self.goal_reached(), self.numactions,
            self.agent.optimal))
        self.resfile.flush()

    def draw(self):
        self.screen.fill(white)

        score_label = self.myfont.render(str(self.score), 100, pygame.color.THECOLORS['black'])
        self.screen.blit(score_label, (20, 10))

        # count_label = self.myfont.render(str(self.paddle_hit_count), 100, pygame.color.THECOLORS['brown'])
        # self.screen.blit(count_label, (70, 10))

        if self.command < self.nactions:
            x = self.getstate()
            cmd = self.action_names[self.command]
            s = '%d %s' % (x, cmd)
            count_label = self.myfont.render(s, 100, pygame.color.THECOLORS['brown'])
            self.screen.blit(count_label, (60, 10))

        sinv = ''
        for t in self.tasks.keys():
            ltl = self.tasks[t]
            i = 0
            st = '-'
            for tl in ltl:
                if self.task_state[(t, i)] == len(tl):
                    st = '*'
            sinv += st

        inv_label = self.myfont.render(sinv, 100, pygame.color.THECOLORS['blue'])
        self.screen.blit(inv_label, (200, 10))

        if self.isAuto is True:
            auto_label = self.myfont.render("Auto", 100, pygame.color.THECOLORS['red'])
            self.screen.blit(auto_label, (self.win_width - 160, 10))
        if self.agent.optimal:
            opt_label = self.myfont.render("Best", 100, pygame.color.THECOLORS['red'])
            self.screen.blit(opt_label, (self.win_width - 80, 10))
        elif self.agent.option_enabled:
            opt_label = self.myfont.render("PB", 100, pygame.color.THECOLORS['red'])
            self.screen.blit(opt_label, (self.win_width - 80, 10))

        # grid
        for i in range(0, self.cols + 1):
            ox = self.offx + i * self.size_square
            pygame.draw.line(self.screen, black, [ox, self.offy], [ox, self.offy + self.rows * self.size_square])
        for i in range(0, self.rows + 1):
            oy = self.offy + i * self.size_square
            pygame.draw.line(self.screen, black, [self.offx, oy], [self.offx + self.cols * self.size_square, oy])

        # world elements
        for t in self.locations:
            col = t[1]
            u = t[2]
            v = t[3]
            dx = int(self.offx + u * self.size_square)
            dy = int(self.offy + (self.rows - v - 1) * self.size_square)
            sqsz = (dx + 5, dy + 5, self.size_square - 10, self.size_square - 10)
            pygame.draw.rect(self.screen, col, sqsz)
            if (u, v) in self.actionlocation:
                pygame.draw.rect(self.screen, pygame.color.THECOLORS['black'],
                                 (dx + 15, dy + 15, self.size_square - 30, self.size_square - 30))

        # agent position
        dx = int(self.offx + self.pos_x * self.size_square)
        dy = int(self.offy + (self.rows - self.pos_y - 1) * self.size_square)

        pygame.draw.circle(self.screen, orange, [dx + self.size_square // 2, dy + self.size_square // 2],
                           2 * self.radius, 0)

        # agent orientation

        ox = 0
        oy = 0
        if self.pos_th == 0:  # right
            ox = self.radius
        elif self.pos_th == 90:  # up
            oy = -self.radius
        elif self.pos_th == 180:  # left
            ox = -self.radius
        elif self.pos_th == 270:  # down
            oy = self.radius

        pygame.draw.circle(self.screen, pygame.color.THECOLORS['black'],
                           [int(dx + self.size_square / 2 + ox), int(dy + self.size_square / 2 + oy)], 5, 0)

        pygame.display.update()

    def quit(self):
        self.resfile.close()
        pygame.quit()

max_stars_repo_name: amrufathy/LPOPL

max_issues_repo_name: amrufathy/LPOPL

max_forks_repo_name: amrufathy/LPOPL


text:
__copyright__ = "Copyright (C) 2009 Andreas Kloeckner"

__license__ = """
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import pyopencl as cl
import numpy as np

# {{{ timing helpers

class Timer:
    def __init__(self, queue):
        self.queue = queue

    def start(self):
        pass

    def stop(self):
        pass

    def add_event(self, evt):
        pass

    def get_elapsed(self):
        pass

class WallTimer(Timer):
    def start(self):
        from time import time
        self.queue.finish()
        self.start_time = time()

    def stop(self):
        from time import time
        self.queue.finish()
        self.end_time = time()

    def get_elapsed(self):
        return self.end_time-self.start_time

def _get_time(queue, f, timer_factory=None, desired_duration=0.1,
        warmup_rounds=3):

    if timer_factory is None:
        timer_factory = WallTimer

    count = 1

    while True:
        timer = timer_factory(queue)

        for _i in range(warmup_rounds):
            f()
        warmup_rounds = 0

        timer.start()
        for _i in range(count):
            timer.add_event(f())
        timer.stop()

        elapsed = timer.get_elapsed()
        if elapsed < desired_duration:
            if elapsed == 0:
                count *= 5
            else:
                new_count = int(desired_duration/elapsed)

                new_count = max(2*count, new_count)
                new_count = min(10*count, new_count)
                count = new_count

        else:
            return elapsed/count

# }}}

# {{{ transfer measurements

class HostDeviceTransferBase:
    def __init__(self, queue, block_size):
        self.queue = queue
        self.host_buf = np.empty(block_size, dtype=np.uint8)
        self.dev_buf = cl.Buffer(queue.context, cl.mem_flags.READ_WRITE, block_size)

class HostToDeviceTransfer(HostDeviceTransferBase):
    def do(self):
        return cl.enqueue_copy(self. queue, self.dev_buf, self.host_buf)

class DeviceToHostTransfer(HostDeviceTransferBase):
    def do(self):
        return cl.enqueue_copy(self. queue, self.host_buf, self.dev_buf)

class DeviceToDeviceTransfer:
    def __init__(self, queue, block_size):
        self.queue = queue
        mf = cl.mem_flags
        self.dev_buf_1 = cl.Buffer(queue.context, mf.READ_WRITE, block_size)
        self.dev_buf_2 = cl.Buffer(queue.context, mf.READ_WRITE, block_size)

    def do(self):
        return cl.enqueue_copy(self. queue, self.dev_buf_2, self.dev_buf_1)

def transfer_latency(queue, transfer_type, timer_factory=None):
    transfer = transfer_type(queue, 1)
    return _get_time(queue, transfer.do, timer_factory=timer_factory)

def transfer_bandwidth(queue, transfer_type, block_size, timer_factory=None):
    """Measures one-sided bandwidth."""

    transfer = transfer_type(queue, block_size)
    return block_size/_get_time(queue, transfer.do, timer_factory=timer_factory)

# }}}

def get_profiling_overhead(ctx, timer_factory=None):
    no_prof_queue = cl.CommandQueue(ctx)
    transfer = DeviceToDeviceTransfer(no_prof_queue, 1)
    no_prof_time = _get_time(no_prof_queue, transfer.do, timer_factory=timer_factory)

    prof_queue = cl.CommandQueue(ctx,
            properties=cl.command_queue_properties.PROFILING_ENABLE)
    transfer = DeviceToDeviceTransfer(prof_queue, 1)
    prof_time = _get_time(prof_queue, transfer.do, timer_factory=timer_factory)

    return prof_time - no_prof_time, prof_time

def get_empty_kernel_time(queue, timer_factory=None):
    prg = cl.Program(queue.context, """
        __kernel void empty()
        { }
        """).build()

    knl = prg.empty

    def f():
        knl(queue, (1,), None)

    return _get_time(queue, f, timer_factory=timer_factory)

def _get_full_machine_kernel_rate(queue, src, args, name="benchmark",
        timer_factory=None):
    prg = cl.Program(queue.context, src).build()

    knl = getattr(prg, name)

    dev = queue.device
    global_size = 4 * dev.max_compute_units

    def f():
        knl(queue, (global_size,), None, *args)

    rates = []
    num_dips = 0

    while True:
        elapsed = _get_time(queue, f, timer_factory=timer_factory)
        rate = global_size/elapsed
        print(global_size, rate, num_dips)

        keep_trying = not rates

        if rates and rate > 1.05*max(rates):  # big improvement
            keep_trying = True
            num_dips = 0

        if rates and rate < 0.9*max(rates) and num_dips < 3:  # big dip
            keep_trying = True
            num_dips += 1

        if keep_trying:
            global_size *= 2
            rates.append(rate)
        else:
            rates.append(rate)
            return max(rates)

def get_add_rate(queue, type="float", timer_factory=None):
    return 50*10*_get_full_machine_kernel_rate(queue, """
        typedef %(op_t)s op_t;
        __kernel void benchmark()
        {
            local op_t tgt[1024];
            op_t val = get_global_id(0);

            for (int i = 0; i < 10; ++i)
            {
                val += val; val += val; val += val; val += val; val += val;
                val += val; val += val; val += val; val += val; val += val;

                val += val; val += val; val += val; val += val; val += val;
                val += val; val += val; val += val; val += val; val += val;

                val += val; val += val; val += val; val += val; val += val;
                val += val; val += val; val += val; val += val; val += val;

                val += val; val += val; val += val; val += val; val += val;
                val += val; val += val; val += val; val += val; val += val;

                val += val; val += val; val += val; val += val; val += val;
                val += val; val += val; val += val; val += val; val += val;
            }
            tgt[get_local_id(0)] = val;
        }
        """ % dict(op_t=type), ())

# vim: foldmethod=marker:filetype=pyopencl

max_stars_repo_name: kanderso-nrel/pyopencl

max_issues_repo_name: kanderso-nrel/pyopencl

max_forks_repo_name: kanderso-nrel/pyopencl


text:
# -*- coding: utf-8 -*-

__all__ = ["kepler", "quad_solution_vector", "contact_points"]

from functools import partial

import numpy as np
from jax import core
from jax import numpy as jnp
from jax.abstract_arrays import ShapedArray
from jax.interpreters import ad, batching, xla
from jax.lib import xla_client

from . import cpu_driver

xops = xla_client.ops

for _name, _value in cpu_driver.registrations().items():
    xla_client.register_cpu_custom_call_target(_name, _value)

try:
    from . import gpu_driver
except ImportError:
    pass
else:
    for _name, _value in gpu_driver.registrations().items():
        xla_client.register_custom_call_target(_name, _value, platform="gpu")

# **********
# * KEPLER *
# **********
def kepler(M, ecc):
    r"""Solve Kepler's equation

    This op numerically evaluates the solution to Kepler's equation for given
    mean anomaly ``M`` and eccentricity ``e``:

    .. math::

        M = E - e sin(E)

    For computational efficiency this op actually returns ``cos(f)`` and
    ``sin(f)``, where ``f`` is the true anomaly, defined as

    .. math::

        f = 2\,\arctan\left(\sqrt{\frac{1 + e}{1 - e}}\,\tan\frac{E}{2}\right)

    Args:
        mean_anomaly: The mean anomaly
        eccentricity: Eccentricity, like it says on the box

    Returns:
        (cos(f), sin(f)): The cosine and sine of the true anomaly ``f``.

    """
    return kepler_prim.bind(M, ecc)

def _kepler_abstract_eval(M, ecc):
    if M.dtype != np.float64 or ecc.dtype != np.float64:
        raise ValueError("float64 precision is required")
    if M.shape != ecc.shape:
        raise ValueError("Dimension mismatch")
    out_shape = ShapedArray(M.shape, np.float64)
    return (out_shape, out_shape)

def _kepler_translation_rule(c, M, ecc):
    shapes = (c.get_shape(M), c.get_shape(ecc))
    if any(shape.element_type() != np.float64 for shape in shapes):
        raise ValueError("float64 precision is required")

    dims = tuple(shapes[0].dimensions())
    if dims != shapes[1].dimensions():
        raise ValueError("Dimension mismatch")
    N = np.prod(dims).astype(np.int32)

    order = tuple(range(len(dims) - 1, -1, -1))
    shape = xla_client.Shape.array_shape(jnp.dtype(np.float64), dims, order)

    return xops.CustomCallWithLayout(
        c,
        b"solve_kepler",
        operands=(xops.ConstantLiteral(c, N), M, ecc),
        shape_with_layout=xla_client.Shape.tuple_shape((shape, shape)),
        operand_shapes_with_layout=(
            xla_client.Shape.array_shape(jnp.dtype(jnp.int32), (), ()),
            shape,
            shape,
        ),
    )

def _kepler_gpu_translation_rule(c, M, ecc):
    shapes = (c.get_shape(M), c.get_shape(ecc))
    if any(shape.element_type() != np.float64 for shape in shapes):
        raise ValueError("float64 precision is required")

    dims = tuple(shapes[0].dimensions())
    if dims != shapes[1].dimensions():
        raise ValueError("Dimension mismatch")
    N = np.prod(dims).astype(np.int32)

    order = tuple(range(len(dims) - 1, -1, -1))
    shape = xla_client.Shape.array_shape(jnp.dtype(np.float64), dims, order)
    opaque = gpu_driver.cuda_descriptor(N)

    return xops.CustomCallWithLayout(
        c,
        b"cuda_kepler",
        operands=(M, ecc),
        shape_with_layout=xla_client.Shape.tuple_shape((shape, shape)),
        operand_shapes_with_layout=(shape, shape),
        opaque=opaque,
    )

def _kepler_jvp(args, tangents):
    M, e = args
    dM, de = tangents
    sinf, cosf = kepler(M, e)

    # Pre-compute some things
    ecosf = e * cosf
    ome2 = 1 - e ** 2

    # Propagate the derivatives
    df = 0.0
    if type(dM) is not ad.Zero:
        df += dM * (1 + ecosf) ** 2 / ome2 ** 1.5
    if type(de) is not ad.Zero:
        df += de * (2 + ecosf) * sinf / ome2

    return (sinf, cosf), (cosf * df, -sinf * df)

def _kepler_batch(args, axes):
    assert axes[0] == axes[1]
    return kepler(*args), axes

kepler_prim = core.Primitive("kepler")
kepler_prim.multiple_results = True
kepler_prim.def_impl(partial(xla.apply_primitive, kepler_prim))
kepler_prim.def_abstract_eval(_kepler_abstract_eval)
xla.backend_specific_translations["cpu"][
    kepler_prim
] = _kepler_translation_rule
xla.backend_specific_translations["gpu"][
    kepler_prim
] = _kepler_gpu_translation_rule
ad.primitive_jvps[kepler_prim] = _kepler_jvp
batching.primitive_batchers[kepler_prim] = _kepler_batch

# **********
# * STARRY *
# **********
def _base_quad_solution_vector(b, r):
    return quad_solution_vector_prim.bind(b, r)

def quad_solution_vector(b, r):
    r"""Compute the "solution vector" for a quadratic limb-darkening model

    This will return a tensor with an extra dimension of size 3 on the right
    hand side which represents the solution vector for each pair of impact
    parameter ``b`` and radius ratio ``r`` values. See `Agol+ (2020)
    <https://arxiv.org/abs/1908.03222>`_ for more details.

    Args:
        b: The impact parameter
        r: The radius ratio

    Returns:
        The solution vector with the shape ``[..., 3]``, where ``...``
        indicates the shape of ``b`` or ``r``.

    """
    return _base_quad_solution_vector(b, r)[0]

def _quad_solution_vector_abstract_eval(b, r):
    if b.dtype != np.float64 or r.dtype != np.float64:
        raise ValueError("float64 precision is required")
    if b.shape != r.shape:
        raise ValueError("Dimension mismatch")
    out_shape = ShapedArray(tuple(b.shape) + (3,), np.float64)
    return (out_shape, out_shape, out_shape)

def _quad_solution_vector_translation_rule(c, b, r):
    shapes = (c.get_shape(b), c.get_shape(r))
    if any(shape.element_type() != np.float64 for shape in shapes):
        raise ValueError("float64 precision is required")

    dims = tuple(shapes[0].dimensions())
    if dims != shapes[1].dimensions():
        raise ValueError("Dimension mismatch")
    N = np.prod(dims).astype(np.int32)

    out_shape = xla_client.Shape.array_shape(
        jnp.dtype(np.float64),
        dims + (3,),
        tuple(range(len(dims), -1, -1)),
    )

    return xops.CustomCallWithLayout(
        c,
        b"quad_solution_vector",
        operands=(xops.ConstantLiteral(c, N), b, r),
        shape_with_layout=xla_client.Shape.tuple_shape(
            (out_shape, out_shape, out_shape)
        ),
        operand_shapes_with_layout=(
            xla_client.Shape.array_shape(jnp.dtype(jnp.int32), (), ()),
            xla_client.Shape.array_shape(
                jnp.dtype(np.float64),
                dims,
                tuple(range(len(dims) - 1, -1, -1)),
            ),
            xla_client.Shape.array_shape(
                jnp.dtype(np.float64),
                dims,
                tuple(range(len(dims) - 1, -1, -1)),
            ),
        ),
    )

def _quad_solution_vector_gpu_translation_rule(c, b, r):
    shapes = (c.get_shape(b), c.get_shape(r))
    if any(shape.element_type() != np.float64 for shape in shapes):
        raise ValueError("float64 precision is required")

    dims = tuple(shapes[0].dimensions())
    if dims != shapes[1].dimensions():
        raise ValueError("Dimension mismatch")
    N = np.prod(dims).astype(np.int32)

    out_shape = xla_client.Shape.array_shape(
        jnp.dtype(np.float64),
        dims + (3,),
        tuple(range(len(dims), -1, -1)),
    )
    opaque = gpu_driver.cuda_descriptor(N)

    return xops.CustomCallWithLayout(
        c,
        b"cuda_quad_solution_vector",
        operands=(b, r),
        shape_with_layout=xla_client.Shape.tuple_shape(
            (out_shape, out_shape, out_shape)
        ),
        operand_shapes_with_layout=(
            xla_client.Shape.array_shape(
                jnp.dtype(np.float64),
                dims,
                tuple(range(len(dims) - 1, -1, -1)),
            ),
            xla_client.Shape.array_shape(
                jnp.dtype(np.float64),
                dims,
                tuple(range(len(dims) - 1, -1, -1)),
            ),
        ),
        opaque=opaque,
    )

# Note: this implementation only supports first-order differentiation and
#       returning None will cause higher order operations to fail with a
#       confusing error so there's probably a better solution. But that
#       solution is not returning ad.Zero because that makes higher-order
#       derivatives execute and spit out nonsense.
def _quad_solution_vector_jvp(args, tangents):
    b, r = args
    db, dr = tangents

    s, dsdb, dsdr = _base_quad_solution_vector(b, r)

    disconnected_b = type(db) is ad.Zero
    disconnected_r = type(dr) is ad.Zero
    if disconnected_b and disconnected_r:
        return (s, dsdb, dsdr), (ad.Zero(s.aval), None, None)

    if disconnected_r and not disconnected_b:
        ds = db[..., None] * dsdb
    elif disconnected_b and not disconnected_r:
        ds = dr[..., None] * dsdr
    else:
        ds = db[..., None] * dsdb + dr[..., None] * dsdr

    return (s, dsdb, dsdr), (ds, None, None)

def _quad_solution_vector_batch(args, axes):
    assert axes[0] == axes[1]
    return quad_solution_vector(*args), axes

quad_solution_vector_prim = core.Primitive("quad_solution_vector")
quad_solution_vector_prim.multiple_results = True
quad_solution_vector_prim.def_impl(
    partial(xla.apply_primitive, quad_solution_vector_prim)
)
quad_solution_vector_prim.def_abstract_eval(
    _quad_solution_vector_abstract_eval
)
xla.backend_specific_translations["cpu"][
    quad_solution_vector_prim
] = _quad_solution_vector_translation_rule
xla.backend_specific_translations["gpu"][
    quad_solution_vector_prim
] = _quad_solution_vector_gpu_translation_rule
ad.primitive_jvps[quad_solution_vector_prim] = _quad_solution_vector_jvp
batching.primitive_batchers[
    quad_solution_vector_prim
] = _quad_solution_vector_batch

# ******************
# * CONTACT POINTS *
# ******************
def contact_points(a, e, cosw, sinw, cosi, sini, L):
    return contact_points_prim.bind(a, e, cosw, sinw, cosi, sini, L)

def _contact_points_abstract_eval(*args):
    if any(a.dtype != np.float64 for a in args):
        raise ValueError("float64 precision is required")
    shape = args[0].shape
    if any(a.shape != shape for a in args[1:]):
        raise ValueError("Dimension mismatch")
    return (
        ShapedArray(shape, np.float64),
        ShapedArray(shape, np.float64),
        ShapedArray(shape, np.int32),
    )

def _contact_points_translation_rule(c, *args):
    shapes = list(map(c.get_shape, args))
    if any(shape.element_type() != np.float64 for shape in shapes):
        raise ValueError("float64 precision is required")

    dims = tuple(shapes[0].dimensions())
    if any(dims != s.dimensions() for s in shapes):
        raise ValueError("Dimension mismatch")
    N = np.prod(dims).astype(np.int32)

    order = tuple(range(len(dims) - 1, -1, -1))
    shape = xla_client.Shape.array_shape(jnp.dtype(np.float64), dims, order)

    return xops.CustomCallWithLayout(
        c,
        b"contact_points",
        operands=(xops.ConstantLiteral(c, N),) + args,
        shape_with_layout=xla_client.Shape.tuple_shape(
            (
                shape,
                shape,
                xla_client.Shape.array_shape(jnp.dtype(np.int32), dims, order),
            )
        ),
        operand_shapes_with_layout=(
            xla_client.Shape.array_shape(jnp.dtype(jnp.int32), (), ()),
        )
        + tuple(shape for _ in args),
    )

def _contact_points_batch(args, axes):
    assert axes[0] == axes[1]
    return contact_points(*args), axes

contact_points_prim = core.Primitive("contact_points")
contact_points_prim.multiple_results = True
contact_points_prim.def_impl(partial(xla.apply_primitive, contact_points_prim))
contact_points_prim.def_abstract_eval(_contact_points_abstract_eval)
xla.backend_specific_translations["cpu"][
    contact_points_prim
] = _contact_points_translation_rule
batching.primitive_batchers[contact_points_prim] = _contact_points_batch

max_stars_repo_name: ethankruse/exoplanet-core

max_issues_repo_name: ethankruse/exoplanet-core

max_forks_repo_name: ethankruse/exoplanet-core

reason: import numpy,from jax


text:
import numpy as np
import onnx
from onnx import numpy_helper
from boxprop import *

def boxprop(box, model):

  resources = {}

  for initial in model.graph.initializer:
    const = numpy_helper.to_array(initial)
    resources[initial.name] = const

  ignoredNodes = {'Constant', 'Reshape', 'Concat', 'Unsqueeze', 'Shape', 'Gather'}

  for node in model.graph.node:

    if node.op_type == 'Relu':
      # print('box.relu()')
      box.relu()

    elif node.op_type == 'Tanh':
      # print('box.tanh()')
      box.tanh()

    elif node.op_type == 'Conv':
      weight = None
      bias = None
      for inp in node.input:
        if inp.split('.')[-1] == 'weight':
          weight = resources[inp]
        if inp.split('.')[-1] == 'bias':
          bias = resources[inp]

      c_out = weight.shape[0]
      kernel_size = 1
      stride = 1
      padding = 0

      for attr in node.attribute:
        if attr.name == 'kernel_shape':
          kernel_size = attr.ints[0]
        elif attr.name == 'strides':
          stride = attr.ints[0]
        elif attr.name == 'pads':
          padding = attr.ints[0]
      # print('box.conv2d','weight',c_out,kernel_size[0],stride, padding,'bias')
      box.conv2d(weight,c_out,kernel_size,stride,padding,bias)

    elif node.op_type == 'ConvTranspose':
      weight = None
      for inp in node.input:
        if inp.split('.')[-1] == 'weight':
          weight = resources[inp]

      c_out = weight.shape[1]
      kernel_size = [0,0]
      stride = 1
      padding = 0

      for attr in node.attribute:
        if attr.name == 'kernel_shape':
          for i,ints in enumerate(attr.ints):
            kernel_size[i] = ints
        elif attr.name == 'strides':
          stride = attr.ints[0]
        elif attr.name == 'pads':
          padding = attr.ints[0]
      # print('box.convTranspose2d','weight', c_out, kernel_size, stride, padding)
      box.convTranspose2d(weight, c_out, kernel_size, stride, padding)

    elif node.op_type == 'BatchNormalization':
      mean = None
      var = None
      weight = None
      bias = None
      for inp in node.input:
        if inp.split('.')[-1] == 'weight':
          weight = resources[inp]
        elif inp.split('.')[-1] == 'bias':
          bias = resources[inp]
        elif inp.split('.')[-1] == 'running_mean':
          mean = resources[inp]
        elif inp.split('.')[-1] == 'running_var':
          var = resources[inp]

      eps = 0
      for attr in node.attribute:
        if attr.name == 'epsilon':
          eps = attr.f
      # print('box.batchNorm2d','mean', 'var', eps, weight, bias)
      box.batchNorm2d(mean, var, eps, weight, bias)

    elif node.op_type == 'MaxPool':
      kernel_size = 1
      for attr in node.attribute:
        if attr.name == 'kernel_shape':
          kernel_size = attr.ints[0]
      # print('box.maxpool2d',kernel_size)
      box.maxpool2d(kernel_size)

    elif node.op_type == 'Gemm':
      weight = None
      bias = None

      for inp in node.input:
        if inp.split('.')[-1] == 'weight':
          weight = resources[inp]
        if inp.split('.')[-1] == 'bias':
          bias = resources[inp]
      # print('box.linear','weight','bias')
      box.linear(weight,bias)

    elif node.op_type not in ignoredNodes:
      raise ValueError('Cannot handle layer of type ' + node.op_type)

# def main():

  # for node in model.graph.node:
  #   print(node)

#   boxprop(None, 'cifar_g.onnx')

# if __name__ == "__main__":
#     main()

max_stars_repo_name: ksarangmath/prolip

max_issues_repo_name: ksarangmath/prolip

max_forks_repo_name: ksarangmath/prolip