text:
import unittest

import numpy

from chainer.backends import cuda
from chainer import functions
from chainer import testing
from chainer.testing import attr
from chainer.utils import force_array
from chainer.utils import type_check

@testing.parameterize(*(testing.product_dict(
    [
        {'shape': (1,), 'axis': 0},
        {'shape': (2, 3, 4), 'axis': 0},
        {'shape': (2, 3, 4), 'axis': 1},
        {'shape': (2, 3, 4), 'axis': 2},
        {'shape': (2, 3, 4), 'axis': -3},
        {'shape': (2, 3, 4), 'axis': -2},
        {'shape': (2, 3, 4), 'axis': -1},
        {'shape': (2, 3, 4), 'axis': None},
    ],
    testing.product({
        'dtype': [numpy.float16, numpy.float32, numpy.float64],
        'contain_zero': [True, False],
    }),
) + testing.product({
    'shape': [(0, 3)],
    'axis': [-2, 1, None],
    'dtype': [numpy.float64],
    'contain_zero': [False],
})))
@testing.fix_random()
@testing.inject_backend_tests(
    None,
    # CPU tests
    [
        {},
    ]
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'cuda_device': [0, 1],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0', 'cuda:1'],
    })
)
class TestCumprod(testing.FunctionTestCase):

    def setUp(self):
        if self.dtype == numpy.float16:
            self.check_forward_options.update({'atol': 1e-2, 'rtol': 1e-2})
            self.check_backward_options.update({'atol': 1e-2, 'rtol': 1e-2})
            self.check_double_backward_options.update({
                'atol': 1e-1, 'rtol': 1e-1, 'eps': 0.01})
        elif self.dtype == numpy.float32:
            self.check_double_backward_options.update({
                'atol': 1e-3, 'rtol': 1e-3})

    def generate_inputs(self):
        x = numpy.random.uniform(-2, 2, self.shape).astype(self.dtype)
        if self.contain_zero:
            index = numpy.random.choice(x.size)
            x.ravel()[index] = 0
        return x,

    def forward(self, inputs, device):
        x, = inputs
        return functions.cumprod(x, axis=self.axis),

    def forward_expected(self, inputs):
        x, = inputs
        expected = numpy.cumprod(x, axis=self.axis)
        expected = force_array(expected)
        return expected,

@testing.parameterize(
    {'axis': 3},
    {'axis': -4},
)
class TestCumprodInvalidTypeAxis(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3, 4)).astype('f')

    def check_type_error(self, x):
        with self.assertRaises(type_check.InvalidType):
            functions.cumprod(x, self.axis)

    def test_type_error_cpu(self):
        self.check_type_error(self.x)

    @attr.gpu
    def test_type_error_gpu(self):
        self.check_type_error(cuda.to_gpu(self.x))

class TestCumprodInvalidTypeError(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3, 4)).astype('f')

    def test_invalid_type_axis(self):
        with self.assertRaises(TypeError):
            functions.cumprod(self.x, [0])
        with self.assertRaises(TypeError):
            functions.cumprod(self.x, (0,))

testing.run_module(__name__, __file__)

max_stars_repo_name: zaltoprofen/chainer

max_issues_repo_name: zaltoprofen/chainer

max_forks_repo_name: zaltoprofen/chainer


text:
import simplejson as json
import networkx as nx

# https://prodevsblog.com/questions/139341/method-to-save-networkx-graph-to-json-graph/
def store(G, filename):
    json.dump(dict(nodes = [[n, G.nodes[n]] for n in G.nodes()],
                   edges = [[u, v, G.edges[u, v]] for u, v in G.edges()]),
                   open(filename, 'w'), indent = 2)

def load(filename):
    G = nx.Graph()
    d = json.load(open(filename))
    for node in d['nodes']:
        label = str(node[0])
        attributes = node[1]
        color = attributes.get('color', '#000000') # default black
        (x, y) = attributes['pos']
        value = attributes.get('value', '[]')
        G.add_node(label, pos = (x, y), value = value, color = color)
    for e in d['edges']:
        v = str(e[0])
        u = str(e[1])
        G.add_edge(u, v)
    return G

ext: py

max_stars_repo_name: satuelisa/BBICA

max_issues_repo_name: satuelisa/BBICA

max_forks_repo_name: satuelisa/BBICA

reason: import networkx


text:
import os
import numpy as np

from nipype.interfaces.base import CommandLine, CommandLineInputSpec
from nipype.interfaces.base import (TraitedSpec, File, traits, InputMultiPath,isdefined)

class MorphOutput(TraitedSpec):
    out_file = File(exists=True, desc="mincmorphed image")

class MorphInput(CommandLineInputSpec):
    in_file = File(position=0, argstr="%s", mandatory=True, desc="image to mincmorph")
    out_file = File(position=1, argstr="%s", mandatory=True, desc="mincmorphed image")

    successive = traits.Str(position=2, argstr="-successive %s", mandatory=True, desc="Successive operations")

    clobber = traits.Bool(argstr="-clobber", usedefault=True, default_value=True, desc="Overwrite output file")
    verbose = traits.Bool(argstr="-verbose", usedefault=True, default_value=True, desc="Write messages indicating progress")

class MorphCommand(CommandLine):
    _cmd = "mincmorph"
    _suffix = "_mincmorph"
    input_spec = MorphInput
    output_spec = MorphOutput

    def _parse_inputs(self, skip=None):
        if skip is None:
            skip = []
        if not isdefined(self.inputs.out_file):
            self.inputs.out_file = self._gen_fname(self.inputs.in_file, suffix=self._suffix)

        return super(MorphCommand, self)._parse_inputs(skip=skip)

    def _list_outputs(self):
        outputs = self.output_spec().get()
        outputs["out_file"] = self.inputs.out_file
        return outputs

    def _gen_filename(self, name):
        if name == "out_file":
            return self._list_outputs()["out_file"]
        return None

max_stars_repo_name: niefermar/APPIAN-PET-APPIAN

max_issues_repo_name: niefermar/APPIAN-PET-APPIAN

max_forks_repo_name: niefermar/APPIAN-PET-APPIAN

reason: import numpy


text:
#!/usr/bin/env python
'''
@author Luke Campbell <luke.s.campbell at gmail dot-ish-thing com>
@file ion_functions/data/test/test_performance.py
@brief Profiling of Data Functions
'''

from nose.plugins.attrib import attr
from unittest import TestCase

import time
import numpy as np

a_deca = 10000                  # 10,000 values
a_year = 60 * 60 * 24 * 365     # a years worth of 1 Hz data
a_week = 60 * 60 * 24 * 7       # a weeks worth of 1 Hz data
a_month = 60 * 60 * 24 * 30     # a months worth of 1 Hz data
a_day = 60 * 60 * 24            # a days worth of 1 Hz data

class TimeIt(object):
    def __init__(self, results=[]):
        self.results = results

    def __enter__(self):
        self.i = time.time()

    def __exit__(self, type, value, traceback):
        v = time.time() - self.i
        self.results.append(v)

@attr('PERF')
class PerformanceTestCase(TestCase):
    def profile(self, stats, func, *args, **kwargs):
        for i in xrange(10):
            with TimeIt(stats):
                func(*args, **kwargs)
            print 'Run %i: %s' % (i, stats[i])
            if stats[i] > 10:
                raise AssertionError('Performance standard failed. Method exceeded 10 seconds')

        print 'Mean: ', np.average(np.array(stats))

max_stars_repo_name: steinermg/ion-functions

max_issues_repo_name: steinermg/ion-functions

max_forks_repo_name: steinermg/ion-functions


text:
"""An example of implementing a centralized critic with ObservationFunction.

The advantage of this approach is that it's very simple and you don't have to
change the algorithm at all -- just use callbacks and a custom model.
However, it is a bit less principled in that you have to change the agent
observation spaces to include data that is only used at train time.

See also: centralized_critic.py for an alternative approach that instead
modifies the policy to add a centralized value function.
"""

import numpy as np
from gym.spaces import Box, Dict, Discrete
import argparse

from ray import tune
from ray.rllib.agents.callbacks import DefaultCallbacks
from ray.rllib.examples.twostep_game import TwoStepGame
from ray.rllib.models import ModelCatalog
from ray.rllib.models.tf.tf_modelv2 import TFModelV2
from ray.rllib.models.tf.fcnet_v2 import FullyConnectedNetwork
from ray.rllib.policy.sample_batch import SampleBatch
from ray.rllib.utils import try_import_tf

tf = try_import_tf()

parser = argparse.ArgumentParser()
parser.add_argument("--stop", type=int, default=100000)

class CentralizedCriticModel(TFModelV2):
    """Multi-agent model that implements a centralized VF.

    It assumes the observation is a dict with 'own_obs' and 'opponent_obs', the
    former of which can be used for computing actions (i.e., decentralized
    execution), and the latter for optimization (i.e., centralized learning).

    This model has two parts:
    - An action model that looks at just 'own_obs' to compute actions
    - A value model that also looks at the 'opponent_obs' / 'opponent_action'
      to compute the value (it does this by using the 'obs_flat' tensor).
    """

    def __init__(self, obs_space, action_space, num_outputs, model_config,
                 name):
        super(CentralizedCriticModel, self).__init__(
            obs_space, action_space, num_outputs, model_config, name)

        self.action_model = FullyConnectedNetwork(
            Box(low=0, high=1, shape=(6, )),  # one-hot encoded Discrete(6)
            action_space,
            num_outputs,
            model_config,
            name + "_action")
        self.register_variables(self.action_model.variables())

        self.value_model = FullyConnectedNetwork(obs_space, action_space, 1,
                                                 model_config, name + "_vf")
        self.register_variables(self.value_model.variables())

    def forward(self, input_dict, state, seq_lens):
        self._value_out, _ = self.value_model({
            "obs": input_dict["obs_flat"]
        }, state, seq_lens)
        return self.action_model({
            "obs": input_dict["obs"]["own_obs"]
        }, state, seq_lens)

    def value_function(self):
        return tf.reshape(self._value_out, [-1])

class FillInActions(DefaultCallbacks):
    """Fills in the opponent actions info in the training batches."""

    def on_postprocess_trajectory(self, worker, episode, agent_id, policy_id,
                                  policies, postprocessed_batch,
                                  original_batches, **kwargs):
        to_update = postprocessed_batch[SampleBatch.CUR_OBS]
        other_id = 1 if agent_id == 0 else 0
        action_encoder = ModelCatalog.get_preprocessor_for_space(Discrete(2))

        # set the opponent actions into the observation
        _, opponent_batch = original_batches[other_id]
        opponent_actions = np.array([
            action_encoder.transform(a)
            for a in opponent_batch[SampleBatch.ACTIONS]
        ])
        to_update[:, -2:] = opponent_actions

def central_critic_observer(agent_obs, **kw):
    """Rewrites the agent obs to include opponent data for training."""

    new_obs = {
        0: {
            "own_obs": agent_obs[0],
            "opponent_obs": agent_obs[1],
            "opponent_action": 0,  # filled in by FillInActions
        },
        1: {
            "own_obs": agent_obs[1],
            "opponent_obs": agent_obs[0],
            "opponent_action": 0,  # filled in by FillInActions
        },
    }
    return new_obs

if __name__ == "__main__":
    args = parser.parse_args()
    ModelCatalog.register_custom_model("cc_model", CentralizedCriticModel)
    action_space = Discrete(2)
    observer_space = Dict({
        "own_obs": Discrete(6),
        # These two fields are filled in by the CentralCriticObserver, and are
        # not used for inference, only for training.
        "opponent_obs": Discrete(6),
        "opponent_action": Discrete(2),
    })
    tune.run(
        "PPO",
        stop={
            "timesteps_total": args.stop,
            "episode_reward_mean": 7.99,
        },
        config={
            "env": TwoStepGame,
            "batch_mode": "complete_episodes",
            "callbacks": FillInActions,
            "num_workers": 0,
            "multiagent": {
                "policies": {
                    "pol1": (None, observer_space, action_space, {}),
                    "pol2": (None, observer_space, action_space, {}),
                },
                "policy_mapping_fn": lambda x: "pol1" if x == 0 else "pol2",
                "observation_fn": central_critic_observer,
            },
            "model": {
                "custom_model": "cc_model",
            },
        })

max_stars_repo_name: janblumenkamp/ray

max_issues_repo_name: janblumenkamp/ray

max_forks_repo_name: janblumenkamp/ray


text:
from data_loader.data_utils import gen_batch
from utils.math_utils import evaluation, custom_loss

import tensorflow as tf
import tensorflow.keras as keras
import numpy as np
import time

def multi_pred(model, seq, batch_size, n_his, n_pred, dynamic_batch=True):
    '''
    Multi_prediction function.
    :param model: tf.keras Model.
    :param y_pred: placeholder.
    :param seq: np.ndarray, [len_seq, n_frame, n_route, C_0].
    :param batch_size: int, the size of batch.
    :param n_his: int, size of historical records for training.
    :param n_pred: list, time steps of prediction. Eg: [1,3] - print prediction evaluation at 1 and 3 time_steps.
    :param dynamic_batch: bool, whether changes the batch size in the last one if its length is less than the default.
    :return pred_array : np.ndarray, [len(seq), max(n_pred), n_route, channels].
    '''
    pred_array = np.zeros((seq.shape[0], max(n_pred), *seq.shape[2:]))
    batch_size = min(batch_size, len(seq))
    for i, data in enumerate(gen_batch(seq, batch_size, dynamic_batch=dynamic_batch)):
        # Note: use np.copy() to avoid the modification of source data.
        test = np.copy(data[:, 0:n_his, :, :])
        for j in range(max(n_pred)):
            pred = model(test).numpy()
            pred = np.reshape(pred, (test.shape[0], test.shape[2], test.shape[3]))
            test[:, 0:n_his - 1, :, :] = test[:, 1:n_his, :, :]
            test[:, n_his - 1, :, :] = pred
            pred_array[i*batch_size:(i+1)*batch_size, j, :, :] = pred
    return pred_array

def model_inference(model, inputs, batch_size, n_his, n_pred):
    '''
    Model inference function.
    :param model: tf Model.
    :param inputs: instance of class Dataset, data source for inference.
    :param batch_size: int, the size of batch.
    :param n_his: int, the length of historical records for training.
    :param n_pred: list, time steps of prediction. Eg: [1,3] - print prediction evaluation at 1 and 3 time_steps.
    :return eval_test: dict, evaluation results per data per time step.
    :return y_test: npp.ndarray, model prediction for max(n_pred) steps.
    '''
    x_test, x_stats = inputs.get_data('test'), inputs.get_stats()

    y_test = multi_pred(model, x_test, batch_size, n_his, n_pred)
    eval_test = {}
    for i in n_pred:
        eval_test[i] = []
        for j in range(y_test.shape[-1]):
            eval_test[i].append(evaluation(x_test[:, n_his+i-1, :, j], y_test[:, i-1, :, j], x_stats).tolist())
    return y_test, eval_test

def model_test(inputs, batch_size, n_his, n_pred, load_path, columns):
    '''
    Load and test saved model from the checkpoint.
    :param inputs: instance of class Dataset, data source for test.
    :param batch_size: int, the size of batch.
    :param n_his: int, the length of historical records for training.
    :param n_pred: list, time steps of prediction. Eg: [1,3] - print prediction evaluation at 1 and 3 time_steps.
    :param load_path: str, the path of loaded model.
    :param columns: list, same as args.datafiles.
    '''
    start_time = time.time()
    model = keras.models.load_model(load_path, {"custom_loss": custom_loss})
    print(f'>> Loading saved model from {load_path} ...')

    x_test, x_stats = inputs.get_data('test'), inputs.get_stats()

    y_test = multi_pred(model, x_test, batch_size, n_his, n_pred)
    eval_test = {}
    for i in n_pred:
        eval_test[i] = []
        for j in range(y_test.shape[-1]):
            eval_test[i].append(evaluation(x_test[:, n_his+i-1, :, j], y_test[:, i-1, :, j], x_stats).tolist())

    for key in eval_test.keys():
        mets = eval_test[key]
        for col, met in zip(columns, met):
            print("%s\tMAPE %.4f, MAE %.4f, RMSE %.4f, Corr %.4f, R2 %.4f" % (col, *mets))

    print(f'Model Test Time {time.time() - start_time:.3f}s')
    print('Testing model finished!')
    return y_test, eval_test


text:
import argparse
import os.path
import sys
import utils
from utils import of,tf
from sklearn.externals import joblib
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

def main():
    o_model_rf = 'randfor.sav'
    o_model_xgb = 'xgboost.sav'

    usage="\n %(prog)s -r full_path_to_file -m action -f model -o optimize [options]"

    parser = argparse.ArgumentParser(usage=usage)
    parser.add_argument("-q", action="store_true", dest="quite", default=False,
                        help="don't print status messages to stdout")

    parser.add_argument("-r", dest="filename",
                        help="full path to read_map file")
    parser.add_argument("-m", dest="action",
                        help="train, looptrain, or predict?")
    parser.add_argument("-f", dest="fitmodel",
                        help="model to train or to load, (randfor, xgboost, both)")
    parser.add_argument("-o", dest="optimize",
                        help="optimize training paramters")

    args = parser.parse_args()
    if not args.action or not args.filename or not args.fitmodel or not args.optimize:
        print("Some input is missing!")
        parser.print_help()
        sys.exit(1)

    if not os.path.exists(args.filename):
        print("Sorry, file ", args.filename, "does not exists")
        sys.exit(1)

    # Train
    if args.action == 'train' or args.action == 'looptrain':
        if args.fitmodel != "randfor" and args.fitmodel != "xgboost" and args.fitmodel != "both":
            print("\n Model ", args.fitmodel,
                  " not recognized, please choose between \"randfor\" (Random Forest), \"xgboost\" (xgboost), or \"both\" " )
            sys.exit(1)

    # Predict
    elif args.action == 'predict':

        if args.fitmodel == "randfor" or args.fitmodel == "both":
            if not os.path.exists(o_model_rf):
                print("Sorry, Random Forest model to load not found")
                sys.exit(1)
        if args.fitmodel == "xgboost" or args.fitmodel == "both":
            if not os.path.exists(o_model_xgb):
                print("Sorry, XGBoost model to load not found")
                sys.exit(1)

        if args.fitmodel != "randfor" and args.fitmodel != "xgboost" and  args.fitmodel != "both":
               if not os.path.exists(args.fitmodel):
                   print("Sorry, model to load not found", args.fitmodel)
                   sys.exit(1)

    # Wrong command
    else:
        print("I don't understand your command:", args.action, "\nPlease choose: train or predict")
        sys.exit(1)

    inputfile = args.filename
    action =  args.action
    fitmodel = args.fitmodel
    optimize = int(args.optimize)

    # minimum scaffold size
    min_size = 100000

    ############################
    ########## Train ###########
    # loop over scaffold min_size
    if action == "looptrain":
         print("Loop Training on scaffold length")

         sizes=[50000, 100000, 500000, 700000, 800000, 900000, 1000000, 1500000, 2000000]
         si=['50K', '100K', '500k', '700K', '800K', '900K', '1M', '1.5M', '2M']

         x = None; y =  None; X_train = None;
         X_test = None; y_train = None; y_test = None

         filename = fitmodel + 'loop_train.txt'
         fp =  open(filename, 'w')
         for i, min_size in enumerate(sizes):
             del x, y, X_train, X_test, y_train, y_test

             x, y = utils.read_data(inputfile, min_size, 1)

             ### Split sample in Train and Test to estimate Accuracy ###
             test_perc=0.2  # perc size of test sample
             X_train, X_test, y_train, y_test = utils.resize(x, y, test_perc)

             print(" Sample", si[i])

             if fitmodel == "randfor" or fitmodel == "both":

                 n_est = 100 # same scores as 700 ??
                 maxf = 'log2'
                 crit = 'entropy'
                 min_s_leaf = 3 #10

                 if optimize == 1:
                     n_est, maxf, crit, min_s_leaf = utils.best_randforest_cl(x, y)

                 clf_rf = utils.randforest_cl(X_train, y_train, n_est, maxf, crit, min_s_leaf)
                 #joblib.dump(clf_rf,o_model_rf)

             if fitmodel == "xgboost" or fitmodel == "both":
                 booster='gbtree'
                 eta=0.1
                 max_depth=30
                 subsample=0.5

                 if optimize == 1:
                     max_depth, subsample  = utils.best_xgboost_cl(x, y)

                 clf_xgb = utils.xgboost_cl(X_train, y_train, booster, eta, max_depth, subsample)
                 #joblib.dump(clf_xgb,o_model_xgb)

             ######################################
             ### Test Predict with chosen Model ###

             print("\n Test sample: ")
             if fitmodel == "randfor":
                 y_pred = clf_rf.predict(X_test)
                 score, pos_ok, false_neg, err_pos, neg_ok, false_pos, err_neg = utils.get_accuracy_cl(y_pred, y_test)
                 rf_scores = str(tf(score)) + " " + str(pos_ok) + " " + str(false_neg) + " " + str(of(err_pos)) + " " + str(neg_ok) + " " + str(false_pos) + " " + str(of(err_neg))
                 xgb_scores = ''
                 comb_score = ''

             elif fitmodel == "xgboost":
                 y_pred = clf_xgb.predict(X_test)
                 score, pos_ok, false_neg, err_pos, neg_ok, false_pos, err_neg = utils.get_accuracy_cl(y_pred, y_test)
                 xgb_scores = str(tf(score)) + " " + str(pos_ok) + " " + str(false_neg) + " " + str(of(err_pos)) + " " + str(neg_ok) + " " + str(false_pos) + " " + str(of(err_neg))
                 rf_scores = ''
                 comb_score = ''

             elif fitmodel == "both":
                 y_pred_rf = clf_rf.predict(X_test)
                 score, pos_ok, false_neg, err_pos, neg_ok, false_pos, err_neg = utils.get_accuracy_cl(y_pred_rf, y_test)
                 rf_scores = str(tf(score)) + " " + str(pos_ok) + " " + str(false_neg) + " " + str(of(err_pos) ) + " " + str(neg_ok) + " " + str(false_pos) + " " + str(of(err_neg))

                 y_pred_xgb = clf_xgb.predict(X_test)
                 score, pos_ok, false_neg, err_pos, neg_ok, false_pos, err_neg = utils.get_accuracy_cl(y_pred_xgb, y_test)
                 xgb_scores = str(tf(score)) + " " + str(pos_ok) + " " + str(false_neg) + " " + str(of(err_pos))  + " " + str(neg_ok) + " " + str(false_pos) + " " + str(of(err_neg))

                 # Combine models
                 y_pred = np.array(y_pred_rf) | np.array(y_pred_xgb)

                 ### Estimate Accuracy and Confusion Matrix on Test sample
                 score, pos_ok, false_neg, err_pos, neg_ok,  false_pos, err_neg = utils.get_accuracy_cl(y_pred, y_test)
                 comb_score = str(tf(score)) + " " + str(pos_ok) + " " + str(false_neg) + " " + str(of(err_pos) ) + " " + str(neg_ok) + " " + str(false_pos) + " " + str(of(err_neg))
                 #print("  Score :", (tf(tf(score))),"\n  Pos_ok:", pos_ok, "False Neg:", false_neg, " Pos error:", of(of(err_pos)) ,
                 #   "%\n  Neg_ok:", neg_ok, "False_pos:", false_pos, " Neg error:", of(of(err_neg)) ,"%")

             line = si[i] + " " + rf_scores + " " + xgb_scores + " " + comb_score + "\n"
             print(line)
             fp.write(line)

         fp.close()

         # Plot results for score and positive: how many we miss? how much contamination?
         col_names=['sample', 'score_rf', 'pos_rf', 'fneg_rf', 'epos_rf', 'neg_rf', 'fpos_rf', 'eneg_rf',
                    'score_xgb','pos_xgb', 'fneg_xgb', 'epos_xgb', 'neg_xgb', 'fpos_xgb', 'eneg_xgb',
                    'score','pos', 'fneg', 'epos', 'neg', 'fpos', 'eneg']
         df = pd.read_csv(filename, sep=" ", names =  col_names, header=None)

         fig, axes = plt.subplots(nrows=2, ncols=2)
         i=0
         j=0
         df.score_rf.plot(ax=axes[i,j], label='Random Forest', legend=True, title = "Accuracy Score on Test")
         df.score_xgb.plot(ax=axes[i,j], label='XGBoost', legend=True)
         df.score.plot(ax=axes[i,j], label='Combined', legend=True)
         j=1
         df.pos_rf.plot(ax=axes[i,j], label='Random Forest', legend=True, title = "Correct Positive")
         df.pos_xgb.plot(ax=axes[i,j], label='XGBoost', legend=True)
         df.pos.plot(ax=axes[i,j], label='Combined', legend=True)
         i=1
         j=0
         df.fneg_rf.plot(ax=axes[i,j], label='Random Forest', legend=True, title = "False Negative =  Missing")
         df.fneg_xgb.plot(ax=axes[i,j], label='XGBoost', legend=True)
         df.fneg.plot(ax=axes[i,j], label='Combined', legend=True)
         j=1
         df.fpos_rf.plot(ax=axes[i,j], label='Random Forest', legend=True, title = "False Positive = Contamination")
         df.fpos_xgb.plot(ax=axes[i,j], label='XGBoost', legend=True)
         df.fpos.plot(ax=axes[i,j], label='Combined', legend=True)

         #plt.show()
         fig.savefig(filename+'.png')
         plt.close(fig)

    elif action == "train":
         print("Training with scaffold > ", min_size)

         x, y = utils.read_data(inputfile, min_size, 1)

         ### Split sample in Train and Test to estimate Accuracy ###
         test_perc=0.2  # perc size of test sample
         X_train, X_test, y_train, y_test = utils.resize(x, y, test_perc)

         ### Pick Model, Fit and Dump to file ###
         if fitmodel == "randfor" or fitmodel == "both":

             n_est = 100 # same scores as 700 ??
             maxf = 'log2'
             crit = 'entropy'
             min_s_leaf = 3 #10

             if optimize == 1:
                 n_est, maxf, crit, min_s_leaf = utils.best_randforest_cl(x, y)
                 print(" Best xgboost")
                 #{'criterion': 'entropy', 'max_features': 'log2',
                 #     'min_samples_leaf': 10, 'n_estimators': 700}
               #for min_size=100K: {'criterion': 'gini', 'max_features': 'log2', 'min_samples_leaf': 1, 'n_estimators': 1000}

             clf_rf = utils.randforest_cl(X_train, y_train, n_est, maxf, crit, min_s_leaf)
             joblib.dump(clf_rf,o_model_rf)

         if fitmodel == "xgboost" or fitmodel == "both":
             booster='gbtree'
             eta=0.1
             max_depth=30
             subsample=0.5

             if optimize == 1:
                 print(" Best xgboost")
                 max_depth, subsample  = utils.best_xgboost_cl(x, y)
                 # {'max_depth': 30, 'subsample': 0.5} #booster, eta, not in version?
                 #for min_size=100K:
             clf_xgb = utils.xgboost_cl(X_train, y_train, booster, eta, max_depth, subsample)
             joblib.dump(clf_xgb,o_model_xgb)

         print("\n Train sample: ")
         ### Predict with chosen Model ###
         if fitmodel == "randfor":
             y_pred = clf_rf.predict(X_train)
         elif fitmodel == "xgboost":
             y_pred = clf_xgb.predict(X_train)
         elif fitmodel == "both":
             y_pred_rf = clf_rf.predict(X_train)
             print("Random Forest Accuracy")
             score, pos_ok, false_neg, err_pos, neg_ok,  false_pos, err_neg = utils.get_accuracy_cl(y_pred_rf, y_train)
             print("  Score :", (tf(score)),"\n  Pos_ok:", pos_ok, "False Neg:", false_neg, " Pos error:", of(err_pos),
                   "%\n  Neg_ok:", neg_ok, "False_pos:", false_pos, " Neg error:", of(err_neg),"%")

             y_pred_xgb = clf_xgb.predict(X_train)
             print("Xgboost Accuracy")
             score, pos_ok, false_neg, err_pos, neg_ok,  false_pos, err_neg = utils.get_accuracy_cl(y_pred_xgb, y_train)
             print("  Score :", (tf(score)),"\n  Pos_ok:", pos_ok, "False Neg:", false_neg, " Pos error:", of(err_pos),
                   "%\n  Neg_ok:", neg_ok, "False_pos:", false_pos, " Neg error:", of(err_neg),"%")

             #y_pred = y_pred_rf * y_pred_xgb
             y_pred = np.array(y_pred_rf) | np.array(y_pred_xgb)
             print("Xgboost|RandForest Accuracy")
             #[x or y for (x, y) in zip(a, b)]   wo numpy

         ### Estimate Accuracy and Confusion Matrix on Training sample ###
         score, pos_ok, false_neg, err_pos, neg_ok,  false_pos, err_neg = utils.get_accuracy_cl(y_pred, y_train)
         ### Print Score and Confusion Matrix ###
         print("  Score :", (tf(score)), " f1 score :", tf(f1score),
          "\n  Pos_ok:", pos_ok,
          "False Neg:", false_neg,
          " Pos error:", of(false_neg*100/(false_neg + pos_ok)),
          "%\n  Neg_ok:", neg_ok, "False_pos:", false_pos,
          " Neg error:", of(false_pos*100/(false_pos + neg_ok)),"%")

         print(len(y_pred), y_pred.sum())

         ######################################
         ### Test Predict with chosen Model ###

         print("\n Test sample: ")
         #print("  Tot Real Pos:", sum(y_test))
         if fitmodel == "randfor":
             y_pred = clf_rf.predict(X_test)
         elif fitmodel == "xgboost":
             y_pred = clf_xgb.predict(X_test)
         elif fitmodel == "both":
             y_pred_rf = clf_rf.predict(X_test)
             print("Random Forest Accuracy")
             #print("  Predicted pos: ", y_pred_rf.sum(), " Predicted Neg: ", len(y_pred_rf)-y_pred_rf.sum() )
             score, pos_ok, false_neg, err_pos, neg_ok,  false_pos, err_neg = utils.get_accuracy_cl(y_pred_rf, y_test)
             print("  Score :", (tf(score)),"\n  Pos_ok:", pos_ok, "False Neg:", false_neg, " Pos error:", of(err_pos),
                   "%\n  Neg_ok:", neg_ok, "False_pos:", false_pos, " Neg error:", of(err_neg),"%")
             y_pred_xgb = clf_xgb.predict(X_test)
             print("Xgboost Accuracy")
             #print("  Predicted pos: ", y_pred_xgb.sum(), " Predicted Neg: ", len(y_pred_xgb)-y_pred_xgb.sum() )
             score, pos_ok, false_neg, err_pos, neg_ok,  false_pos, err_neg = utils.get_accuracy_cl(y_pred_xgb, y_test)
             print("  Score :", (tf(score)),"\n  Pos_ok:", pos_ok, "False Neg:", false_neg, " Pos error:", of(err_pos),
                   "%\n  Neg_ok:", neg_ok, "False_pos:", false_pos, " Neg error:", of(err_neg),"%")

             # Combine models?
             #y_pred = y_pred_rf * y_pred_xgb
             y_pred = np.array(y_pred_rf) | np.array(y_pred_xgb)
             print("Xgboost|RandForest Accuracy")

         ### Estimate Accuracy and Confusion Matrix on Test sample ###
         #print("  Predicted pos: ", y_pred.sum(), " Tot Neg: ", len(y_pred)-y_pred.sum() )
         score, pos_ok, false_neg, err_pos, neg_ok,  false_pos, err_neg = utils.get_accuracy_cl(y_pred, y_test)
         print("  Score :", (tf(score)),"\n  Pos_ok:", pos_ok, "False Neg:", false_neg, " Pos error:", of(err_pos),
               "%\n  Neg_ok:", neg_ok, "False_pos:", false_pos, " Neg error:", of(err_neg),"%")

    ############################
    ######## Predict ###########

    elif action == "predict":
        print("Predicting with scaffold > ", min_size)

        x, pairs = utils.read_data(inputfile, min_size, 0)
        # Important: x and pairs must keep the same indexes: no shuffling, sorting or similar

        ### Pick and Load Model ###
        if fitmodel == "randfor":
            clf = joblib.load('randfor.sav')
            y_pred = clf.predict(x)
        elif fitmodel == "xgboost":
            clf = joblib.load('xgboost.sav')
            y_pred = clf.predict(x)

        elif fitmodel == "both":
            clf_rf = joblib.load('randfor.sav')
            y_pred_rf = clf_rf.predict(x)

            clf_xgb = joblib.load('xgboost.sav')
            y_pred_xgb = clf_xgb.predict(x)

            y_pred = np.array(y_pred_rf) | np.array(y_pred_xgb)

        else:
            clf = joblib.load(fitmodel)
            y_pred = clf.predict(x)
            fitmodel = os.path.split(fitmodel)[1]

        ### Dump prediction to file ###
        o_pairs ='real_pairs_' + fitmodel + '.txt'
        fp =  open(o_pairs, 'w')
        for i,j in enumerate(y_pred):
            ### Dump to File if it's predicted to be a Real Connection ###
            if j == 1:
                fp.write(pairs[i,0]+" "+pairs[i,1]+"\n")

        fp.close()

if __name__ == "__main__":
    main()


text:
#!/usr/bin/env python

"""Train MNIST classifier."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import mnist

import numpy as np
import tensorflow as tf

# training specific hyperparameters
batch_size = 128
epochs = 1

# Load the data, shuffled and split between train and test sets
data = mnist.load_data({'dataset': {}})
x_train = data['x_train']
y_train = data['y_train']
x_test = data['x_test']
y_test = data['y_test']

feature_columns = [tf.contrib.layers.real_valued_column("", dimension=1024)]

# Build 3 layer DNN with 10, 20, 10 units respectively.
classifier = tf.contrib.learn.DNNClassifier(feature_columns=feature_columns,
                                            hidden_units=[10, 20, 10],
                                            n_classes=3,
                                            model_dir="/tmp/iris_model")

# Define the training inputs
def get_train_inputs():
    x = tf.constant(x_train)
    y = tf.constant(y_train)

    return x, y

# Fit model.
classifier.fit(input_fn=get_train_inputs, steps=2000)

# Define the test inputs
def get_test_inputs():
    x = tf.constant(x_test)
    y = tf.constant(y_test)

    return x, y

# Evaluate accuracy.
accuracy_score = classifier.evaluate(input_fn=get_test_inputs,
                                     steps=1)["accuracy"]

print("\nTest Accuracy: {0:f}\n".format(accuracy_score))

predictions = list(classifier.predict(input_fn=x_test[:2]))

print(
    "New Samples, Class Predictions:    {}\n"
    .format(predictions))

max_stars_repo_name: PepSalehi/algorithms

max_issues_repo_name: PepSalehi/algorithms

max_forks_repo_name: PepSalehi/algorithms


text:
from bispy.saha.saha import saha as saha_algorithm
from bispy.utilities.graph_normalization import (
    check_normal_integer_graph,
    convert_to_integer_graph,
    back_to_original,
)
from bispy.utilities.graph_decorator import decorate_nx_graph, to_tuple_list
from bispy.paige_tarjan.paige_tarjan import paige_tarjan_qblocks
import networkx as nx
from typing import Union, List, Dict, Any, Tuple
from bispy.utilities.graph_entities import _QBlock, _Vertex

class SahaPartition:
    """
    A convenient class to recompute incrementally the maximum bisimulation
    using *Saha*'s algorithm. Create instances using :meth:`saha`.

    :param qblocks: The current partition of the nodes of the graph.
    :param qblocks: Nodes in the graph.
    :param node_to_idx: A `dict` which maps nodes from the original graph
        to nodes of the isomorphic integer graph (see
        :mod:`bispy.utilities.graph_normalization`).
    """

    def __init__(
        self,
        qblocks: List[_QBlock],
        vertexes: List[_QBlock],
        node_to_idx: Dict[Any, int],
    ):
        self.qblocks = qblocks
        self.vertexes = vertexes
        self.node_to_idx = node_to_idx

    def add_edge(
        self, edge: Tuple[Any, Any], verbose=True
    ) -> Union[None, List[Tuple[Any]]]:
        """Add a new edge to the graph, and recompute its maximum bisimulation
        incrementally.

        :param edge: The new edge (the first item is the source, the second
            item is the destination).
        :param verbose: If `True`, this methods returns the new maximum
            bisimulation after the addition of the new edge. Disabling this
            feature may increase performance (an additive factor
            :math:`\\Theta(|V|)` is saved). Defaults to True.
        :return: The maximim bisimulation after the addition of the new edge
            if `verbose` is `True`.
        """

        # if the original graph was not integer the user is going to expect
        # to be able to insert a new edge mentioning the original nodes
        if self.node_to_idx is not None:
            edge = (self.node_to_idx[edge[0]], self.node_to_idx[edge[1]])

        self.qblocks = saha_algorithm(self.qblocks, self.vertexes, edge)
        if verbose:
            max_bisi = to_tuple_list(self.qblocks)
            if self.node_to_idx is None:
                return max_bisi
            else:
                return back_to_original(max_bisi, self.node_to_idx)

    def add_edges(
        self, edges: List[Tuple[Any, Any]], verbose: bool = True
    ) -> Union[None, List[Tuple[Any]]]:
        """Add multiple edges and compute the maximum bisimulation
        incrementally.

        :param edges: A list of edges, represented by 2-tuples.
        :param verbose: If `True`, the maximum bisimulation will be returned
            after the last iteration. Otherwise the method returns `None`.
            Defaults to `True`.
        :return: The maximim bisimulation after the addition of the last edge
            if `verbose` is `True`.
        """

        for idx, edge in enumerate(edges):
            if idx == len(edges) - 1 and verbose:
                return self.add_edge(edge)
            else:
                self.add_edge(edge, verbose=False)

def saha(
    graph, initial_partition=None, is_integer_graph=False
) -> SahaPartition:
    """
    Returns an instance of the class :class:`SahaPartition` which can be used
    to recompute the maximum bisimulation incrementally.

    :param graph: The initial graph.
    :initial_partition: The initial partition, or labeling set. This is
        **not** the partition from which we start, but an indication of which
        nodes cannot be bisimilar. Defaultsto `None`, in which case the trivial
        labeling set (one block which contains all the nodes) is used.
    :param is_integer_graph: If `True`, the function assumes that
        the graph is integer, and skips the integer check (may slightly
        improve performance). Defaults to `False`.
    """

    if not isinstance(graph, nx.DiGraph):
        raise Exception("graph should be a directed graph (nx.DiGraph)")

    # if True, the input graph is already an integer graph
    original_graph_is_integer = is_integer_graph or check_normal_integer_graph(
        graph
    )
    if not original_graph_is_integer:
        # convert the graph to an "integer" graph
        integer_graph, node_to_idx = convert_to_integer_graph(graph)

        if initial_partition is not None:
            # convert the initial partition to a integer partition
            integer_initial_partition = [
                [node_to_idx[old_node] for old_node in block]
                for block in initial_partition
            ]
        else:
            integer_initial_partition = None
    else:
        integer_graph = graph
        integer_initial_partition = initial_partition
        node_to_idx = None

    vertexes, q_partition = decorate_nx_graph(
        integer_graph,
        integer_initial_partition,
    )

    # compute the current maximum bisimulation
    q_partition = paige_tarjan_qblocks(q_partition)
    return SahaPartition(q_partition, vertexes, node_to_idx)

max_stars_repo_name: fAndreuzzi/BisPy

max_issues_repo_name: fAndreuzzi/BisPy

max_forks_repo_name: fAndreuzzi/Bisimulation-Algorithms


text:
def plotCurvaNivel(x_inicial=(), x_final=(), y_inicial=(), y_final=(), n=()):
    from numpy import sin, sqrt
    import numpy as np
    import matplotlib.pyplot as plt

    xlist = np.linspace(x_inicial, x_final, n)
    ylist = np.linspace(y_inicial, y_final, n)
    X, Y = np.meshgrid(xlist, ylist)
    Z = 0.5 - ((sin(sqrt(X**2 + Y**2)))**2 - 0.5) / (1 + 0.001 * (X**2 + Y**2))**2

    plt.figure()
    cp = plt.contourf(X, Y, Z)

    return cp

# import matplotlib.pyplot as plt
# cp = plotCurvaNivel(x_inicial=(-5), x_final=(5), y_inicial=(-5), y_final=(5), n=(1000))
# plt.colorbar(cp)
# plt.title('Curvas de nível da função F6(x, y)')
# plt.xlabel('x')
# plt.ylabel('y')
# plt.show()

reason: import numpy,from numpy


text:
import os
import ffmpeg
import numpy as np
# from spleeter import *
# from spleeter.audio.adapter import get_default_audio_adapter
# from spleeter.separator import Separator
# from spleeter.utils import *
from django.conf import settings
from .models import ProcessedTrack

class SpleeterSeparator:
    """Performs source separation using Spleeter API."""
    def __init__(self, config=None):
        """Default constructor.

        :param config: Separator config, defaults to None
        """
        if config is None:
            self.audio_bitrate = '256k'
            self.audio_format = 'wav'
            self.sample_rate = 44100
            self.spleeter_stem = 'config/4stems-16kHz.json'
        else:
            self.audio_bitrate = config['audio_bitrate']
            self.audio_format = config['audio_format']
            self.sample_rate = config['sample_rate']
            self.spleeter_stem = config['spleeter_stem']
        # Use librosa backend as it is less memory intensive
        self.separator = Separator(self.spleeter_stem, stft_backend='librosa', multiprocess=False)
        self.audio_adapter = get_default_audio_adapter()

    def separate(self, parts, input_path, dir_name, file_name):
        """Performs source separation by adding together the parts to be kept.

        :param parts: List of parts to keep ('vocals', 'drums', 'bass', 'other')
        :param input_path: Path to source file
        :param output_path: Path to output file
        :raises e: FFMPEG error
        """
        waveform, _ = self.audio_adapter.load(input_path, sample_rate=self.sample_rate)
        prediction = self.separator.separate(waveform)
        out = np.zeros_like(prediction['vocals'])
        part_count = 0

        ret = {}

        # Add up parts that were requested
        for key in prediction:
            dir = os.path.join(dir_name, key+".mp3")
            self.audio_adapter.save(os.path.join(settings.MEDIA_ROOT, dir), prediction[key], self.separator._sample_rate, self.audio_format, self.audio_bitrate)
            ret[key] = dir
            if parts[key]:
                out += prediction[key]
                part_count += 1
        out /= part_count
        req_path = os.path.join(dir_name, file_name)
        self.audio_adapter.save(os.path.join(settings.MEDIA_ROOT, req_path), out, self.separator._sample_rate, self.audio_format, self.audio_bitrate)
        ret["req"] = req_path
        return ret

    def cached(self, parts, source_track, dir_name, file_name):
        bass_path = ProcessedTrack.objects.filter(source_track=source_track,    bass=True,  vocals=False, other=False, drums=False).first().file.name
        vocals_path = ProcessedTrack.objects.filter(source_track=source_track,  bass=False, vocals=True,  other=False, drums=False).first().file.name
        other_path = ProcessedTrack.objects.filter(source_track=source_track,   bass=False, vocals=False, other=True,  drums=False).first().file.name
        drums_path = ProcessedTrack.objects.filter(source_track=source_track,    bass=False, vocals=False, other=False, drums=True).first().file.name

        ret = {
            "bass": bass_path,
            "vocals": vocals_path,
            "other": other_path,
            "drums": drums_path,
        }
        load_paths = {k:ret[k] for k in parts if parts[k]}
        arrs = [self.audio_adapter.load(os.path.join(settings.MEDIA_ROOT, p))[0] \
            for p in load_paths.values()]
        out = sum(arrs) / len(arrs)
        req_path = os.path.join(dir_name, file_name)
        self.audio_adapter.save(os.path.join(settings.MEDIA_ROOT, req_path), out, self.separator._sample_rate, self.audio_format, self.audio_bitrate)
        ret["req"] = req_path
        return ret


text:
import h5py
import torch
import torchvision
import torchvision.transforms.transforms as transforms
import torchvision.datasets as datasets
import numpy as np

def store_many_hdf5(images, labels, name):
    """ Stores an array of images to HDF5.
        Parameters:
        ---------------
        images       images array, (N, 32, 32, 3) to be stored
        labels       labels array, (N, 1) to be stored
    """
    num_images = len(images)

    # Create a new HDF5 file
    file = h5py.File(f"{name}.h5", "w")

    # Create a dataset in the file
    dataset = file.create_dataset(
        "images", data=images
    )
    meta_set = file.create_dataset(
        "meta", data=labels
    )
    file.close()

def read_many_hdf5(name):
    """ Reads image from HDF5.
        Parameters:
        ---------------
        num_images   number of images to read

        Returns:
        ----------
        images      images array, (N, 32, 32, 3) to be stored
        labels      associated meta data, int label (N, 1)
    """
    images, labels = [], []

    # Open the HDF5 file
    file = h5py.File(f"{name}.h5", "r+")

    images = np.array(file["/images"])
    labels = np.array(file["/meta"])

    return images, labels

def generate_data(data_folder, filename):
    data_folder = 'D:\CharsFont\Bmp'
    transform = {
        'train': transforms.Compose(
            [transforms.Resize([28, 28]),
             transforms.Grayscale(num_output_channels=1),
             transforms.ToTensor(),
             transforms.Normalize((0.1307,), (0.3081,))]),

        'test': transforms.Compose(
            [transforms.Resize([28, 28]),
             transforms.Grayscale(num_output_channels=1),
             transforms.ToTensor(),
             transforms.Normalize((0.1307,), (0.3081,))])
    }
    data = datasets.ImageFolder(root = data_folder, transform=transform['train'])
    data_loader = torch.utils.data.DataLoader(
        data, batch_size=len(data), shuffle=True)

    test_dataset_array = next(iter(data_loader))
    images = test_dataset_array[0].numpy()
    labels = test_dataset_array[1].numpy()

    store_many_hdf5(images, labels, filename)

max_stars_repo_name: PetrovAlexey/CapsNet

max_issues_repo_name: PetrovAlexey/CapsNet

max_forks_repo_name: PetrovAlexey/CapsNet


text:
#-*- coding: utf-8 -*-

import numpy as np

# ############ #
# Abstractions #
# ############ #

# Losses
class Loss:

    #Calcule la valeur du loss étant données les valeurs prédites et désirées
    def getLossValue(self,predicted_output,desired_output):
        pass

    #Calcule le gradient (pour chaque cellule d'entrée) du coût
    def backward(self, predicted_output,desired_output):
        pass

# Module
class Module:

    #Permet le calcul de la sortie du module
    def forward(self,input):
        pass

    #Permet le calcul du gradient des cellules d'entrée
    def backward_delta(self,input,delta_module_suivant):
        pass

    #Permet d'initialiser le gradient du module
    def init_gradient(self):
        pass

    #Permet la mise à jour des parmaètres du module avcec la valeur courante di gradient
    def update_parameters(self,gradient_step):
        pass

    #Permet de mettre à jour la valeur courante du gradient par addition
    def backward_update_gradient(self,input,delta_module_suivant):
        pass

    #Permet de faire les deux backwar simultanément
    def backward(self,input,delta_module_suivant):
        self.backward_update_gradient(input,delta_module_suivant)
        return self.backward_delte(input,delta_module_suivant)

    #Retourne les paramètres du module
    def get_parameters(self):
        pass

    #Initialize aléatoirement les paramètres du module
    def randomize_parameters(self, variance):
        pass

# ############## #
# Implémentation #
# ############## #

#########LOSSES

#Square Loss
class SquareLoss(Loss):
    def getLossValue(self,predicted_output,desired_output):
        return np.power(desired_output-predicted_output,2)

    def backward(self, predicted_output,desired_output):
        return 2*(predicted_output-desired_output)

#HingeLoss

class HingeLoss(Loss):
    def getLossValue(self,predicted_output,desired_output):
        return np.max(np.zeros(predicted_output.size), -desired_output*predicted_output)

    def backward(self, predicted_output,desired_output):
        res = np.zeros(desired_output.size)
        prod = -desired_output*predicted_output
        index = np.where(prod >=0 )
        res[index] = -desired_output[index]

        return res

#########Modules

#Module linéaire
#In => [Vector] Out <V.Parameters>
class LinearModule(Module):

    #Permet le calcul de la sortie du module
    def __init__(self,entry_size,layer_size):
        self.entry_size = entry_size
        self.layer_size = layer_size
        self.init_gradient()
        self.randomize_parameters()

    def forward(self,input):
        return np.dot(self.parameters,input)

    #Permet le calcul du gradient des cellules d'entrée
    def backward_delta(self,input,delta_module_suivant):
        return np.sum((delta_module_suivant*self.parameters.T).T,axis=0)

    #Permet d'initialiser le gradient du module
    def init_gradient(self):
        self.gradient = np.zeros((self.layer_size,self.entry_size))
        return

    #Permet la mise à jour des parmaètres du module avcec la valeur courante di gradient
    def update_parameters(self,gradient_step):
        self.parameters -= self.gradient*gradient_step
        self.gradient = np.zeros((self.layer_size,self.entry_size))
        return

    #Permet de mettre à jour la valeur courante du gradient par addition
    # def backward_update_gradient(self,input,delta_module_suivant):
    #     newGrad = np.zeros((self.layer_size,self.entry_size))
    #     for i in xrange(0,self.layer_size):
    #         di = delta_module_suivant[i]
    #         newGrad[i,:] =  di*input

    #     self.gradient += newGrad
    #     return

    def backward_update_gradient(self,input,delta_module_suivant):
        self.gradient += np.tile(input,(self.layer_size,1)) * np.reshape(delta_module_suivant,(self.layer_size,1))

    #Permet de faire les deux backwar simultanément
    def backward(self,input,delta_module_suivant):
        self.backward_update_gradient(input,delta_module_suivant)
        return self.backward_delta(input,delta_module_suivant)

    #Retourne les paramètres du module
    def get_parameters(self):
        return self.parameters

    #Initialize aléatoirement les paramètres du module
    def randomize_parameters(self):
        self.parameters = np.random.randn(self.layer_size,self.entry_size)
        return

# Tanh Activation Function
class TanhModule(Module):

    #Permet le calcul de la sortie du module
    def __init__(self,entry_size,layer_size):
        pass

    def forward(self,input):
        return np.tanh(input)

    #Permet le calcul du gradient des cellules d'entrée
    def backward_delta(self,input,delta_module_suivant):
        return (1-np.power(np.tanh(input),2))*delta_module_suivant

    #Permet d'initialiser le gradient du module
    def init_gradient(self):
        pass

    #Permet la mise à jour des parmaètres du module avcec la valeur courante di gradient
    def update_parameters(self,gradient_step):
        pass

    #Permet de mettre à jour la valeur courante du gradient par addition
    def backward_update_gradient(self,input,delta_module_suivant):
        pass

    #Permet de faire les deux backwar simultanément
    def backward(self,input,delta_module_suivant):
        return self.backward_delta(input,delta_module_suivant)

    #Retourne les paramètres du module
    def get_parameters(self):
        pass

    #Initialize aléatoirement les paramètres du module
    def randomize_parameters(self):
        pass

# Logistic Activation Function
class LogisticModule(Module):

    #Permet le calcul de la sortie du module
    def __init__(self,entry_size,layer_size):
        pass

    def forward(self,input):
        return np.power((1-np.exp(-1*input)),-1)

    #Permet le calcul du gradient des cellules d'entrée
    def backward_delta(self,input,delta_module_suivant):
        return self.forward(input)*(1-self.forward(input))*delta_module_suivant

    #Permet d'initialiser le gradient du module
    def init_gradient(self):
        pass

    #Permet la mise à jour des parmaètres du module avcec la valeur courante di gradient
    def update_parameters(self,gradient_step):
        pass
    #Permet de mettre à jour la valeur courante du gradient par addition
    def backward_update_gradient(self,input,delta_module_suivant):
        pass

    #Permet de faire les deux backwar simultanément
    def backward(self,input,delta_module_suivant):
        return self.backward_delta(input,delta_module_suivant)

    #Retourne les paramètres du module
    def get_parameters(self):
        pass

    #Initialize aléatoirement les paramètres du module
    def randomize_parameters(self):
        pass

class DropoutModule(Module):
    #Permet le calcul de la sortie du module
    def __init__(self,entry_size,layer_size):
        pass

    def forward(self,input):
        self.randomActivation = np.random.random_integers(0,1,len(input))
        return input*self.randomActivation

    #Permet le calcul du gradient des cellules d'entrée
    def backward_delta(self,input,delta_module_suivant):
        return delta_module_suivant

    #Permet d'initialiser le gradient du module
    def init_gradient(self):
        pass

    #Permet la mise à jour des parmaètres du module avcec la valeur courante di gradient
    def update_parameters(self,gradient_step):
        pass
    #Permet de mettre à jour la valeur courante du gradient par addition
    def backward_update_gradient(self,input,delta_module_suivant):
        pass

    #Permet de faire les deux backwar simultanément
    def backward(self,input,delta_module_suivant):
        return self.backward_delta(input,delta_module_suivant)

    #Retourne les paramètres du module
    def get_parameters(self):
        pass

    #Initialize aléatoirement les paramètres du module
    def randomize_parameters(self):
        pass

#########################
# AGGREGATION FUNCTION  #
#########################
class sumAggregModule(Module):
    #Permet le calcul de la sortie du module
    def __init__(self,entry_size,layer_size):
        pass

    def forward(self,input):
        return np.sum(input,axis=0)

    #Permet le calcul du gradient des cellules d'entrée
    def backward_delta(self,input,delta_module_suivant):
        return delta_module_suivant

    #Permet d'initialiser le gradient du module
    def init_gradient(self):
        pass

    #Permet la mise à jour des parmaètres du module avcec la valeur courante di gradient
    def update_parameters(self,gradient_step):
        pass
    #Permet de mettre à jour la valeur courante du gradient par addition
    def backward_update_gradient(self,input,delta_module_suivant):
        pass

    #Permet de faire les deux backwar simultanément
    def backward(self,input,delta_module_suivant):
        return self.backward_delta(input,delta_module_suivant)

    #Retourne les paramètres du module
    def get_parameters(self):
        pass

    #Initialize aléatoirement les paramètres du module
    def randomize_parameters(self):
        pass

#########################
# MULTI MODULES CLASSES #
#########################

#Network Module Class
class NetworkModule():

    def __init__(self,modules,loss):
        self.modules = modules
        self.loss = loss

    def forwardIteration(self,input):
        for module in self.modules:
            input = module.forward_all(input)
        return input

    def forwardAll(self,examples):
        return [self.forwardIteration(example) for example in examples]

    #Permet le calcul du gradient des cellules d'entrée
    def backwardIteration(self,predicted,wanted,batch=False,gradient_step=0.001):
        loss_delta = self.loss.backward(predicted,wanted)

        for mod in reversed(self.modules):
            loss_delta = mod.backward_all(loss_delta)

            if not batch:
                self.update_parameters(gradient_step)

        return loss_delta

    def update_parameters(self,gradient_step):
        for module in self.modules:
            module.update_all_parameters(gradient_step)
        return

    def stochasticIter(self,examples,labels,gradient_step=0.001, verbose=False):
        for example, label in zip(examples,labels):
            pred = self.forwardIteration(example)
            loss = self.backwardIteration(pred,label,gradient_step=gradient_step)

            if verbose:
                print loss
        return loss

    def batchIter(self,examples,labels,gradient_step=0.001, verbose=False):
        for example, label in zip(examples,labels):
            pred = self.forwardIteration(example)
            loss = self.backwardIteration(pred,label,batch=True,gradient_step=gradient_step)

            if verbose:
                print loss
        self.update_parameters(gradient_step)
        return

    def miniBatchIter(self,examples,labels,batch_size=10, gradient_step=0.001, verbose=False):
        for i, (example, label) in enumerate(zip(examples,labels)):
            pred = self.forwardIteration(example)
            loss = self.backwardIteration(pred,label,batch=True,gradient_step=gradient_step)

            if verbose:
                print loss
            if i%batch_size == 0:
                self.update_parameters(gradient_step)
        self.update_parameters(gradient_step)
        return

    #Quick utility function works for 1/-1 labels only
    def trainTest(self,trainV,trainL,testV,testL,nbIter,gradient_step):
        print("=======TRAIN ERROR=======")
        for i in xrange(0,nbIter):
            self.stochasticIter(trainV, trainL, gradient_step, verbose=False)
            predicted = self.forwardAll(trainV)
            ok=0
            ko=0
            for pred,exp in zip(predicted,trainL):
                if pred[0]*exp[0] > 0:
                    ok+=1
                else:
                    ko+=1

            print ("%d correct (%f%%), %d incorrect (%f%%) " % (ok,ok/(ok+ko+0.0)*100,ko,ko/(ok+ko+0.0)*100))
            print ("Learning done")

        print ("=======TEST ERROR=======")
        predicted = self.forwardAll(testV)

        ok=0
        ko=0

        for pred,exp in zip(predicted,testL):
            if pred[0]*exp[0] > 0:
                ok+=1
            else:
                ko+=1
        print ("%d correct (%f%%), %d incorrect (%f%%) " % (ok,ok/(ok+ko+0.0)*100,ko,ko/(ok+ko+0.0)*100))

#Horizontal Module Class
class HorizontalModule():

    def __init__(self,modules):
        self.modules = modules

    def forward_all(self,input):
        self.inputs = []
        for module in self.modules:
            self.inputs.append(input)
            input = module.forward(input)
        return input

    #Permet le calcul du gradient des cellules d'entrée
    def backward_all(self,loss_delta):
        for module,input in zip(reversed(self.modules),reversed(self.inputs)):
            loss_delta = module.backward(input,loss_delta)

        return loss_delta

    def update_all_parameters(self,gradient_step):
        for module in self.modules:
            module.update_parameters(gradient_step)
        return

#Vertical Module Class
class VerticalModule():

    #Permet le calcul de la sortie du module
    def __init__(self,HModules,aggreg):
        self.modules = HModules
        self.aggreg = aggreg

    def forward_all(self,input):

        if len(input) < len(self.HModules):
            raise "Not enough input in vertical module"

        self.outputs = [ module.forward(input[index]) for index , module in enumerate(self.modules) ]

        return aggreg.forward(self.outputs)

    #Permet le calcul du gradient des cellules d'entrée
    def backward_all(self,delta):

        for module,output in zip(self.modules,self.output):
            module_delta = aggreg.backward_delta(output,delta)
            module.backward()

            if not batch:
                module.update_parameters(gradient_step)

        return loss_delta

    def update_all_parameters(self,gradient_step):
        for module in self.modules:
            module.update_parameters(gradient_step)
        return

max_stars_repo_name: cedias/NNPy

max_issues_repo_name: cedias/NNPy

max_forks_repo_name: cedias/NNPy


text:
import argparse
import json
import pprint
import collections
import os
import sys
import tensorflow as tf
import subprocess
from PIL import Image
import numpy as np
import io
import hashlib
import tqdm
import itertools
from google.cloud import storage
from tempfile import TemporaryDirectory
from multiprocessing import Pool

sys.path.append('../../../')

# tokenizer for inputs
from utils.encode.encoder import get_encoder

_tokenizer = get_encoder()

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('input_story_json')
    parser.add_argument('image_directory')
    parser.add_argument('--num_folds', default=64, type=int)
    parser.add_argument('--num_threads', default=8, type=int)
    parser.add_argument('--pad_to_batch_size', default=32, type=int)
    parser.add_argument('--just_one_perm', default=0, type=int, help='should we save just one permutation')
    parser.add_argument('--save_dir', default='', type=str, help='Directory to save files to')
    args = parser.parse_args()
    args.mode = args.input_story_json.split('/')[-1].split('.')[0].split('_')[0]
    return args

class GCSTFRecordWriter(object):
    def __init__(self, fn):
        self.fn = fn
        if fn.startswith('gs://'):
            self.s3client = None
            self.gclient = storage.Client()
            self.storage_dir = TemporaryDirectory()
            self.writer = tf.python_io.TFRecordWriter(os.path.join(self.storage_dir.name, 'temp.tfrecord'))
            self.bucket_name, self.file_name = self.fn.split('gs://', 1)[1].split('/', 1)

        else:
            self.s3client = None
            self.gclient = None
            self.bucket_name = None
            self.file_name = None
            self.storage_dir = None
            self.writer = tf.python_io.TFRecordWriter(fn)

    def write(self, x):
        self.writer.write(x)

    def close(self):
        self.writer.close()
        if self.gclient is not None:
            bucket = self.gclient.get_bucket(self.bucket_name)
            blob = bucket.blob(self.file_name)
            blob.upload_from_filename(os.path.join(self.storage_dir.name, 'temp.tfrecord'))
            self.storage_dir.cleanup()

    def __enter__(self):
        # Called when entering "with" context.
        return self

    def __exit__(self, *_):
        # Called when exiting "with" context.
        # Upload shit
        print("CALLING CLOSE")
        self.close()

def int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def int64_list_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def bytes_list_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))

def float_feature(value):
    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))

def float_list_feature(value):
    return tf.train.Feature(float_list=tf.train.FloatList(value=value))

from io import BytesIO
def convert_PIL_to_jpgstring(image):
    """
    :param image: Numpy array of an image [H, W, 3]
    :return: it, as a jpg string
    """
    if image.mode == "RGBA":
        image = image.convert("RGB")
    with BytesIO() as output:
        image.save(output, format='JPEG', quality=95)
        return output.getvalue()

def process(fname, image_directory, writer, fold, args):
    with open(fname) as f:
        data = json.load(f)

    # we need to make a flattened list of stories

    storyid2anns = collections.defaultdict(list)
    for ann in data['annotations']:
        ann = ann[0]
        storyid2anns[int(ann['story_id'])].append(ann)

    for k in storyid2anns.keys():
        assert len(storyid2anns[k]) == 5
        assert set([ann['worker_arranged_photo_order'] for ann in storyid2anns[k]]) == {0,1,2,3,4}
        storyid2anns[k] = sorted(storyid2anns[k], key=lambda x: x['worker_arranged_photo_order'])

    # get paths
    image_paths = os.listdir(image_directory)
    image_paths = {k.split('.')[0]: image_directory + '/' + k for k in image_paths}

    print('{} stories'.format(len(storyid2anns)))
    sorted_story_ids = list(sorted(storyid2anns.keys()))
    n_written = 0

    to_write = []
    for idx, story_id in tqdm.tqdm(list(enumerate(sorted_story_ids))):

        # check if we should be doing this in the current batch.
        if idx % args.num_folds != fold: continue

        anns = storyid2anns[story_id]
        assert [a['worker_arranged_photo_order'] for a in anns] == [0,1,2,3,4]

        texts = [_tokenizer.encode(a['original_text']) for a in anns]
        images = [Image.open(image_paths[a['photo_flickr_id']]) for a in anns]
        for imidx, im in enumerate(images):
            try:
                im.thumbnail((800, 800))
            except:
                print('***INVALID IMAGE***'*10)
                pprint.pprint(anns[imidx])
                quit()

        jpgstrimages = []
        for imidx, im in enumerate(images):
            try:
                jpgstrimages.append(convert_PIL_to_jpgstring(im))
            except:
                print('***INVALID IMAGE***'*10)
                pprint.pprint(anns[imidx])
                quit()

        images = jpgstrimages
        image_hashes = [hashlib.sha256(im).hexdigest().encode('utf-8') for im in images]

        for perm in list(itertools.permutations(range(5))) if not args.just_one_perm else [[0,1,2,3,4]]:
            #9ABCDE
            perm_int_encode = int(''.join([str(x) for x in perm])) + 900000

            feature_dict = {
                'permutation_identity_encode': int64_feature(perm_int_encode),
                'story_id': int64_feature(story_id)}

            for perm_idx, perm_sorted_idx in enumerate(perm):
                feature_dict.update({
                    'sentence/sentence_{}'.format(perm_idx): int64_list_feature(texts[perm_sorted_idx]),
                    'image/image_{}_sha256'.format(perm_idx): bytes_feature(image_hashes[perm_sorted_idx]),
                    'image/image_{}_encoded'.format(perm_idx): bytes_feature(images[perm_sorted_idx]),
                    'image/image_{}_format'.format(perm_idx): bytes_feature('jpeg'.encode('utf8')),
                    'image/image_{}_is_valid'.format(perm_idx): int64_feature(1)})

            example = tf.train.Example(features=tf.train.Features(feature=feature_dict))
            writer.write(example.SerializeToString())
            n_written += 1

    print('Wrote {}'.format(n_written))
    return n_written, example

def _process(idx_args):
    idx, args = idx_args
    if args.just_one_perm:
        output_name = '{}-justone-{:05d}-of-{:05d}.tfrecord'.format(args.save_dir + '{}'.format(args.mode), idx, args.num_folds)
    else:
        output_name = '{}-{:05d}-of-{:05d}.tfrecord'.format(args.save_dir + '{}'.format(args.mode), idx, args.num_folds)

    print('writing to {}'.format(output_name))
    writer = GCSTFRecordWriter(output_name)
    count, _ = process(args.input_story_json,
                       args.image_directory,
                       writer,
                       idx,
                       args)
    writer.close()
    return count

def _process_final(idx_args_total):
    idx, args, total = idx_args_total
    if args.just_one_perm:
        output_name = '{}-justone-{:05d}-of-{:05d}.tfrecord'.format(args.save_dir + '{}'.format(args.mode), idx, args.num_folds)
    else:
        output_name = '{}-{:05d}-of-{:05d}.tfrecord'.format(args.save_dir + '{}'.format(args.mode), idx, args.num_folds)
    print('writing to {}'.format(output_name))
    writer = GCSTFRecordWriter(output_name)
    count, final_ex = process(args.input_story_json,
                              args.image_directory,
                              writer,
                              idx,
                              args)
    n_to_add = args.pad_to_batch_size - (total + count) % args.pad_to_batch_size
    if args.mode != 'train' and n_to_add > 0:
        print('we are not in training, so we will add {} to make batch size {} work.'.format(
            n_to_add, args.pad_to_batch_size))

        for _ in range(n_to_add):
            writer.write(final_ex.SerializeToString())
            count += 1

    writer.close()

    if args.mode == 'train':
        return count, 0
    else:
        return count, n_to_add

def main():
    args = parse_args()

    with Pool(args.num_threads) as p:
        res = p.map(_process, [(idx, args) for idx in range(args.num_folds-1)])
    print('{} total examples written in parallel'.format(np.sum(res)))

    total = int(np.sum(res))

    # for the final one, we will add replicas of the final example.
    count, fakes = _process_final((args.num_folds-1, args, total))
    total += count
    print('{} final examplds written, which includes {} fakes to make batch size work.'.format(total, fakes))

if __name__ == '__main__':
    main()

max_stars_repo_name: dexuiz/merlot

max_issues_repo_name: dexuiz/merlot

max_forks_repo_name: dexuiz/merlot


text:
# coding: utf-8

# In[ ]:

import dask.array as da
import os
import numpy as np
import pylab as plt
import h5py
from dask.dot import dot_graph
#from dask.multiprocessing import get
from dask import get
from functools import partial
from time import sleep, clock
from scipy.integrate import simps

from dask.callbacks import Callback
from distributed import Client
#from multiprocessing.pool import ThreadPool

from ForwardEquation import forward_equation, forward_equation_dask
from Gradient import compute_gradient, compute_gradient_dask
from tri_cubic import TriCubic
from LineSearch import line_search
from InfoCompleteness import precondition
from InitialModel import create_initial_model
from CalcRays import calc_rays,calc_rays_dask
from real_data import plot_datapack
from PlotTools import animate_tci_slices
from Covariance import Covariance

def store_Fdot(resettable,output_folder,n1,n2,Fdot,gamma,beta,dm,dgamma,v,sigma_m,L_m):
    filename="{}/F{}gamma{}.hdf5".format(output_folder,n1,n2)
    if os.path.isfile(filename) and resettable:
        return filename
    #gamma.save(filename)
    out = Fdot.copy()
    xvec = dm.xvec
    yvec = dm.yvec
    zvec = dm.zvec
    gamma_dm = scalarProduct(gamma.get_shaped_array(),dm.get_shaped_array(),sigma_m,L_m,xvec,yvec,zvec)
    a = dm.m*(beta*gamma_dm - scalarProduct(dgamma.get_shaped_array(),Fdot.get_shaped_array(),sigma_m,L_m,xvec,yvec,zvec))
    a -= v.m*gamma_dm
    a /= scalarProduct(dgamma.get_shaped_array(),dm.get_shaped_array(),sigma_m,L_m,xvec,yvec,zvec)
    out.m + a
    print("Beta: {}".format(beta))
    print("Difference: {}".format(np.dot(out.m,gamma.m)/np.linalg.norm(out.m)/np.linalg.norm(gamma.m)))

    if resettable:
        out.save(filename)
        return filename
    else:
        return out

def pull_Fdot(resettable,filename):
    if resettable:
        return TriCubic(filename=filename)
    else:
        return filename

def pull_gamma(resettable,filename):
    if resettable:
        return TriCubic(filename=filename)
    else:
        return filename

def store_gamma(resettable,output_folder,n,rays, g, dobs, i0, K_ne, m_tci, mPrior, CdCt, sigma_m, Nkernel, size_cell,cov_obj):
    filename='{}/gamma_{}.hdf5'.format(output_folder,n)
    if os.path.isfile(filename) and resettable:
        return filename
    gradient = compute_gradient_dask(rays, g, dobs, i0, K_ne, m_tci, mPrior.get_shaped_array(), CdCt, sigma_m, Nkernel, size_cell,cov_obj)
    TCI = TriCubic(m_tci.xvec,m_tci.yvec,m_tci.zvec,gradient)

    if resettable:
        TCI.save(filename)
        return filename
    else:
        return TCI

def plot_gamma(output_folder,n,TCI):
    foldername = '{}/gamma_{}'.format(output_folder,n)
    animate_tci_slices(TCI,foldername,num_seconds=20.)
    return foldername

def store_forwardEq(resettable,output_folder,n,template_datapack,ant_idx,time_idx,dir_idx,rays,K_ne,m_tci,i0):
    filename = "{}/g_{}.hdf5".format(output_folder,n)
    if os.path.isfile(filename) and resettable:
        return filename
    assert not np.any(np.isnan(m_tci.m)), "nans in model"
    g = forward_equation(rays,K_ne,m_tci,i0)
    assert not np.any(np.isnan(g)), "nans in g"
    datapack = template_datapack.clone()
    datapack.set_dtec(g,ant_idx=ant_idx,time_idx=time_idx,dir_idx = dir_idx)

    dobs = template_datapack.get_dtec(ant_idx=ant_idx,time_idx=time_idx,dir_idx = dir_idx)
    vmin = np.min(dobs)
    vmax = np.max(dobs)
    plot_datapack(datapack,ant_idx=ant_idx,time_idx=time_idx,dir_idx = dir_idx,
                figname=filename.split('.')[0], vmin = vmin, vmax = vmax)#replace('hdf5','png'))
    if resettable:
        datapack.save(filename)
        return filename
    else:
        return datapack

def pull_forwardEq(resettable,filename,ant_idx,time_idx,dir_idx):
    if resettable:
        datapack = DataPack(filename=filename)
        g = datapack.get_dtec(ant_idx=ant_idx,time_idx=time_idx,dir_idx = dir_idx)
        return g
    else:
        g = filename.get_dtec(ant_idx=ant_idx,time_idx=time_idx,dir_idx = dir_idx)
        return g

def calc_epsilon(output_folder,n,phi,m_tci,rays,K_ne,i0,g,dobs,CdCt):
    bins = max(10,int(np.ceil(np.sqrt(g.size))))
    drad = 8.44797256e-7/120e6
    dtau = 1.34453659e-7/120e6**2
    r = dtau/drad*1e9#mu sec factor
    plt.figure()
    plt.hist(g.flatten(),alpha=0.2,label='g',bins=bins)
    plt.hist(dobs.flatten(),alpha=0.2,label='dobs',bins=bins)
    plt.legend(frameon=False)
    plt.savefig("{}/data-hist-{}.png".format(output_folder,n))
    plt.clf()
    plt.hist((g-dobs).flatten()*drad*1e16,bins=bins)
    plt.xlabel(r"$d\phi$ [rad] | {:.2f} delay [ns]".format(r))
    plt.savefig("{}/datadiff-hist-{}.png".format(output_folder,n))
    plt.close('all')
    ep,S,reduction = line_search(rays,K_ne,m_tci,i0,phi.get_shaped_array(),g,dobs,CdCt,figname="{}/line_search{}".format(output_folder,n))
    return ep,S,reduction

def store_m(resettable,output_folder,n,m_tci0,phi,rays,K_ne,i0,g,dobs,CdCt,state_file):
    filename = "{}/m_{}.hdf5".format(output_folder,n)
    with h5py.File(state_file,'w') as state:
        if '/{}/epsilon_n'.format(n) not in state:
            epsilon_n,S,reduction = calc_epsilon(output_folder,n,phi,m_tci0,rays,K_ne,i0,g,dobs,CdCt)
            state['/{}/epsilon_n'.format(n)] = epsilon_n
            state['/{}/S'.format(n)] = S
            state['/{}/reduction'.format(n)] = reduction
            state.flush()
        else:
            epsilon_n,S,reduction = state['/{}/epsilon_n'.format(n)], state['/{}/S'.format(n)], state['/{}/reduction'.format(n)]

    if os.path.isfile(filename) and resettable:
        return filename
    m_tci = m_tci0.copy()
    m_tci.m -= epsilon_n*phi.m
    if resettable:
        m_tci.save(filename)
        return filename
    else:
        return m_tci

def pull_m(resettable,filename):
    if resettable:
        return TriCubic(filename=filename)
    else:
        return filename #object not filename

def scalarProduct(a,b,sigma_m,L_m,xvec,yvec,zvec):
    out = a*b
    out = simps(simps(simps(out,zvec,axis=2),yvec,axis=1),xvec,axis=0)
    #out /= (np.pi*8.*sigma_m**2 * L_m**3)
    return out

def calcBeta(dgamma, v, dm,sigma_m,L_m):
    xvec = dgamma.xvec
    yvec = dgamma.yvec
    zvec = dgamma.zvec
    beta = 1. + scalarProduct(dgamma.get_shaped_array(),v.get_shaped_array(),sigma_m,L_m,xvec,yvec,zvec)/(scalarProduct(dgamma.get_shaped_array(),dm.get_shaped_array(),sigma_m,L_m,xvec,yvec,zvec) + 1e-15)
    print("E[|dm|] = {} | E[|dgamma|] = {}".format(np.mean(np.abs(dm.m)),np.mean(np.abs(dgamma.m))))
    return beta

def diffTCI(TCI1,TCI2):
    TCI = TCI1.copy()
    TCI.m -= TCI2.m
    return TCI

def store_F0dot(resettable,output_folder,n,F0,gamma):
    filename="{}/F0gamma{}.hdf5".format(output_folder,n)
    if os.path.isfile(filename) and resettable:
        return filename
    out = gamma.copy()
    out.m *= F0.m
    if resettable:
        out.save(filename)
        return filename
    else:
        return out

def plot_model(output_folder,n,mModel,mPrior,K_ne):
    tmp = mModel.m.copy()
    np.exp(mModel.m,out=mModel.m)
    mModel.m *= K_ne
    mModel.m -= K_ne*np.exp(mPrior.m)
    foldername = '{}/m_{}'.format(output_folder,n)
    animate_tci_slices(mModel,foldername,num_seconds=20.)
    mModel.m = tmp
    print("Animation of model - prior in {}".format(foldername))
    return foldername

def create_bfgs_dask(resettable,output_folder,N,datapack,L_ne,size_cell,i0, ant_idx=-1, dir_idx=-1, time_idx = [0]):
    try:
        os.makedirs(output_folder)
    except:
        pass
    print("Using output folder: {}".format(output_folder))
    state_file = "{}/state".format(output_folder)
    straight_line_approx = True
    tmax = 1000.
    antennas,antenna_labels = datapack.get_antennas(ant_idx = ant_idx)
    patches, patch_names = datapack.get_directions(dir_idx = dir_idx)
    times,timestamps = datapack.get_times(time_idx=time_idx)
    datapack.set_reference_antenna(antenna_labels[i0])
    #plot_datapack(datapack,ant_idx = ant_idx, time_idx = time_idx, dir_idx = dir_idx,figname='{}/dobs'.format(output_folder))
    dobs = datapack.get_dtec(ant_idx = ant_idx, time_idx = time_idx, dir_idx = dir_idx)
    Na = len(antennas)
    Nt = len(times)
    Nd = len(patches)
    fixtime = times[Nt>>1]
    phase = datapack.get_center_direction()
    array_center = datapack.radio_array.get_center()
    #Average time axis down and center on fixtime
    if Nt == 1:
        var = (0.5*np.percentile(dobs[dobs>0],25) + 0.5*np.percentile(-dobs[dobs<0],25))**2
        Cd = np.ones([Na,1,Nd],dtype=np.double)*var
        Ct = (np.abs(dobs)*0.05)**2
        CdCt = Cd + Ct
    else:
        dt = times[1].gps - times[0].gps
        print("Averaging down window of length {} seconds [{} timestamps]".format(dt*Nt, Nt))
        Cd = np.stack([np.var(dobs,axis=1)],axis=1)
        dobs = np.stack([np.mean(dobs,axis=1)],axis=1)
        Ct = (np.abs(dobs)*0.05)**2
        CdCt = Cd + Ct
        time_idx = [Nt>>1]
        times,timestamps = datapack.get_times(time_idx=time_idx)
        Nt = len(times)
    print("E[S/N]: {} +/- {}".format(np.mean(np.abs(dobs)/np.sqrt(CdCt+1e-15)),np.std(np.abs(dobs)/np.sqrt(CdCt+1e-15))))
    vmin = np.min(datapack.get_dtec(ant_idx = ant_idx, time_idx = time_idx, dir_idx = dir_idx))
    vmax = np.max(datapack.get_dtec(ant_idx = ant_idx, time_idx = time_idx, dir_idx = dir_idx))
    plot_datapack(datapack,ant_idx=ant_idx,time_idx=time_idx,dir_idx = dir_idx,
            figname='{}/dobs'.format(output_folder), vmin = vmin, vmax = vmax)#replace('hdf5','png'))
    ne_tci = create_initial_model(datapack,ant_idx = ant_idx, time_idx = time_idx, dir_idx = dir_idx, zmax = tmax,spacing=size_cell)
    #make uniform
    #ne_tci.m[:] = np.mean(ne_tci.m)
    ne_tci.save("{}/nePriori.hdf5".format(output_folder))
    rays = calc_rays(antennas,patches,times, array_center, fixtime, phase, ne_tci, datapack.radio_array.frequency,
                    straight_line_approx, tmax, ne_tci.nz)
    m_tci = ne_tci.copy()
    K_ne = np.mean(m_tci.m)
    m_tci.m /= K_ne
    np.log(m_tci.m,out=m_tci.m)

    Nkernel = max(1,int(float(L_ne)/size_cell))
    sigma_m = np.log(10.)#ne = K*exp(m+dm) = K*exp(m)*exp(dm), exp(dm) in (0.1,10) -> dm = (log(10) - log(0.1))/2.
    cov_obj = Covariance(m_tci,sigma_m,L_ne,7./2.)
    #uvw = UVW(location = datapack.radio_array.get_center().earth_location,obstime = fixtime,phase = phase)
    #ants_uvw = antennas.transform_to(uvw).cartesian.xyz.to(au.km).value.transpose()
    #dirs_uvw = patches.transform_to(uvw).cartesian.xyz.value.transpose()
    F0 = precondition(ne_tci, datapack,ant_idx=ant_idx, dir_idx=dir_idx, time_idx = time_idx)
    #
    dsk = {}
    for n in range(int(N)):
        #g_n
        dsk['store_forwardEq{}'.format(n)] = (store_forwardEq,resettable,'output_folder',n,'template_datapack','ant_idx','time_idx','dir_idx','rays',
                                              'K_ne','pull_m{}'.format(n),'i0')
        dsk['pull_forwardEq{}'.format(n)] = (pull_forwardEq,resettable,'store_forwardEq{}'.format(n),'ant_idx','time_idx','dir_idx')
        #gradient
        dsk['store_gamma{}'.format(n)] = (store_gamma,resettable,'output_folder',n,'rays', 'pull_forwardEq{}'.format(n), 'dobs', 'i0', 'K_ne',
                                        'pull_m{}'.format(n),'mprior', 'CdCt', 'sigma_m', 'Nkernel', 'size_cell', 'cov_obj')
        dsk['pull_gamma{}'.format(n)] = (pull_gamma,resettable,'store_gamma{}'.format(n))
        #m update
        dsk['store_m{}'.format(n+1)] = (store_m,resettable,'output_folder',n+1,'pull_m{}'.format(n),'pull_phi{}'.format(n),'rays',
                                    'K_ne','i0','pull_forwardEq{}'.format(n),'dobs','CdCt','state_file')
        dsk['pull_m{}'.format(n+1)] = (pull_m,resettable,'store_m{}'.format(n+1))
        dsk['plot_m{}'.format(n+1)] = (plot_model,'output_folder',n+1,'pull_m{}'.format(n+1),'mprior','K_ne')
        dsk['plot_gamma{}'.format(n)] = (plot_gamma,'output_folder',n,'pull_gamma{}'.format(n))
        #phi
        dsk['pull_phi{}'.format(n)] = (pull_Fdot,resettable,'store_F{}(gamma{})'.format(n,n))
        dsk['store_F{}(gamma{})'.format(n+1,n+1)] = (store_Fdot,resettable,'output_folder', n+1, n+1 ,
                                                     'pull_F{}(gamma{})'.format(n,n+1),
                                                     'pull_gamma{}'.format(n+1),
                                                     'beta{}'.format(n),
                                                     'dm{}'.format(n),
                                                     'dgamma{}'.format(n),
                                                     'v{}'.format(n),
                                                     'sigma_m','L_m'
                                                    )
        for i in range(1,n+1):
            dsk['store_F{}(gamma{})'.format(i,n+1)] = (store_Fdot, resettable,'output_folder',i, n+1 ,
                                                     'pull_F{}(gamma{})'.format(i-1,n+1),
                                                     'pull_gamma{}'.format(n+1),
                                                     'beta{}'.format(i-1),
                                                     'dm{}'.format(i-1),
                                                     'dgamma{}'.format(i-1),
                                                     'v{}'.format(i-1),
                                                       'sigma_m','L_m'
                                                    )
            dsk['pull_F{}(gamma{})'.format(i,n+1)] = (pull_Fdot,resettable,'store_F{}(gamma{})'.format(i,n+1))
        #should replace for n=0
        dsk['store_F0(gamma{})'.format(n)] = (store_F0dot, resettable,'output_folder',n, 'pull_F0','pull_gamma{}'.format(n))
        dsk['pull_F0(gamma{})'.format(n)] = (pull_Fdot,resettable,'store_F0(gamma{})'.format(n))
#         #epsilon_n
#         dsk['ep{}'.format(n)] = (calc_epsilon,n,'pull_phi{}'.format(n),'pull_m{}'.format(n),'rays',
#                                     'K_ne','i0','pull_forwardEq{}'.format(n),'dobs','CdCt')
        #
        dsk['beta{}'.format(n)] = (calcBeta,'dgamma{}'.format(n),'v{}'.format(n),'dm{}'.format(n),'sigma_m','L_m')
        dsk['dgamma{}'.format(n)] = (diffTCI,'pull_gamma{}'.format(n+1),'pull_gamma{}'.format(n))
        dsk['dm{}'.format(n)] = (diffTCI,'pull_m{}'.format(n+1),'pull_m{}'.format(n))
        dsk['v{}'.format(n)] = (diffTCI,'pull_F{}(gamma{})'.format(n,n+1),'pull_phi{}'.format(n))
    dsk['pull_F0'] = F0
    dsk['template_datapack'] = datapack
    dsk['ant_idx'] = ant_idx
    dsk['time_idx'] = time_idx
    dsk['dir_idx'] = dir_idx
    dsk['pull_m0'] = 'mprior'
    dsk['i0'] = i0
    dsk['K_ne'] = K_ne
    dsk['dobs'] = dobs
    dsk['mprior'] = m_tci
    dsk['CdCt'] = CdCt
    dsk['sigma_m'] = sigma_m
    dsk['Nkernel'] = Nkernel
    dsk['L_m'] = L_ne
    dsk['size_cell'] = size_cell
    dsk['cov_obj'] = cov_obj
    #calc rays
    #dsk['rays'] = (calc_rays_dask,'antennas','patches','times', 'array_center', 'fixtime', 'phase', 'ne_tci', 'frequency',  'straight_line_approx','tmax')
    dsk['rays'] = rays
    dsk['output_folder'] = output_folder
    dsk['resettable'] = resettable
    dsk['state_file'] = state_file

    return dsk

class TrackingCallbacks(Callback):
    def _start(self,dsk):
        self.start_time = clock()
    def _pretask(self, key, dask, state):
        """Print the key of every task as it's started"""
        self.t1 = clock()
        print('Starting {} at {} seconds'.format(key,self.t1-self.start_time))
    def _posttask(self,key,result,dsk,state,id):
        print("{} took {} seconds".format(repr(key),clock() - self.t1))
    def _finish(self,dsk,state,errored):
        self.endTime = clock()
        dt = (self.endTime - self.start_time)
        print("Approximate time to complete: {} time units".format(dt))

if __name__=='__main__':
    from real_data import DataPack
    from AntennaFacetSelection import select_antennas_facets
    from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler
    from dask.diagnostics import visualize
    #from InitialModel import createTurbulentlModel
    i0 = 0
    datapack = DataPack(filename="output/test/simulate/simulate_3/datapack_sim.hdf5")
    #datapack = DataPack(filename="output/test/datapack_obs.hdf5")
    #flags = datapack.find_flagged_antennas()
    #datapack.flag_antennas(flags)
    datapack_sel = select_antennas_facets(20, datapack, ant_idx=-1, dir_idx=-1, time_idx = np.arange(1))
    #pert_tci = createTurbulentlModel(datapack_sel,ant_idx = -1, time_idx = -1, dir_idx = -1, zmax = 1000.)
    L_ne = 25.
    size_cell = 5.
    dsk = create_bfgs_dask(True, "output/test/bfgs_3_1", 25,datapack_sel,L_ne,size_cell,i0, ant_idx=-1, dir_idx=-1, time_idx = np.arange(1))
    #dot_graph(dsk,filename="{}/BFGS_graph".format(output_folder),format='png')
    #dot_graph(dsk,filename="{}/BFGS_graph".format(output_folder),format='svg')
    #client = Client()
    #with TrackingCallbacks():
    with Profiler() as prof, ResourceProfiler(dt=0.25) as rprof, CacheProfiler() as cprof:
        get(dsk,['plot_m25'])
    visualize([prof,rprof,cprof])

max_stars_repo_name: Joshuaalbert/IonoTomo

max_issues_repo_name: Joshuaalbert/IonoTomo

max_forks_repo_name: Joshuaalbert/IonoTomo

reason: import numpy,from scipy


text:
import time
import numpy as np
import os

from pycompss.api.task import task
from pycompss.api.api import compss_wait_on, compss_barrier
from pycompss.api.reduction import reduction
from pycompss.api.parameter import COLLECTION_IN

from tad4bj.slurm import handler as tadh

#############################################
# Constants / experiment values:
#############################################

POINTS_PER_FRAGMENT = int(os.environ["POINTS_PER_FRAGMENT"])
NUMBER_OF_FRAGMENTS = int(os.environ["NUMBER_OF_FRAGMENTS"])
NUMBER_OF_ITERATIONS = int(os.getenv("NUMBER_OF_ITERATIONS", "10"))

USE_REDUCTION_DECORATOR = bool(int(os.environ["USE_REDUCTION_DECORATOR"]))
REDUCTION_CHUNK_SIZE = int(os.getenv("REDUCTION_CHUNK_SIZE", "48"))

SEED = 420

#############################################
#############################################

@task(partials=COLLECTION_IN, returns=object)
def sum_partials(partials):
    return np.sum(partials, axis=0)

if USE_REDUCTION_DECORATOR:
    # Non-canonical way of applying a decorator programatically
    sum_partials = reduction(chunk_size=str(REDUCTION_CHUNK_SIZE))(sum_partials)
    # dear reader: despite your reluctance, I assure you this is legit.
    # p.s.: unless you are wondering why chunk_size parameter is a str. Beats me. COMPSs' doc says so.

@task(returns=object)
def generate_fragment(seed):
    # Random generation distributions
    np.random.seed(seed)
    return np.random.f(10, 2, POINTS_PER_FRAGMENT)

@task(returns=object)
def partial_histogram(values, bins):
    values, _ = np.histogram(values, bins)
    return values

def histogram(experiment):
    bins = np.concatenate((np.arange(0,10, 0.1), np.arange(10, 50), [np.infty]))

    partials = list()

    for fragment in experiment:
        partial = partial_histogram(fragment, bins)
        partials.append(partial)

    result = sum_partials(partials)

    return compss_wait_on(result)

def main():
    print(f"""Starting experiment with the following:

POINTS_PER_FRAGMENT = {POINTS_PER_FRAGMENT}
NUMBER_OF_FRAGMENTS = {NUMBER_OF_FRAGMENTS}
NUMBER_OF_ITERATIONS = {NUMBER_OF_ITERATIONS}
""")
    start_time = time.time()

    experiment = list()
    for i in range(NUMBER_OF_FRAGMENTS):
        print("Proceeding to generate fragment #%d" % (i + 1))
        experiment.append(generate_fragment(SEED + i))

    compss_barrier()

    print("Generation/Load done")
    initialization_time = time.time() - start_time
    print("Starting histogram")

    print("-----------------------------------------")
    print("-------------- RESULTS ------------------")
    print("-----------------------------------------")
    print("Initialization time: %f" % initialization_time)

    tadh["initialization_time"] = initialization_time
    tadh.write_all()
    tadh["iteration_time"] = list()

    time.sleep(10)

    # Run histogram
    result = None
    for i in range(NUMBER_OF_ITERATIONS):
        start_t = time.time()
        result = histogram(experiment)
        end_t = time.time()

        histogram_time = end_t - start_t
        print("Histogram time (#%d/%d): %f" % (i + 1, NUMBER_OF_ITERATIONS, histogram_time))
        tadh["iteration_time"].append(histogram_time)

    tadh.write_all()
    print(result)

if __name__ == "__main__":
    main()

max_stars_repo_name: bsc-dom/split-miniapps

max_issues_repo_name: bsc-dom/split-miniapps

max_forks_repo_name: bsc-dom/split-miniapps