text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Testing suite for ARG class.

"""
from __future__ import print_function, division

import unittest as ut
import numpy as np

from argamma import ARG, ARGparams, days_in_year

class ARGTestCase(ut.TestCase):
    """Test ARG model class."""

    def test_conversion_to_q(self):
        """Test conversion to Q measure."""
        mean, rho, delta = .1**2, .9, 1.1
        phi, price_vol, price_ret = -.5, -1, .5
        param = ARGparams(mean=mean, rho=rho, delta=delta,
                          phi=phi, price_ret=price_ret, price_vol=price_vol)
        scale = param.get_scale()
        argmodel = ARG()
        param_q = argmodel.convert_to_q(param)

        self.assertIsInstance(param_q, ARGparams)

        factor = 1/(1 + scale * (price_vol \
            + argmodel.alpha(price_ret, param)))
        scale = scale * factor
        beta = param.get_beta() * factor
        rho = scale * beta

        self.assertEqual(param_q.scale, scale)
        self.assertEqual(param_q.rho, rho)
        self.assertEqual(param_q.delta, delta)

    def test_uncond_moments(self):
        """Test unconditional moments of the ARG model."""

        mean, rho, delta = 1, .9, 3
        param = ARGparams(mean=mean, rho=rho, delta=delta)
        argmodel = ARG()

        self.assertIsInstance(argmodel.umean(param), float)
        self.assertIsInstance(argmodel.uvar(param), float)
        self.assertIsInstance(argmodel.ustd(param), float)

        # TODO : test using symbolic library
        # that these moments coincide with derivatives of a, b, c

    def test_abc_functions(self):
        """Test functions a, b, c of ARG model."""

        param = ARGparams()
        argmodel = ARG()
        for i in [1, 10]:
            uarg = np.linspace(-50, 100, i)

            self.assertIsInstance(argmodel.afun(uarg, param), np.ndarray)
            self.assertIsInstance(argmodel.bfun(uarg, param), np.ndarray)
            self.assertIsInstance(argmodel.cfun(uarg, param), np.ndarray)
            self.assertIsInstance(argmodel.afun_q(uarg, param), np.ndarray)
            self.assertIsInstance(argmodel.bfun_q(uarg, param), np.ndarray)

            self.assertEqual(uarg.shape, argmodel.afun(uarg, param).shape)
            self.assertEqual(uarg.shape, argmodel.bfun(uarg, param).shape)
            self.assertEqual(uarg.shape, argmodel.cfun(uarg, param).shape)
            self.assertEqual(uarg.shape, argmodel.afun_q(uarg, param).shape)
            self.assertEqual(uarg.shape, argmodel.bfun_q(uarg, param).shape)

    def test_ret_functions(self):
        """Test functions alpha, beta, gamma of ARG model."""

        param = ARGparams()
        argmodel = ARG()
        for i in [1, 10]:
            uarg = np.linspace(-50, 100, i)

            self.assertIsInstance(argmodel.alpha(uarg, param), np.ndarray)
            self.assertIsInstance(argmodel.beta(uarg, param), np.ndarray)
            self.assertIsInstance(argmodel.gamma(uarg, param), np.ndarray)

            self.assertEqual(uarg.shape, argmodel.alpha(uarg, param).shape)
            self.assertEqual(uarg.shape, argmodel.beta(uarg, param).shape)
            self.assertEqual(uarg.shape, argmodel.gamma(uarg, param).shape)

    def test_joint_functions(self):
        """Test functions lfun, gfun of ARG model."""
        param = ARGparams()
        argmodel = ARG()
        uarg, varg = 1., 2.

        lfun, gfun = argmodel.lgfun(uarg, varg, param)
        lfunq, gfunq = argmodel.lgfun_q(uarg, varg, param)

        self.assertIsInstance(lfun, float)
        self.assertIsInstance(gfun, float)
        self.assertIsInstance(lfunq, float)
        self.assertIsInstance(gfunq, float)

        uarg, varg = 1., np.arange(5)
        lfun, gfun = argmodel.lgfun(uarg, varg, param)
        lfunq, gfunq = argmodel.lgfun_q(uarg, varg, param)

        self.assertEqual(lfun.shape, varg.shape)
        self.assertEqual(gfun.shape, varg.shape)
        self.assertEqual(lfunq.shape, varg.shape)
        self.assertEqual(gfunq.shape, varg.shape)

    def test_joint_multiperiod_functions(self):
        """Test functions psin, upsn of ARG model."""
        param = ARGparams()
        varg = np.atleast_2d(1.)
        days = 5
        maturity = days / days_in_year()
        argmodel = ARG()
        argmodel.maturity = maturity

        psi, ups = argmodel.ch_fun_elements(varg, param)
        self.assertEqual(psi.shape, (1, 1))
        self.assertEqual(ups.shape, (1, 1))

        varg = np.arange(5)[:, np.newaxis]

        psi, ups = argmodel.ch_fun_elements(varg, param)
        self.assertEqual(psi.shape, varg.shape)
        self.assertEqual(ups.shape, varg.shape)

        psi, ups = argmodel.store_ch_fun_elements(varg, param)
        self.assertIsInstance(psi, dict)
        self.assertIsInstance(ups, dict)
        for day in range(1, days):
            self.assertEqual(psi[day].shape, varg.shape)
            self.assertEqual(ups[day].shape, varg.shape)

    def test_char_fun_ret_q(self):
        """Test risk-neutral return charcteristic function."""
        param = ARGparams()
        nobs = 5
        vol = np.arange(nobs)
        maturity = np.ones(nobs) * 5/days_in_year()
        riskfree = 0.
        argmodel = ARG()
        argmodel.riskfree = riskfree
        argmodel.maturity = maturity
        argmodel.load_data(vol=vol)
        varg = np.atleast_2d(1.)

        self.assertEqual(argmodel.char_fun_ret_q(varg, param).shape, (1, nobs))

        narg = 10
        varg = np.ones((narg, 1))
        size = (narg, nobs)
        self.assertEqual(argmodel.char_fun_ret_q(varg, param).shape, size)

        param = ARGparams()
        vol = 1.
        argmodel = ARG()
        argmodel.maturity = 5/days_in_year()
        argmodel.riskfree = 0.
        argmodel.load_data(vol=vol)
        varg = np.atleast_2d(1.)

        cfun = argmodel.char_fun_ret_q(varg, param)

        self.assertIsInstance(cfun[0, 0], complex)

    def test_charfun(self):
        """Test risk-neutral return charcteristic function."""
        param = ARGparams()
        vol = 1.
        maturity = 5/days_in_year()
        riskfree = 0.
        argmodel = ARG(param=param)
        argmodel.riskfree = riskfree
        argmodel.maturity = maturity
        argmodel.load_data(vol=vol)
        narg = 5
        varg = np.arange(narg)[:, np.newaxis]
        cfun = argmodel.charfun(varg)

        self.assertIsInstance(cfun, np.ndarray)
        self.assertEqual(argmodel.charfun(varg).size, narg)

        nobs = 10
        vol = np.ones(nobs)
        maturity = maturity * np.ones(nobs)
        argmodel = ARG(param=param)
        argmodel.riskfree = riskfree
        argmodel.maturity = maturity
        argmodel.load_data(vol=vol)
        self.assertEqual(argmodel.charfun(varg).shape, (narg, nobs))

    def test_cos_restriction(self):
        """Test cos_restriction method."""
        param = ARGparams()
        vol = 1.
        maturity = 5/days_in_year()
        riskfree = 0.
        argmodel = ARG(param=param)
        argmodel.riskfree = riskfree
        argmodel.maturity = maturity

        argmodel.load_data(vol=vol)
        alim, blim = argmodel.cos_restriction()

        self.assertIsInstance(alim[0], float)
        self.assertIsInstance(blim[0], float)

        nobs = 10
        vol = np.ones(nobs)
        argmodel = ARG(param=param)
        argmodel.riskfree = riskfree
        argmodel.maturity = maturity
        argmodel.load_data(vol=vol)
        alim, blim = argmodel.cos_restriction()

        self.assertEqual(alim.size, nobs)
        self.assertEqual(blim.size, nobs)

    def test_abc_derivatives(self):
        """Test derivatives of functions a, b, c of ARG model."""

        param = ARGparams()
        argmodel = ARG()
        for i in [1, 10]:
            uarg = np.linspace(-50, 100, i)

            self.assertIsInstance(argmodel.dafun(uarg, param), np.ndarray)
            self.assertIsInstance(argmodel.dbfun(uarg, param), np.ndarray)

            self.assertEqual(argmodel.dafun(uarg, param).shape,
                             (3, uarg.shape[0]))
            self.assertEqual(argmodel.dbfun(uarg, param).shape,
                             (3, uarg.shape[0]))

    def test_load_data(self):
        """Test load data into the model object."""
        argmodel = ARG()

        self.assertEqual(argmodel.vol, None)
        self.assertEqual(argmodel.ret, None)

        argmodel.load_data(vol=0)

        self.assertEqual(argmodel.vol, 0)
        self.assertEqual(argmodel.ret, None)

        argmodel.load_data(ret=1)

        self.assertEqual(argmodel.vol, 0)
        self.assertEqual(argmodel.ret, 1)

        argmodel.load_data(ret=2, vol=3)

        self.assertEqual(argmodel.vol, 3)
        self.assertEqual(argmodel.ret, 2)

    def test_simulations(self):
        """Test simulation of ARG model."""

        param = ARGparams()
        argmodel = ARG()

        self.assertIsInstance(argmodel.vsim(param=param), np.ndarray)
        self.assertIsInstance(argmodel.vsim2(param=param), np.ndarray)

        vol = argmodel.vsim2(param=param)
        argmodel.load_data(vol=vol)

        self.assertIsInstance(argmodel.rsim(param=param), np.ndarray)

        shapes = []
        nsim, nobs = 1, 1
        shapes.append((nobs, nsim))
        nsim, nobs = 2, 2
        shapes.append((nobs, nsim))
        nsim, nobs = int(1e3), int(1e3)
        shapes.append((nobs, nsim))

        for shape in shapes:
            nsim, nobs = shape

            self.assertEqual(argmodel.vsim(nsim=nsim, nobs=nobs,
                                           param=param).shape, shape)
            vol = argmodel.vsim2(nsim=nsim, nobs=nobs, param=param)
            argmodel.load_data(vol=vol)
            self.assertEqual(vol.shape, shape)
            self.assertEqual(argmodel.rsim(param=param).shape, shape)

            self.assertEqual(argmodel.vsim_last(nsim=nsim, nobs=nobs,
                                                param=param).shape, (nsim, ))
            self.assertGreater(argmodel.vsim(nsim=nsim, nobs=nobs,
                                             param=param).min(), 0)
            self.assertGreater(argmodel.vsim2(nsim=nsim, nobs=nobs,
                                              param=param).min(), 0)

    def test_option_premium(self):
        """Test option pricing for the model."""
        price, strike = 100, 90
        riskfree, maturity = 0, 5/days_in_year()
        moneyness = np.log(strike/price) - riskfree * maturity
        vol = .2**2/days_in_year()
        call = True

        rho = .55
        delta = .75
        mean = .2**2/days_in_year()
        phi = -.0
        price_vol = -16.0
        price_ret = 20.95

        param = ARGparams(mean=mean, rho=rho, delta=delta,
                          phi=phi, price_ret=price_ret, price_vol=price_vol)
        argmodel = ARG(param=param)

        premium = argmodel.option_premium(vol=vol, moneyness=moneyness,
                                          maturity=maturity, riskfree=riskfree,
                                          call=call)
        self.assertEqual(premium.shape, (1,))

        nobs = 10
        strike = np.exp(np.linspace(-.1, .1, nobs))
        moneyness = np.log(strike/price) - riskfree * maturity
        maturity = 5/days_in_year() * np.ones(10)

        premium = argmodel.option_premium(vol=vol, moneyness=moneyness,
                                          maturity=maturity, riskfree=riskfree,
                                          call=call)
        self.assertEqual(premium.shape, strike.shape)

        data = {'vol': vol, 'moneyness': moneyness, 'maturity': maturity,
                'riskfree': riskfree, 'call': call}
        premium = argmodel.option_premium(data=data)
        self.assertEqual(premium.shape, strike.shape)

if __name__ == '__main__':
    ut.main()

max_stars_repo_name: khrapovs/argamma

max_issues_repo_name: khrapovs/argamma

max_forks_repo_name: khrapovs/argamma


text:
import os
import time
import multiprocessing as mp
import numpy as np
import tensorflow as tf
import tensorflow.contrib.eager as tfe
from a2c_eager.utils import make_atari
from a2c_eager.policies import ActorCritic
from a2c_eager.config.run_configuration import Config
from a2c_eager.utils import boltzmann, greedy, discount, gae, Logger

# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

gpu_options = tf.GPUOptions(allow_growth=True)
tf_config = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options)
tf.enable_eager_execution(config=tf_config)

def generate_gradients(timestep):
    loader.restore(tf.train.latest_checkpoint(config.save_dir))
    observations, actions, rewards, values = [], [], [], []
    ob = env.reset()
    done = False

    while not done:
        logits, v = model.forward([ob])
        probs = tf.nn.softmax(logits)
        a = boltzmann(probs)
        next_ob, r, done, _ = env.step(a)
        observations.append(ob)
        actions.append(a)
        rewards.append(r)
        values.append(v)
        ob = next_ob

    model.policy.reset()
    advantages = gae(rewards, values, 0, gamma=config.gamma, tau=config.tau)
    discounted_rewards = discount(rewards, config.gamma)
    gradients, log_values = model.gradient(observations, advantages, discounted_rewards, actions)
    model.policy.reset()
    logger.log_performance(rewards, actions, *log_values)
    # Convert EagerTensors to numpy arrays for IPC.
    return [gradient.numpy() for gradient in gradients]

def initialize_process():
    global env, model, loader, config, logger
    config = Config("config/config.json")
    env = make_atari(config.env)
    # env = gym.make(config.env)
    model = ActorCritic(env.action_space.n, policy=config.policy)
    loader = tfe.Checkpoint(model=model.policy, optimizer_step=tf.train.get_or_create_global_step())
    logger = Logger("Worker_{}".format(os.getpid()))

def train():
    mp.set_start_method('spawn', force=True)

    config = Config("config/config.json")
    env = make_atari(config.env)
    # env = gym.make(config.env)
    step = tf.train.get_or_create_global_step()
    optimizer = tf.train.AdamOptimizer(learning_rate=config.lr)
    model = ActorCritic(env.action_space.n, policy=config.policy)
    saver = tfe.Checkpoint(optimizer=optimizer, model=model.policy, optimizer_step=step)
    pool = mp.Pool(processes=config.processes, initializer=initialize_process)
    saver.restore(tf.train.latest_checkpoint(config.save_dir))
    logger = Logger('global')

    #   Initialize model.
    model.forward([env.reset()])
    ts = time.time()

    for t in range(config.steps):
        gradients = []
        roll = pool.map(generate_gradients, [t] * config.processes)

        for tup in zip(*roll):
            averaged = np.mean(tup, axis=0)
            gradients.append(tf.constant(averaged, dtype=tf.float32))

        clipped, _ = tf.clip_by_global_norm(gradients, config.max_norm)
        gnorms = [tf.norm(grad) for grad in clipped]
        logger.log_gradients(gnorms)
        logger.log_weights(model.policy.trainable_variables)
        optimizer.apply_gradients(zip(clipped, model.policy.trainable_weights), global_step=step)
        saver.save(file_prefix=config.file_prefix)

        print("Epoch took: {}".format(time.time() - ts))
        ts = time.time()

def test():
    config = Config("config/config.json")
    env = make_atari(config.env)
    model = ActorCritic(env.action_space.n, policy=config.policy)
    saver = tfe.Checkpoint(model=model.policy)
    saver.restore(tf.train.latest_checkpoint(config.save_dir))
    ob = env.reset()
    s = model.policy.s0

    while True:
        env.render()
        logits, _, s = model.forward([ob], s)
        probs = tf.nn.softmax(logits)
        a = greedy(probs)
        ob, _, done, _ = env.step(a)

        if done:
            ob = env.reset()

if __name__ == '__main__':
    train()

ext: py

max_stars_repo_name: Jventajas/Reinforcement-Learning

max_issues_repo_name: Jventajas/Reinforcement-Learning

max_forks_repo_name: Jventajas/Reinforcement-Learning

reason: import numpy


text:
################################################
#      Point, Shapes, Painter Classes by      #
#    By Geoff Samuel www.GeoffSamuel.com       #
################################################

# Import Modules
from PyQt5 import QtGui,QtCore, uic, QtWidgets
import os.path, sys
import tensorflow as tf
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)+'/Neural Network/')
import numpy as np
from scipy import ndimage

from dnn import *
from plotting import *
from imagePreprocessing import *

# Load the UI File
gui_model = 'GUI.ui'
form, base = uic.loadUiType(gui_model)
image_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir) + '/Images/'

# Point class for shapes
class Point:
    x, y = 0, 0
    def __init__(self, nx=0, ny=0):
        self.x = nx
        self.y = ny

# Single shape class
class Shape:
    location = Point()
    number = 0
    def __init__(self, L, S):
        self.location = L
        self.number = S

# Cotainer Class for all shapes
class Shapes:
    shapes = []
    def __init__(self):
        self.shapes = []
    # Returns the number of shapes
    def NumberOfShapes(self):
        return len(self.shapes)
    # Add a shape to the database, recording its position
    def NewShape(self, L, S):
        shape = Shape(L,S)
        self.shapes.append(shape)
    # Returns a shape of the requested data.
    def GetShape(self, Index):
        return self.shapes[Index]

# Class for painting widget
class Painter(QtWidgets.QWidget):
    ParentLink = 0
    MouseLoc = Point(0,0)
    LastPos = Point(0,0)
    def __init__(self, parent):
        super(Painter, self).__init__()
        self.ParentLink = parent
        self.MouseLoc = Point(0,0)
        self.LastPos = Point(0,0)
    #Mouse down event
    def mousePressEvent(self, event):
        self.ParentLink.IsPainting = True
        self.ParentLink.ShapeNum += 1
        self.LastPos = Point(0,0)
    #Mouse Move event
    def mouseMoveEvent(self, event):
        if(self.ParentLink.IsPainting == True):
            self.MouseLoc = Point(event.x(),event.y())
            if((self.LastPos.x != self.MouseLoc.x) and (self.LastPos.y != self.MouseLoc.y)):
                self.LastPos =  Point(event.x(),event.y())
                self.ParentLink.DrawingShapes.NewShape(self.LastPos, self.ParentLink.ShapeNum)
            self.repaint()
    #Mose Up Event
    def mouseReleaseEvent(self, event):
        if(self.ParentLink.IsPainting == True):
            self.ParentLink.IsPainting = False
    # Paint Event
    def paintEvent(self,event):
        painter = QtGui.QPainter()
        painter.begin(self)
        self.drawLines(event, painter)
        painter.end()
    # Draw the line
    def drawLines(self, event, painter):
        for i in range(self.ParentLink.DrawingShapes.NumberOfShapes()-1):
            T = self.ParentLink.DrawingShapes.GetShape(i)
            T1 = self.ParentLink.DrawingShapes.GetShape(i+1)
            if(T.number== T1.number):
                pen = QtGui.QPen(QtGui.QColor(0, 0, 0), 7, QtCore.Qt.SolidLine)
                painter.setPen(pen)
                painter.drawLine(T.location.x, T.location.y, T1.location.x, T1.location.y)

#Main UI Class
class CreateUI(base, form):
    DrawingShapes = Shapes()
    PaintPanel = 0
    IsPainting = False
    ShapeNum = 0
    def __init__(self):
        # Set up main window and widgets
        super(base,self).__init__()
        self.setupUi(self)
        self.setObjectName('Rig Helper')
        self.PaintPanel = Painter(self)
        self.PaintPanel.close()
        self.DrawingFrame.insertWidget(0,self.PaintPanel)
        self.DrawingFrame.setCurrentWidget(self.PaintPanel)
        # Set up Label for on hold picture
        self.label = QtWidgets.QLabel(self)
        self.label.setGeometry(QtCore.QRect(460, 70, 280, 280))
        self.pixmap = QtGui.QPixmap(image_path + str(-1) +".png")
        self.label.setPixmap(self.pixmap)
        self.Clear_Button.clicked.connect(self.ClearSlate)
        self.Predict_Button.clicked.connect(self.PredictNumber)
    # Reset Button
    def ClearSlate(self):
        self.DrawingShapes = Shapes()
        self.PaintPanel.repaint()
        self.pixmap = QtGui.QPixmap(image_path + str(-1) +".png")
        self.label.setPixmap(self.pixmap)
    # Predict Button
    def PredictNumber(self):
        image = get_image(self.DrawingFrame)
        pred = nn_predict(image=image)
        self.pixmap = QtGui.QPixmap(image_path + str(int(pred)) +".png")
        self.label.setPixmap(self.pixmap)

# Starting the GUI Application
if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    main_window = CreateUI()
    main_window.show()
    sys.exit(app.exec_())

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# coding: utf-8

import os
from os import path
import argparse
import json

import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torchvision import datasets, transforms, models

from collections import OrderedDict
import time
import numpy as np

import constructor

# Args
parser = argparse.ArgumentParser(description='Model trainer')

parser.add_argument('data_directory', action="store", type=str, help='Parent-Path to training data')
parser.add_argument('--save_dir', action="store", type=str, dest="save_dir", help='where to save checkpoints', default="checkpoint")
parser.add_argument('--arch', action="store", dest="net", default="vgg19", help="Defines NN. Possible values: 'alexnet' or 'vgg19'")
parser.add_argument('--hidden_units', action="store", dest="fc1_hidden_layers", type=int, default="4096", help="No of hidden layers")
parser.add_argument('--dropout', action="store", dest="dropout", type=float, default=0.1, help="Dropout rate (0-1). Less should be better")
parser.add_argument('--gpu', action="store_true", dest="cuda", help="Calc on GPU. Should be way faster")
parser.add_argument('--batchsize', action="store", dest="batchsize", type=int, default=64, help="Batchsize, default is 64")
parser.add_argument('--epochs', action="store", dest="epochs", type=int, default=7, help="Nr. of epochs to train. 7 should be sufficient")
parser.add_argument('--lr', action="store", dest="learning_rate", type=float, default=0.001, help="Learning rate, default 0.001")

results = parser.parse_args()

data_dir = results.data_directory
train_dir = data_dir + '/train'
valid_dir = data_dir + '/valid'
test_dir = data_dir + '/test'

folders = 0
for _, dirnames, _ in os.walk(train_dir):
    folders += len(dirnames)
num_labels = folders

checkpoint_dir = results.save_dir

try:
    if True:
        path.exists(checkpoint_dir)
        print("Path to checkpoint already exits ", checkpoint_dir)
except:
    try:
        os.mkdir(checkpoint_dir)
    except OSError:
        print ("Creation of the directory %s failed" % checkpoint_dir)
    else:
        print ("Successfully created the directory %s " % checkpoint_dir)

net = results.net
if net == "vgg19":
    fc1_in_size = 25088
elif net == "alexnet":
    fc1_in_size = 9216

dropout = results.dropout

if results.cuda == True:
    machine = 'cuda'
else:
    machine = 'cpu'

batchsize = results.batchsize
fc1_hidden_layers = results.fc1_hidden_layers
epochs = results.epochs
learning_rate = results.learning_rate

print("\r\nTraining parameter: ",
        "\r\n",
        "Neural net: \t", net,
        "\r\n",
        "Dropout: \t", dropout,
        "\r\n",
        "Calculating on: \t", machine,
        "\r\n",
        "Batch size: \t", batchsize,
        "\r\n",
        "Epochs to train: \t", epochs)

# Building
train_transforms = transforms.Compose ([
                            transforms.RandomRotation(30),
                            transforms.RandomResizedCrop(224),
                            transforms.RandomHorizontalFlip(),
                            transforms.ToTensor(),
                            transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
                                            ])

valid_and_test_transforms = transforms.Compose ([
                            transforms.Resize(255),
                            transforms.CenterCrop(224),
                            transforms.ToTensor(),
                            transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
                                            ])

train_datasets = datasets.ImageFolder(train_dir, transform = train_transforms)
valid_image_datasets = datasets.ImageFolder (valid_dir, transform = valid_and_test_transforms)
test_image_datasets = datasets.ImageFolder (test_dir, transform = valid_and_test_transforms)

train_loader = torch.utils.data.DataLoader(train_datasets, batch_size=batchsize, shuffle=True)#, drop_last=True)#, pin_memory=True)#,num_workers=10)
valid_loader = torch.utils.data.DataLoader(valid_image_datasets, batch_size=64, shuffle=True)#, drop_last=True)#, pin_memory=True)#,num_workers=10)
test_loader = torch.utils.data.DataLoader(test_image_datasets, batch_size=64)#, shuffle=True, drop_last=True)#, pin_memory=True)#,num_workers=10)

# Categories
#with open('cat_to_name.json', 'r') as f:
#    cat_to_name = json.load(f)

model, optimizer, criterion = constructor.constructor(net, dropout, fc1_hidden_layers, fc1_in_size, learning_rate, num_labels)

# Defining validation
def validation(model, valid_loader, criterion):
    model.to(machine)

    valid_loss = 0
    accuracy = 0
    for inputs, labels in valid_loader:

        inputs, labels = inputs.to(machine), labels.to(machine)
        output = model.forward(inputs)
        valid_loss += criterion(output, labels).item()

        ps = torch.exp(output)
        equality = (labels.data == ps.max(dim=1)[1])
        accuracy += equality.type(torch.FloatTensor).mean()

    return valid_loss, accuracy

# Training model
model.to(machine)
stepcounter = 40
epochcounter = 0
steps = 0
step_runtime = 0
runtime = 0

print("\r\nTraining model...\tEpoch count: ", epochs)

for e in range (epochs):

    time_per_epoch_start = time.time()

    running_loss = 0
    for ii, (inputs, labels) in enumerate(train_loader):

        time_per_step_start = time.time()

        steps += 1

        inputs, labels = inputs.to(machine), labels.to(machine)

        optimizer.zero_grad()

        # Forward and backward passes
        outputs = model.forward(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()

        if steps % stepcounter == 0:
            model.eval() #switching to evaluation mode, dropout off

            # Turn off gradients for validation
            with torch.no_grad():
                valid_loss, accuracy = validation(model, valid_loader, criterion)

            print("Epoch: {}/{} | ".format(e+1, epochs),
                      "Training Loss: {:.3f} | ".format(running_loss/stepcounter),
                      "Valid Loss: {:.3f} | ".format(valid_loss/len(valid_loader)),
                      "Valid Accuracy: {:.3f}%".format(accuracy/len(valid_loader)*100))

            running_loss = 0

            model.train()

        time_per_step_end = time.time()
        step_runtime += time_per_step_end - time_per_step_start

    epochcounter += 1
    time_per_epoch_end = time.time()
    runtime += time_per_epoch_end - time_per_epoch_start

print("Total Runtime: {:.1f} min".format(runtime / 60),
            "Runtime per Step: {:.1f} sec".format(step_runtime / stepcounter),
            "Runtime per Epoch: {:.1f} sec".format(runtime / epochcounter))

# Do validation on the test set
def check_accuracy_on_test(test_loader,model):
    correct = 0
    total = 0
    model.to(machine)
    with torch.no_grad():
        for inputs, labels in test_loader:
            inputs, labels = inputs.to(machine), labels.to(machine)
            outputs = model(inputs)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    print('Accuracy on test images: %d %%' % (100 * correct / total))
check_accuracy_on_test(test_loader, model)

# Save the checkpoint
print("Saving checkpoint...")

model.class_to_idx = test_image_datasets.class_to_idx

torch.save({    'net':net,
                'dropout':dropout,
                'fc1_hidden_layers':fc1_hidden_layers,
                'fc1_in_size':fc1_in_size,
                'learning_rate':learning_rate,
                'num_labels':num_labels,
                'state_dict':model.state_dict(),
                'class_to_idx':model.class_to_idx},
                checkpoint_dir + '/checkpoint.pth')

print("Checkpoint saved! ", checkpoint_dir)


text:
import argparse
import time
import numpy as np
import os
import torch
from pprint import pprint
from dataloader import DataLoaderPolyvore
from model import CompatibilityGAE
from utils.misc import compute_degree_support, normalize_nonsym_adj, support_dropout, csr_to_sparse_tensor
from copy import deepcopy

# set random seed
# seed = int(time.time())  # 12342
seed = 1234
np.random.seed(seed)
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

parser = argparse.ArgumentParser(description='visual compatibility')
parser.add_argument('-d', '--dataset', type=str, default='polyvore', choices=['polyvore', 'fashiongen'])
parser.add_argument('-lr', '--learning-rate', type=float, default=0.001, help='learning rate')
parser.add_argument('-wd', '--weight-decay', type=float, default=0., help='weight decay')
parser.add_argument('-e', '--epochs', type=int, default=4000, help='Number of training epochs')
parser.add_argument('-hi', '--hidden', type=int, nargs='+', default=[350, 350, 350], help='Number of hidden units in the GCN layers.')
parser.add_argument('-drop', '--dropout', type=float, default=0.5, help='dropout probability')
parser.add_argument('-deg', '--degree', type=int, default=1, help='degree of convolution, i.e., number of support nodes.')
parser.add_argument('-sd', '--support-dropout', type=float, default=0.15, help='Use dropout on support adjacency matrix, dropping all the connections from some nodes')
parser.add_argument('--data-dir', type=str,
                    default='/home/alan/Downloads/fashion/polyvore/dataset2')
parser.add_argument('--save-path', type=str,
                    default='/home/alan/Downloads/fashion/polyvore/')
parser.add_argument('--device', type=str, default='cuda:0')

mg = parser.add_mutually_exclusive_group(required=False)
mg.add_argument('-bn', '--batch-norm', action='store_true',
                help='Option to turn on batchnorm in GCN layers')
mg.add_argument('-no-bn', '--no-batch-norm', action='store_false',
                help='Option to turn off batchnorm in GCN layers')
parser.set_defaults(batch_norm=True)

args = parser.parse_args()
args = vars(args)

print("\nSetting: \n")
for key, val in args.items():
    print("{}: {}".format(key, val))
print()

# Define parameters
DATASET = args['dataset']
NB_EPOCHS = args['epochs']
DROP = args['dropout']
HIDDEN = args['hidden']
LR = args['learning_rate']
NUM_CLASSES = 2  # each edge is either connected or not
DEGREE = args['degree']
BATCH_NORM = args['batch_norm']
SUP_DROP = args['support_dropout']
ADJ_SELF_CONNECTIONS = True
DATA_DIR = args['data_dir']
WD = args['weight_decay']
DEVICE = args['device']

# prepare dataset
if DATASET in ('polyvore', 'fashiongen'):
    if DATASET == 'polyvore':
        dl = DataLoaderPolyvore(DATA_DIR)
    else:
        raise NotImplementedError('Support to fashiongen dataset will be added soon!')

    # node features, message-passing adj, ground-truth labels, start node idx, end node idx of edges to evaluate loss
    train_features, train_mp_adj, train_labels, train_row_idx, train_col_idx = dl.get_phase('train')
    val_features, val_mp_adj, val_labels, val_row_idx, val_col_idx = dl.get_phase('valid')

    # normalize features
    train_features, mean, std = dl.normalize_feature(train_features, return_moments=True)
    val_features = dl.normalize_feature(val_features, mean=mean, std=std, return_moments=False)

else:
    raise NotImplementedError('Dataloader for dataset {} is not supported yet!'.format(DATASET))

# convert features to tensors
train_features = torch.from_numpy(train_features).to(DEVICE)
val_features = torch.from_numpy(val_features).to(DEVICE)
train_labels = torch.from_numpy(train_labels).float().to(DEVICE)
val_labels = torch.from_numpy(val_labels).float().to(DEVICE)

train_row_idx = torch.from_numpy(train_row_idx).long().to(DEVICE)
train_col_idx = torch.from_numpy(train_col_idx).long().to(DEVICE)
val_row_idx = torch.from_numpy(val_row_idx).long().to(DEVICE)
val_col_idx = torch.from_numpy(val_col_idx).long().to(DEVICE)

# get support adjacency matrix [A0, ..., AS]
train_support = compute_degree_support(train_mp_adj, DEGREE, adj_self_connections=ADJ_SELF_CONNECTIONS)
val_support = compute_degree_support(val_mp_adj, DEGREE, adj_self_connections=ADJ_SELF_CONNECTIONS)

# normalize these support adjacency matrices, except the first one which is symmetric
for i in range(1, len(train_support)):
    train_support[i] = normalize_nonsym_adj(train_support[i])
    val_support[i] = normalize_nonsym_adj(val_support[i])

val_support = [csr_to_sparse_tensor(adj).to(DEVICE) for adj in val_support]

num_supports = len(train_support)

settings = {
    'num_support': num_supports,
    'dropout': DROP,
    'batch_norm': BATCH_NORM,
    'learning_rate': LR,
    'wd': WD,
}

# create model
model = CompatibilityGAE(
    input_dim=train_features.shape[1],
    hidden=HIDDEN,
    num_classes=2,
    settings=settings
)
model.to(DEVICE)
model.train()

print("\nModel: ")
print(model)

best_val_acc = 0
best_epoch = 0
best_val_train_acc = 0
best_train_acc = 0

tmp = csr_to_sparse_tensor(train_support[0]).to(DEVICE)
for epoch in range(NB_EPOCHS):
    if SUP_DROP > 0:
        # do not modify the first support adj matrix, which is self-connections
        epoch_train_supports = [tmp] # do not miss the first self-connection edges
        for i in range(1, len(train_support)):
            sampled_adj = support_dropout(train_support[i], SUP_DROP, drop_edge=True)
            # binarilize the support adjacency matrix
            sampled_adj.data[...] = 1
            sampled_adj = normalize_nonsym_adj(sampled_adj)
            # convert adj sparse matrix to sparse tensor
            sampled_adj = csr_to_sparse_tensor(sampled_adj).to(DEVICE)
            # update message passing graph feed to the model
            epoch_train_supports.append(sampled_adj)
    else:
        epoch_train_supports = [csr_to_sparse_tensor(adj).to(DEVICE) for adj in train_support]
    # run one epoch
    train_avg_loss, train_acc = model.train_epoch(train_features, epoch_train_supports, train_row_idx, train_col_idx, train_labels)
    with torch.no_grad():
        model.eval()
        valid_pred = model.predict(val_features, val_support, val_row_idx, val_col_idx)
        valid_acc = model.accuracy(valid_pred, val_labels)
    model.train()

    print("Epoch: {} -- Train Loss: {:.4f} -- Train Acc: {:.4f} -- Valid Acc: {:.4f}".format(epoch, train_avg_loss, train_acc, valid_acc))

    if valid_acc > best_val_acc:
        best_val_acc = valid_acc
        best_epoch = epoch
        best_val_train_acc = train_acc
        # save model
        torch.save(
            {
                'state_dict': model.state_dict(),
                'args': args
            },
            os.path.join(args['save_path'], 'best_model.pth')
        )

    if train_acc > best_train_acc:
        best_train_acc = train_acc

print("Training Done!")
print("Best Epoch: {} -- Best Valid Acc: {:.4f} -- Best Train Acc at epoch {}: {:.4f} -- Best Overall Train Acc: {:.4f}".format(
    best_epoch, best_val_acc, best_epoch, best_val_train_acc, best_train_acc))

max_stars_repo_name: AlenUbuntu/Fashion-AI

max_issues_repo_name: AlenUbuntu/Fashion-AI

max_forks_repo_name: AlenUbuntu/Fashion-AI


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Jul  8 15:07:53 2019

@author: MerrillN
"""
import pandas as pd
import numpy as np
import datetime as dt

report_date = dt.datetime(2020, 4, 30)
lgs_filepath = 'U:/CIO/#Investment_Report/Data/input/returns/returns_2020-04-30.csv'
jpm_returns_filepath = 'U:/CIO/#Data/input/jpm/performance/2020/04/Historical Time Series - Monthly - Main Returns.xlsx'
jpm_benchmarks_filepath = 'U:/CIO/#Data/input/jpm/performance/2020/04/Historical Time Series - Monthly - Main Benchmarks.xlsx'
dict_filepath = 'U:/CIO/#Investment_Report/Data/input/link/Data Dictionary V3.xlsx'
tolerance = 0.01

# Imports LGS dictionary
df_dict = pd.read_excel(
        pd.ExcelFile(dict_filepath),
        sheet_name='Sheet1'
        )

benchmark_list = list(set(df_dict['Associated Benchmark']))

# Imports the LGS return file
df_lgs = pd.read_csv(
        lgs_filepath,
        index_col='Date',
        parse_dates=['Date'],
        infer_datetime_format=True,
        float_precision='round_trip'
        )

df_lgs = df_lgs.transpose()
df_lgs = df_lgs.reset_index(drop=False)
df_lgs = df_lgs.rename(columns={'index': 'Manager'})
df_lgs = pd.melt(df_lgs, id_vars=['Manager'], value_name='LGS Benchmark')
df_lgs = df_lgs.sort_values(['Manager', 'Date'])
df_lgs = df_lgs.reset_index(drop=True)

df_lgs_returns = df_lgs[~df_lgs['Manager'].isin(benchmark_list)]
df_lgs_benchmarks = df_lgs[df_lgs['Manager'].isin(benchmark_list)]

# Imports the JPM return file
use_managerid = [0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 12]
use_accountid = [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12]

# use_managerid = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11]
# use_accountid = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 14]
footnote_rows = 28

df_jpm_returns = pd.read_excel(
        pd.ExcelFile(jpm_returns_filepath),
        sheet_name='Sheet1',
        skiprows=use_accountid,
        skipfooter=footnote_rows,
        header=1
        )

df_jpm_returns = df_jpm_returns.rename(columns={'Unnamed: 0': 'Date'})
df_jpm_returns = df_jpm_returns.set_index('Date')
df_jpm_returns = df_jpm_returns.transpose()
df_jpm_returns = df_jpm_returns.reset_index(drop=False)
df_jpm_returns = df_jpm_returns.rename(columns={'index': 'Manager'})
df_jpm_returns = pd.melt(df_jpm_returns, id_vars=['Manager'], value_name='JPM Return')
df_jpm_returns = df_jpm_returns.sort_values(['Manager', 'Date'])
df_jpm_returns = df_jpm_returns.reset_index(drop=True)

df_jpm_returns = df_jpm_returns.replace('-', np.NaN)
df_jpm_returns['JPM Return'] = df_jpm_returns['JPM Return']/100

df_jpm_benchmarks = pd.read_excel(
        pd.ExcelFile(jpm_benchmarks_filepath),
        sheet_name='Sheet1',
        skiprows=use_accountid,
        skipfooter=footnote_rows,
        header=1
        )

df_jpm_benchmarks = df_jpm_benchmarks.rename(columns={'Unnamed: 0': 'Date'})
df_jpm_benchmarks = df_jpm_benchmarks.set_index('Date')
df_jpm_benchmarks = df_jpm_benchmarks.transpose()
df_jpm_benchmarks = df_jpm_benchmarks.reset_index(drop=False)
df_jpm_benchmarks = df_jpm_benchmarks.rename(columns={'index': 'Manager'})
df_jpm_benchmarks = pd.melt(df_jpm_benchmarks, id_vars=['Manager'], value_name='JPM Benchmark')
df_jpm_benchmarks = df_jpm_benchmarks.sort_values(['Manager', 'Date'])
df_jpm_benchmarks = df_jpm_benchmarks.reset_index(drop=True)

df_jpm_benchmarks = df_jpm_benchmarks.replace('-', np.NaN)
df_jpm_benchmarks['JPM Benchmark'] = df_jpm_benchmarks['JPM Benchmark']/100

# Merges the LGS time series with the LGS data dictionary
# df_lgs_dict = pd.merge(
#         left=df_lgs,
#         right=df_dict,
#         left_on=['Manager'],
#         right_on=['LGS Data'],
#         how='inner'
#         )

df_jpm = pd.merge(
        left=df_jpm_returns,
        right=df_jpm_benchmarks,
        left_on=['Manager', 'Date'],
        right_on=['Manager', 'Date'],
        how='inner'
        )

df_jpm_dict = pd.merge(
        left=df_jpm,
        right=df_dict,
        left_on=['Manager'],
        right_on=['JPM Code'],
        how='inner'
        )

# Diagnostics
# s1 = set(df_lgs_dict['LGS Data'])
# s2 = set(df_jpm_dict['LGS Data'])
# print('These managers are in the LGS file but not in the JPM file file:\n', s1 - s2)

df_lgs_jpm = pd.merge(
        left=df_lgs_benchmarks,
        right=df_jpm_dict,
        left_on=['Manager', 'Date'],
        right_on=['Associated Benchmark', 'Date'],
        how='inner'
        )

df_lgs_jpm['Deviation'] = df_lgs_jpm['JPM Benchmark'] - df_lgs_jpm['LGS Benchmark']
df_lgs_jpm['Deviation_ABS'] = abs(df_lgs_jpm['JPM Benchmark'] - df_lgs_jpm['LGS Benchmark'])
df_lgs_jpm['In Tolerance'] = df_lgs_jpm['Deviation_ABS'] <= tolerance

# Selects columns
select_columns = [
        'Manager_x',
        'Manager_y',
        'JPM Data',
        'Date',
        'JPM Return',
        'JPM Benchmark',
        'LGS Benchmark',
        'Deviation',
        'Deviation_ABS',
        'In Tolerance'
        ]

df_final = df_lgs_jpm[select_columns]

fix_back_fill_list = []
jpm_back_fill_list = []
lgs_back_fill_list = []
for i in range(0, len(df_final)):
    if pd.isna(df_final['JPM Benchmark'][i]) and pd.isna(df_final['LGS Benchmark'][i]):
        fix_back_fill_list.append(0)
        jpm_back_fill_list.append(0)
        lgs_back_fill_list.append(0)
    elif (pd.isna(df_final['JPM Benchmark'][i]) or df_final['JPM Benchmark'][i] == 0) and not (pd.isna(df_final['LGS Benchmark'][i] or df_final['LGS Benchmark'][i] == 0)):
        fix_back_fill_list.append(1)
        jpm_back_fill_list.append(1)
        lgs_back_fill_list.append(0)
    elif not (pd.isna(df_final['JPM Benchmark'][i]) or df_final['JPM Benchmark'][i] == 0) and (pd.isna(df_final['LGS Benchmark'][i] or df_final['LGS Benchmark'][i] == 0)):
        fix_back_fill_list.append(1)
        jpm_back_fill_list.append(0)
        lgs_back_fill_list.append(1)
    elif (df_final['JPM Benchmark'][i] == 0) or (df_final['LGS Benchmark'][i] == 0):
        fix_back_fill_list.append(1)
        jpm_back_fill_list.append(0)
        lgs_back_fill_list.append(0)
    else:
        fix_back_fill_list.append(0)
        jpm_back_fill_list.append(0)
        lgs_back_fill_list.append(0)

df_final['Fix Needs Backfill'] = fix_back_fill_list
df_final['JPM Needs Backfill'] = jpm_back_fill_list
df_final['LGS Needs Backfill'] = lgs_back_fill_list

df_final = df_final.sort_values(['JPM Data', 'Date'])
df_final = df_final[df_final['Date'] >= dt.datetime(2012, 1, 31)].reset_index(drop=True)
df_final_filtered = df_final[~(df_final['JPM Return'].isin([np.nan]))].reset_index(drop=True)

df_final_today_not_filtered = df_final[df_final['Date'].isin([report_date])].reset_index(drop=True)
df_final_today_not_filtered = df_final_today_not_filtered[df_final_today_not_filtered['In Tolerance'].isin([0])].reset_index(drop=True)

df_final_today_filtered = df_final_filtered[df_final_filtered['Date'].isin([report_date])].reset_index(drop=True)
df_final_today_filtered = df_final_today_filtered[df_final_today_filtered['In Tolerance'].isin([0])].reset_index(drop=True)

df_final_deviants_not_filtered = df_final[df_final['In Tolerance'].isin([0])].reset_index(drop=True)

df_final_deviants_filtered = df_final_filtered[df_final_filtered['In Tolerance'].isin([0])].reset_index(drop=True)

df_final_backfill = df_final[df_final['Fix Needs Backfill'].isin([1])].reset_index(drop=True)

# Ask Merrill before touching below this line..................................
# Outputs to csv file
df_final_deviants_not_filtered.to_csv('U:/CIO/#Investment_Report/Data/output/verification/verification_benchmarks_202004_history_not_filtered.csv', index=False)
df_final_deviants_filtered.to_csv('U:/CIO/#Investment_Report/Data/output/verification/verification_benchmarks_202004_history_filtered.csv', index=False)
df_final_today_not_filtered.to_csv('U:/CIO/#Investment_Report/Data/output/verification/verification_benchmarks_202004_today_not_filtered.csv', index=False)
df_final_today_filtered.to_csv('U:/CIO/#Investment_Report/Data/output/verification/verification_benchmarks_202004_today_filtered.csv', index=False)

max_stars_repo_name: ginkgodango/lgs

max_issues_repo_name: ginkgodango/lgs

max_forks_repo_name: ginkgodango/lgs


text:
"""Management of header blocks in .mff binary files

.mff binary files have a blocked structure.  Consecutive blocks can be
separated by a header, which brings us to the topic of this module.

The header consists of either a single flag (`flag=0`) or a block describing
the following bytes of signal data (`flag=1`).  Regardless, the flag is 32-bit
wide.

This module adds functionality to read and write these header blocks.

**Header block structure**

+-------------+-------------+---------------------------------------+
| start byte  |  end byte   |              description              |
+-------------+-------------+---------------------------------------+
| 0           | 4           | header flag, if 1, header present     |
| 4           | 8           | bytes in header := `hb`               |
| 8           | 12          | bytes in data blob w/out header       |
| 12          | 16          | channel count := `nc`                 |
| 16          | 16 + 4 * nc | per-channel byte offset               |
| 16 + 4 * nc | 16 + 8 * nc | per-channel frequency and byte depths |
| 16 + 8 * nc | hb          | optional header bytes                 |
+-------------+-------------+---------------------------------------+

Optional header bytes are described in "./optional_header_block.py"
"""

from typing import Optional, Tuple
from collections import namedtuple

import numpy as np

from .helpers import FileLike, read, skip, write
from . import optional_header_block as opt

HEADER_BLOCK_PRESENT = 1

_HeaderBlock = namedtuple('_HeaderBlock', [
    'header_size',
    'block_size',
    'num_channels',
    'num_samples',
    'sampling_rate',
    'optional_header'
])

class HeaderBlock(_HeaderBlock):

    def __new__(cls,
                block_size: int,
                num_channels: int,
                num_samples: int,
                sampling_rate: int,
                header_size: Optional[int] = None,
                optional_header: opt.BlockTypes = opt.NoOptHeaderBlock()):
        """create new HeaderBlock instance

        Parameters
        ----------
        block_size : byte size of the block
        num_channels : channel count in the block
        num_samples : sample count per channel in the block
        sampling_rate : sampling_rate per channel in the block
        header_size : byte size of the header (computed if None)
        optional_header : optional header with additional fields
        """
        computed_size = cls.compute_byte_size(num_channels, optional_header)
        if header_size and header_size != computed_size:
            raise ValueError(f"""header of inconsistent size:
            {header_size} != {computed_size}""")

        header_size = computed_size
        return super().__new__(cls, header_size, block_size, num_channels,
                               num_samples, sampling_rate, optional_header)

    @classmethod
    def from_file(cls, fp: FileLike):
        """return HeaderBlock, read from fp"""

        # Each block starts with a 4-byte-long header flag which is
        # * `0`: there is no header
        # * `1`: it follows a header
        if read(fp, 'i') == 0:
            return None
        # Read general information
        header_size, block_size, num_channels = read(fp, '3i')
        # number of 4-byte samples per channel in the data block
        num_samples = (block_size//num_channels) // 4
        # Read channel-specific information
        # Skip byte offsets
        skip(fp, 4 * num_channels)
        # Sample rate/depth: Read one skip, over the rest
        # We also check that depth is always 4-byte floats (32 bit)
        sampling_rate, depth = cls.decode_rate_depth(read(fp, 'i'))
        skip(fp, 4 * (num_channels - 1))
        assert depth == 32, f"""
        Unable to read MFF with `depth != 32` [`depth={depth}`]"""
        optional_header = opt.from_file(fp)
        return cls(
            block_size=block_size,
            header_size=header_size,
            num_samples=num_samples,
            num_channels=num_channels,
            sampling_rate=sampling_rate,
            optional_header=optional_header,
        )

    def write(self, fp: FileLike):
        """write HeaderBlock to file pointer `fp`"""
        write(fp, '4i', (
            HEADER_BLOCK_PRESENT,
            self.header_size,
            self.block_size,
            self.num_channels
        ))
        num_samples = (self.block_size//self.num_channels) // 4
        # Write channel offset into the data block
        arr = 4 * num_samples * np.arange(self.num_channels).astype(np.int32)
        fp.write(arr.tobytes())
        # write sampling-rate/depth word
        sr_d = self.encode_rate_depth(self.sampling_rate, 32)
        arr = sr_d * np.ones(self.num_channels, dtype=np.int32)
        fp.write(arr.tobytes())
        self.optional_header.write(fp)

    @staticmethod
    def decode_rate_depth(x: int) -> Tuple[int, int]:
        """return rate and depth from encoded representation"""
        rate = x >> 8
        depth = x & 0xff
        return rate, depth

    @staticmethod
    def encode_rate_depth(rate: int, depth: int) -> int:
        """return joined rate and byte depth of samples

        Sampling rate and sample depth are encoded in a single 4-byte integer.
        The first byte is the depth the last 3 bytes give the sampling rate.
        """
        assert depth < (
            1 << 8), f"depth must be smaller than 256 (got {depth})"
        assert rate < (
            1 << 24), f"depth must be smaller than {1<<24} (got {rate})"
        return (rate << 8) + depth

    @staticmethod
    def compute_byte_size(num_channels: int,
                          optional_header: opt.BlockTypes) -> int:
        """returns sum of header byte size and optional header size

        `(5 + ..)`: The 4-byte int of the optional header byte size constitutes
        the "5", not in `optional_header.byte_size`.  See the file description
        for detailed infos on all bytes.
        """
        return 4 * (5 + 2 * num_channels) + optional_header.byte_size

max_stars_repo_name: BEL-Public/mffpy

max_issues_repo_name: BEL-Public/mffpy

max_forks_repo_name: BEL-Public/mffpy


text:
"""
This module tests element_agent.py

"""
import numpy as np
import sys
# Check the version of Python
is_py2 = sys.version[0] == '2'
if is_py2:
    import Queue as queue
else:
    import queue as queue
import threading

import unittest

# agent and stream and helper_control are in ../core
from IoTPy.core.agent import Agent
from IoTPy.core.stream import Stream, StreamArray, _no_value, _multivalue, run
from IoTPy.core.helper_control import _close
# recent_values are in ../helper_functions
from IoTPy.helper_functions.recent_values import recent_values
from IoTPy.helper_functions.print_stream import print_stream
# op is in ../agent_types
from IoTPy.agent_types.op import map_element, map_element_f
from IoTPy.agent_types.op import filter_element, filter_element_f
from IoTPy.agent_types.op import map_list, map_list_f
from IoTPy.agent_types.op import timed_window
from IoTPy.agent_types.basics import fmap_e, map_e, sink_e, merge_e, merge_asynch
from IoTPy.agent_types.basics import merge_sink_e

#------------------------------------------------------------------------------------------------
#                                     A SIMPLE EXAMPLE TEST
#------------------------------------------------------------------------------------------------
# This example is to illustrate the steps in the test.
# The later examples test several agents whereas this simple
# test only tests a single agent.
# The seven steps in this test may occur in different orders
# in the later tests.

class test_element(unittest.TestCase):

    def test_example_1(self):
        # Specify streams
        x = Stream('x')
        y = Stream('y')
        # Specify encapsulated functions (if any)
        def f(v): return 2*v
        # Specify agents.
        map_element(func=f, in_stream=x, out_stream=y)

        # Execute a step
        # Put test values in the input streams.
        x.extend(list(range(3)))
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(y) == [0, 2, 4]

        # Execute a step
        # Put test values in the input streams.
        x.extend([10, 20, 30])
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(y) == [0, 2, 4, 20, 40, 60]

        # Execute a step
        # Put test values in the input streams.
        x.extend([0, -10])
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(y) == [0, 2, 4, 20, 40, 60, 0, -20]

    def test_example_2(self):
        # Specify streams
        x = Stream('x')
        y = Stream('y')
        # Specify encapsulated functions (if any)
        def f(v): return v < 3
        # Specify agents.
        filter_element(func=f, in_stream=x, out_stream=y)

        # Execute a step
        # Put test values in the input streams.
        x.extend(list(range(5)))
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(y) == [0, 1, 2]

    def test_example_3(self):
        # Specify streams
        x = Stream('x')
        y = Stream('y')
        # Specify encapsulated functions (if any)
        def f(v, state):
            final, prefinal = state
            next_output = final + prefinal
            # In the next state:
            # prefinal becomes final
            # final becomes next_output
            next_state = next_output, final
            return next_output, next_state
        def g(v, divisor):
            if v % divisor == 0:
                return _no_value
            else:
                return v
        # Specify agents.
        map_element(func=f, in_stream=y, out_stream=x, state=(0, 1))
        map_element(func=g, in_stream=x, out_stream=y, divisor=4)
        # Execute a step
        # Put test values in the input streams.
        y.append(0)
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(x) == [1, 1, 2, 3, 5, 8]

        # Execute a step
        # Put test values in the input streams.
        y.append(0)
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(x) == \
          [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]

    def test_example_4(self):
        # Illustrates a cycle of agents and also shows use
        # of a class within a wrapper.
        # Specify network: streams, functions, agents
        # (a) Specify streams
        x = Stream('x')
        y = Stream('y')
        # (b) Specify encapsulated functions (if any)
        def f(v, state):
            final, prefinal = state
            next_output = final + prefinal
            # In the next state:
            # prefinal becomes final
            # final becomes next_output
            next_state = next_output, final
            return next_output, next_state

        class G(object):
            def __init__(self):
                self.divisor = 4
            def g(self, v):
                if v % self.divisor == 0:
                    return _no_value
                else:
                    return v

        # (c) Specify agents.
        encapsulator = G()
        map_element(func=f, in_stream=y, out_stream=x, state=(0, 1))
        map_element(func=encapsulator.g, in_stream=x, out_stream=y)

        # Drive the network in steps.
        # Execute a step
        # Put test values in the input streams.
        y.append(0)
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(x) == [1, 1, 2, 3, 5, 8]

        # Execute a step after changing agent parameters
        encapsulator.divisor = 2
        # Put test values in the input streams.
        y.append(0)
        # Execute a step
        run()
        # Look at recent values of output streams.
        assert recent_values(x) == \
          [1, 1, 2, 3, 5, 8, 13, 21, 34]

    def test_example_5(self):
        # Fibonacci
        # Illustrates use of a dict to save state.
        # Specify network: streams, functions, agents
        # (a) Specify streams
        x = Stream('x')
        y = Stream('y')
        s = {'a':0, 'b':1}
        # (b) Specify encapsulated functions (if any)
        def f(v, s):
            final, prefinal = s['a'], s['b']
            post_final = final + prefinal
            # In the next state:
            # prefinal becomes final
            # final becomes next_output
            s['a'], s['b'] = post_final, final
            return final
        map_element(f, x, y, s=s)
        x.extend(list(range(10)))
        run()
        assert recent_values(y) == [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]

    def test_example_6(self):
        import numpy as np
        # Fibonacci
        # Illustrates use of a dict to save state.
        # Specify network: streams, functions, agents
        # (a) Specify streams
        x = Stream('x')
        y = Stream('y')
        s = {'final':0, 'prefinal': 1}
        # (b) Specify encapsulated functions (if any)
        def f(v, s):
            post_final = s['final'] + s['prefinal']
            # In the next state:
            # prefinal becomes final
            # final becomes next_output
            s['prefinal'] = s['final']
            s['final'] = post_final
            return s['prefinal']
        map_element(f, x, y, s=s)
        x.extend(list(range(10)))
        run()
        assert recent_values(y) == [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]

    def test_1(self):
        # From map_element_examples
        x = Stream('x')
        y = Stream('y')

        def f(in_stream_element):
            out_stream_element = 2*in_stream_element
            return out_stream_element
        map_element(func=f, in_stream=x, out_stream=y)

        x.extend(list(range(5)))
        run()
        assert recent_values(y) == [0, 2, 4, 6, 8]

    def test_2(self):
        # From map_element_examples
        x = Stream('x')
        y = Stream('y')

        def multiply_and_add(
                in_stream_element, multiplicand, addend):
                out_stream_element = \
                  multiplicand*in_stream_element + addend
                return out_stream_element

        map_element(func=multiply_and_add, in_stream=x, out_stream=y,
                    multiplicand=2, addend=10)
        x.extend(list(range(5)))
        run()
        assert recent_values(y) == [10, 12, 14, 16, 18]

    def test_3(self):
        # From map_element_examples
        x = Stream('x')
        y = Stream('y')

        # In this example, the output stream is the same as the input stream
        # except that only values that are less than the threshold are passed
        # through to the output stream. Here threshold is a keyword argument
        def f(in_stream_element, threshold):
            if in_stream_element < threshold:
                    out_stream_element = in_stream_element
            else:
                    out_stream_element = _no_value
            return out_stream_element
        map_element(func=f, in_stream=x, out_stream=y, threshold=5)
        x.extend(list(range(20)))
        run()
        assert recent_values(y) == [0, 1, 2, 3, 4]
        # If x is [0, 1, 2, 3, 4,....20] then y is [0, 1, 2, 3, 4]

    def test_4(self):
        # From map_element_examples
        x = Stream('x')
        y = Stream('y')
        def f(in_stream_element):
                x, y = in_stream_element
                if x > 5 and y > 5:
                        out_stream_element = _multivalue((x,y))
                elif x > 5:
                        out_stream_element = x
                elif y > 5:
                        out_stream_element = y
                else:
                        out_stream_element = _no_value
                return out_stream_element
        map_element(func=f, in_stream=x, out_stream=y)
        x.extend( [(10, 10), (2, 20), (30, 3), (4, 4), (1, 3), (60, 70)] )
        run()
        assert recent_values(y) == [10, 10, 20, 30, 60, 70]

    #------------------------------------------------------------------------------------------------
    #                                     ELEMENT AGENT TESTS
    #------------------------------------------------------------------------------------------------
    def test_element_simple(self):
        # SPECIFY STREAMS
        m = Stream('m')
        n = Stream('n')
        o = Stream('o')
        q = Stream('q')
        r = Stream('r')
        s = Stream('s')
        t = Stream('t')
        u = Stream('u')
        v = Stream('v')
        w = Stream('w')
        x = Stream('x')
        y = Stream('y')
        z = Stream('z')

        #----------------------------------------------------------------
        # Test simple map using map_element
        # func operates on an element of the input stream and returns an element of
        # the output stream.
        # SPECIFY ENCAPSULATED FUNCTIONS (IF ANY)
        def double(v): return 2*v

        # SPECIFY AGENTS
        a = map_element(func=double, in_stream=x, out_stream=y, name='a')
        ymap = map_element_f(func=double, in_stream=x)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test filtering
        def filtering(v): return v <= 2
        # yfilter is a stream consisting of those elements in stream x with
        # values less than or equal to 2.
        # The elements of stream x that satisfy the boolean, filtering(), are
        # passed through.
        yfilter = filter_element_f(func=filtering, in_stream=x)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test map with state using map_element
        # func operates on an element of the input stream and state and returns an
        # element of the output stream and the new state.
        def f(x, state):
            return x+state, state+2

        b = map_element(func=f, in_stream=x, out_stream=z, state=0, name='b')
        bmap = map_element_f(func=f, in_stream=x, state=0)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test map with call streams
        # The agent executes a state transition when a value is added to call_streams.
        c = map_element(func=f, in_stream=x, out_stream=v, state=10,
                              call_streams=[w], name='c')
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test _no_value
        # func returns _no_value to indicate that no value
        # is placed on the output stream.
        def f_no_value(v):
            """ Filters out odd values
            """
            if v%2:
                # v is odd. So filter it out.
                return _no_value
            else:
                # v is even. So, keep it in the output stream.
                return v

        no_value_stream = Stream(name='no_value_stream')
        no_value_agent = map_element(
            func=f_no_value, in_stream=x, out_stream=no_value_stream,
            name='no_value_agent')

        no_value_map = map_element_f(func=f_no_value, in_stream=x)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test _multivalue
        # func returns _multivalue(output_list) to indicate that
        # the list of elements in output_list should be placed in the
        # output stream.
        def f_multivalue(v):
            if v%2:
                return _no_value
            else:
                return _multivalue([v, v*2])

        multivalue_stream = Stream('multivalue_stream')
        multivalue_agent = map_element(
            func=f_multivalue, in_stream=x, out_stream=multivalue_stream,
            name='multivalue_agent')
        multivalue_map = map_element_f(func=f_multivalue, in_stream=x)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test map_element with args
        def function_with_args(x, multiplicand, addition):
            return x*multiplicand+addition

        ## EXPLANATION FOR agent BELOW
        ## agent_test_args = map_element(
        ##     func=function_with_args, in_stream = x, out_stream=r,
        ##     state=None, call_streams=None, name='agent_test_args',
        ##     multiplicand=2, addition=10)

        agent_test_args = map_element(
            function_with_args, x, r,
            None, None, 'agent_test_args',
            2, 10)
        stream_test_args = map_element_f(function_with_args, x, None, 2, 10)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test map_element with kwargs
        agent_test_kwargs = map_element(
            func=function_with_args, in_stream = x, out_stream=u,
            state=None, call_streams=None, name='agent_test_kwargs',
            multiplicand=2, addition=10)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test map_element with state and kwargs
        # func operates on an element of the input stream and state and returns an
        # element of the output stream and the new state.
        def f_map_args_kwargs(u, state, multiplicand, addend):
            return u*multiplicand+addend+state, state+2

        agent_test_kwargs_and_state = map_element(
            func=f_map_args_kwargs, in_stream=x, out_stream=s,
            state=0, name='agent_test_kwargs_and_state',
            multiplicand=2, addend=10)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test map_element with state and args
        aa_map_args_agent = map_element(
            f_map_args_kwargs, x, t,
            0, None, 'aa_map_args_agent',
            2, 10)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test filter_element
        def is_even_number(v):
            return not v%2
        filter_element(func=is_even_number, in_stream=x, out_stream=q)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test filter_element with state
        def less_than_n(v, state):
            return v <= state, state+1
        x0 = Stream('x0')
        q0 = Stream('q0')
        # state[i] = i
        # Pass through elements in x0 where x0[i] <= state[i]
        filter_element(
            func=less_than_n, in_stream=x0, out_stream=q0, state=0)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test filter_element_stream
        # p is a stream consisting of odd-numbered elements of x
        # Even-numbered elements are filtered out.
        p = filter_element_f(is_even_number, x)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # Test cycles in the module connection graph
        filter_element(func=lambda v: v <= 5, in_stream=o, out_stream=n)
        map_element(func=lambda v: v+2, in_stream=n, out_stream=o)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        # PUT TEST VALUES INTO INPUT STREAMS
        #----------------------------------------------------------------
        #   Put test values into streams x, x0 and n.
        x.extend(list(range(3)))
        x0.extend([0, 1, 3, 3, 6, 8])
        n.append(0)

        # STEP 6: EXECUTE A STEP OF THE SCHEDULER
        run()

        # STEP 7: LOOK AT OUTPUT STREAMS
        assert recent_values(x) == [0, 1, 2]
        assert recent_values(y) == [0, 2, 4]
        assert recent_values(q0) == [0, 1, 3]
        assert recent_values(ymap) == recent_values(y)
        assert recent_values(yfilter) == [0, 1, 2]
        assert recent_values(z) == [0, 3, 6]
        assert recent_values(bmap) == recent_values(z)
        assert recent_values(v) == []
        assert recent_values(no_value_stream) == [0, 2]
        assert recent_values(no_value_map) == recent_values(no_value_stream)
        assert recent_values(multivalue_stream) == [0, 0, 2, 4]
        assert recent_values(multivalue_map) == recent_values(multivalue_stream)
        assert recent_values(r) == [10, 12, 14]
        assert recent_values(stream_test_args) == recent_values(r)
        assert recent_values(u) == recent_values(r)
        assert recent_values(s) == [10, 14, 18]
        assert recent_values(s) == recent_values(t)
        assert recent_values(q) == [0, 2]
        assert recent_values(q) == recent_values(p)
        assert recent_values(n) == [0, 2, 4]
        assert recent_values(o) == [2, 4, 6]
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        x.extend(list(range(3, 5, 1)))
        run()
        assert recent_values(x) == [0, 1, 2, 3, 4]
        assert recent_values(y) == [0, 2, 4, 6, 8]
        assert recent_values(ymap) == recent_values(y)
        assert recent_values(yfilter) == [0, 1, 2]
        assert recent_values(z) == [0, 3, 6, 9, 12]
        assert recent_values(bmap) == recent_values(z)
        assert recent_values(no_value_stream) == [0, 2, 4]
        assert recent_values(no_value_map) == recent_values(no_value_stream)
        assert recent_values(multivalue_stream) == [0, 0, 2, 4, 4, 8]
        assert recent_values(multivalue_map) == recent_values(multivalue_stream)
        assert recent_values(r) == [10, 12, 14, 16, 18]
        assert recent_values(stream_test_args) == recent_values(r)
        assert recent_values(u) == recent_values(r)
        assert recent_values(s) == [10, 14, 18, 22, 26]
        assert recent_values(s) == recent_values(t)
        assert recent_values(q) == [0, 2, 4]
        assert recent_values(q) == recent_values(p)
        #----------------------------------------------------------------

        #----------------------------------------------------------------
        w.append(0)
        run()
        assert recent_values(x) == [0, 1, 2, 3, 4]
        assert recent_values(y) == [0, 2, 4, 6, 8]
        assert recent_values(ymap) == recent_values(y)
        assert recent_values(yfilter) == [0, 1, 2]
        assert recent_values(z) == [0, 3, 6, 9, 12]
        assert recent_values(bmap) == recent_values(z)
        assert recent_values(v) == [10, 13, 16, 19, 22]
        assert recent_values(no_value_stream) == [0, 2, 4]
        assert recent_values(no_value_map) == recent_values(no_value_stream)
        assert recent_values(multivalue_stream) == [0, 0, 2, 4, 4, 8]
        assert recent_values(multivalue_map) == recent_values(multivalue_stream)
        assert recent_values(r) == [10, 12, 14, 16, 18]
        assert recent_values(stream_test_args) == recent_values(r)
        assert recent_values(u) == recent_values(r)
        assert recent_values(s) == [10, 14, 18, 22, 26]
        assert recent_values(s) == recent_values(t)
        assert recent_values(q) == [0, 2, 4]
        assert recent_values(q) == recent_values(p)
        #----------------------------------------------------------------

        #------------------------------------------------------------------------------------------------
        #                                     ELEMENT AGENT TESTS FOR STREAM ARRAY
        #------------------------------------------------------------------------------------------------
        import numpy as np

        m = StreamArray('m')
        n = StreamArray('n')
        o = StreamArray('o')

        map_element(func=np.sin, in_stream=m, out_stream=n)
        filter_element(func=lambda v: v <= 0.5, in_stream=n, out_stream=o)
        input_array = np.linspace(0.0, 2*np.pi, 20)
        m.extend(input_array)
        run()
        expected_output = np.sin(input_array)
        assert np.array_equal(recent_values(n), expected_output)
        expected_output = expected_output[expected_output <= 0.5]
        assert np.array_equal(recent_values(o), expected_output)
        return

    def test_timed_window(self):
        x = Stream('x')
        y = Stream('y')

        def f(v): return v

        timed_window(
            func=f, in_stream=x, out_stream=y,
            window_duration=10, step_time=10)
        x.extend([(1, 'a'), (8, 'b'), (12, 'c')])
        run()
        assert(recent_values(y) == [(10, [(1, 'a'), (8, 'b')])])

        x.extend([(14, 'd'), (36, 'e'), (43, 'g'), (75, 'h')])
        run()
        assert(recent_values(y) == [(10, [(1, 'a'), (8, 'b')]),
                                     (20, [(12, 'c'), (14, 'd')]),
                                     (40, [(36, 'e')]), (50, [(43, 'g')])])

        x.extend([(79, 'i'), (101, 'j')])
        run()
        assert(recent_values(y) == [
            (10, [(1, 'a'), (8, 'b')]), (20, [(12, 'c'), (14, 'd')]),
            (40, [(36, 'e')]), (50, [(43, 'g')]), (80, [(75, 'h'), (79, 'i')])])

        return

    def test_map_list(self):
        scheduler = Stream.scheduler
        x = Stream('x')
        y = Stream('y')
        z = Stream('z')
        w = Stream('w')
        map_list(func = lambda v: v, in_stream=x, out_stream=y)
        def f(lst):
            return list(filter(lambda v: v%2, lst))
        def g(lst):
            return [v*2 if v%2 else v/2 for v in lst]
        map_list(f, x, z)
        map_list(g, x, w)

        x_values = list(range(10))
        x.extend(x_values)
        run()
        assert recent_values(y) == recent_values(x)
        assert recent_values(z) == f(x_values)
        assert recent_values(w) == g(x_values)

    def test_stream_arrays_2(self):
        """
        Example where the input stream of an agent is a stream array and
        its output stream is not a stream array.
        """
        x = StreamArray(name='x', dimension=3, dtype=float)
        y = Stream()
        map_element(func=np.median, in_stream=x, out_stream=y)
        x.append(np.array([1., 2., 3.]))
        run()
        assert y.recent[:y.stop] == [2.0]
        x.extend(np.array([[4., 5., 6.], [7., 8., 9.]]))
        run()
        assert y.recent[:y.stop] == [2.0, 5.0, 8.0]

    def test_class(self):
        class example(object):
            def __init__(self, multiplicand):
                self.multiplicand = multiplicand
                self.running_sum = 0
            def step(self, v):
                result = v * self.multiplicand + self.running_sum
                self.running_sum += v
                return result
        x = Stream()
        y = Stream()
        eg = example(multiplicand=2)
        map_element(func=eg.step, in_stream=x, out_stream=y)
        x.extend(list(range(5)))
        run()
        assert y.recent[:y.stop] == [0, 2, 5, 9, 14]

    def test_halt_agent(self):
        def double(v): return v*2
        x = Stream('x')
        y = Stream('y')
        a = map_element(func=double, in_stream=x, out_stream=y)
        x.extend(list(range(5)))
        run()
        assert recent_values(y) == [0, 2, 4, 6, 8]
        a.halt()
        run()
        assert recent_values(y) == [0, 2, 4, 6, 8]
        x.extend(list(range(10,15)))
        run()
        assert recent_values(y) == [0, 2, 4, 6, 8]
        assert recent_values(x) == list(range(5)) + list(range(10,15))
        ##
        ## # What follows is nondeterministic and so may fail
        ## # the test.
        ## a.restart()
        ## run()
        ## assert recent_values(y) == [0, 2, 4, 6, 8]
        ## run()
        ## assert recent_values(y) == [0, 2, 4, 6, 8]
        ## x.extend(list(range(100,101)))
        ## run()
        ## assert recent_values(y) == [
        ##     0, 2, 4, 6, 8, 20, 22, 24, 26, 28, 200]

    def test_initial_value(self):
        def double(v): return v*2
        x = Stream('x')
        y = Stream(name='y', initial_value=[0]*5)
        a = map_element(func=double, in_stream=x, out_stream=y)
        x.extend(list(range(5)))
        run()
        assert recent_values(y) == [0]*5 + [0, 2, 4, 6, 8]

    def test_multiple_relations(self):
        def double(v): return v*2
        def add10(v): return v+10
        x = Stream('x')
        y = Stream('y')
        z = Stream('z')
        a = map_element(func=add10, in_stream=z, out_stream=y)
        b = map_element(func=double, in_stream=x, out_stream=y)
        c = map_element(func=double, in_stream=x, out_stream=y)
        x.extend(list(range(5)))
        z.extend(list(range(100, 106)))
        run()
        ## # Nondeterministic.
        ## assert recent_values(y) == [
        ##     0, 2, 4, 6, 8, 0, 2, 4, 6, 8,
        ##     110, 111, 112, 113, 114, 115]

    def test_multiple_relations_2(self):
        @map_e
        def double(v): return v*2
        x = Stream('x', [10, 11])
        y = Stream('y')
        double(x, y)
        double(x, y)
        x.extend(list(range(5)))
        run()

        ## # Nondeterministic.
        ## assert recent_values(y) == [
        ##     0, 2, 4, 6, 8, 0, 2, 4, 6, 8,
        ##     110, 111, 112, 113, 114, 115]

    def test_multiple_functions(self):
        @fmap_e
        def double(v): return v*2
        @fmap_e
        def add10(v): return v+10
        x = Stream('x')
        y = double(x)
        y = add10(x)
        x.extend(list(range(5)))
        run()
        assert recent_values(y) == [10, 11, 12, 13, 14]

    def test_class(self):
        class C(object):
            def __init__(self):
                return
            def f(self, value):
                if value > 0:
                    return self.pos(value)
                else:
                    return self.neg(value)
            def pos(self, value):
                return value * value
            def neg(self, value):
                return value + value

        s = Stream('s')
        t = Stream('t')
        c = C()
        @map_e
        def g(v): return c.f(v)
        g(in_stream=s, out_stream=t)
        s.extend(list(range(-4, 4)))
        run()
        assert (recent_values(t) == [
            -8, -6, -4, -2, 0, 1, 4, 9])

    def test_None_in_stream(self):
        x = Stream('x', discard_None=False)
        y = Stream(name='y', discard_None=False)
        z = Stream(name='z')
        map_element(lambda v: v, x, y)
        map_element(lambda v: v, x, z)
        x.extend([0, None, 1, None, 2, _no_value, 3])
        run()
        assert (recent_values(y) == [0, None, 1, None, 2, 3])
        assert (recent_values(z) == [0, 1, 2, 3])

    def test_pass_parameter(self):
        result = []
        def f(v, state, result):
            state += v
            result.append(state)
            return v, state
        x = Stream('x')
        y = Stream('y')
        map_element(func=f, in_stream=x, out_stream=y, state=0, result=result)
        x.extend(list(range(5)))
        run()
        assert result == [0, 1, 3, 6, 10]

    def count_pos_and_non_pos(self, count, lst):
        """
        Parameters
        ----------
        count : list
           list with at least 2 elements
        lst : list
           The input list

        Return
        ------
           count[0]: number of non-positive values
              in lst
           count[1]: number of positive values in
              lst.

        """
        @sink_e
        def f(v, count):
            if v > 0: count[1] += 1
            else: count[0] += 1
        x = Stream('x')
        f(x, count=count)
        x.extend(lst)
        run()

    def test_count_pos_and_non_pos(self):
        count = [0, 0]
        lst = [-2, -1, 0, 1, 2]
        self.count_pos_and_non_pos(count, lst)
        assert count == [3, 2]

    def test_thread_1(self):
        def thread_target(q_in, q_out, finished):
            @sink_e
            def f(w): q_out.put(w*2)

            x = Stream('x')
            f(x)

            while True:
                v = q_in.get()
                if v == finished:
                    break
                x.append(v)
                run()
            return

        q_in = queue.Queue()
        q_out = queue.Queue()
        finished = 'finished'
        thr = threading.Thread(target=thread_target, args=(q_in, q_out, finished))
        thr.start()

        # Put data into input queue
        N = 5
        for i in range(N): q_in.put(i)
        q_in.put(finished)
        thr.join()

        # Assert contents of the output queue.
        output_list = []
        while not q_out.empty():
            output_list.append(q_out.get())
        assert output_list == [i*2 for i in range(N)]
        return

    def thread_target(self, q_in, q_out, streams, finished):
        name_to_stream = {}
        for s in streams:
            name_to_stream[s.name] = s

        while True:
            v = q_in.get()
            if v == finished:
                q_out.put(finished)
                break
            s_name, value = v
            s = name_to_stream[s_name]
            s.append(value)
            run()

    def test_thread_2(self):
        @merge_sink_e
        def f(list_of_elements, q_out):
                q_out.put(sum(list_of_elements))

        x = Stream('x')
        y = Stream('y')

        q_in = queue.Queue()
        q_out = queue.Queue()
        f([x, y], q_out=q_out)
        streams = [x, y]
        finished = _close
        thr = threading.Thread(target=self.thread_target,
                               args=(q_in, q_out, streams, finished))
        thr.start()

        # Put data into input queue
        q_in.put(('x', 1))
        q_in.put(('y', 100))

        q_in.put(finished)

        output = []
        while True:
            w = q_out.get()
            if w == finished:
                break
            else:
                output.append(w)
        thr.join()

        assert output == [101]

    def square_and_count_pos_and_non_pos(self, count, input_list):
        @map_e
        def f(v, count):
            if v > 0: count[1] += 1
            else: count[0] += 1
            return v*v

        x = Stream('x')
        y = Stream('y')
        f(in_stream=x, out_stream=y, count=count)
        x.extend(input_list)
        run()
        return recent_values(y)

    def test_square_and_count_pos_and_non_pos(self):
        count = [0, 0]
        input_list = [1, -1, 2, 3, -4]
        y_values = self.square_and_count_pos_and_non_pos(count, input_list)
        assert count == [2, 3]
        assert y_values == [1, 1, 4, 9, 16]

    def test_filter_number_1(self):
        input_list = list(range(-5, 5))
        in_stream = Stream('input')
        out_stream = Stream('output')
        def f(v): return v > 0
        filter_element(f, in_stream, out_stream)
        in_stream.extend(input_list)
        run()
        assert recent_values(out_stream) == list(filter(f, input_list))

    def test_filter_number_2(self):
        in_stream = Stream('input')
        out_stream = Stream('output')
        def f(v, threshold):
            # v is an element of the input stream
            # threshold is a constant
            return v > threshold
        # Create an agent by encapsulating f.
        filter_element(f, in_stream, out_stream, threshold=10)
        input_list = list(range(-20, 20))
        in_stream.extend(input_list)
        run()
        assert recent_values(out_stream) == list(range(11, 20))

    def test_filter_number_3(self):
        in_stream = Stream('input')
        out_stream = Stream('output')
        def f(v, w):
            # w is the current state.
            # v is the current element of the input stream.
            # Returns: (1) a boolean: v <= w
            # If this boolean is true then
            # v appears in the output.
            # (2) the next state, v.
            # The next state is the current input, v.
            # Keeps those elements in the stream that are
            # decreasing
            return v < w, v
         # Create an agent with an initial state of 0
        filter_element(f, in_stream, out_stream, state=0)
        input_list = [10, 9, 5, 7, 8, 12, 11, 3, 6, 9]
        in_stream.extend(input_list)
        run()
        assert recent_values(out_stream) == [9, 5, 11, 3]

    def test_filter_number_4(self):
        in_stream = Stream('input')
        out_stream = Stream('output')
        def f(v, w, increase):
            # v is an element of the input stream
            # w is the current state
            # increase is a constant parameter specified in the
            # call that creates the agent
            # return (1) a Boolean and (2) the next state.
            # In this example, the next state is the current input v.
            return v < w*increase, v

        # Create an agent by encapsulating function f. This agent
        # has an initial state of 0, and a parameter, increase, with constant value 1.01
        filter_element(f, in_stream, out_stream, state=0, increase=1.01)
        input_list = [10, 10, 9, 200, 201, 200, 202, 203, 1, 0, 2]
        in_stream.extend(input_list)
        run()
        assert recent_values(out_stream) == [10, 9, 201, 200, 203, 1, 0]

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: AssembleSoftware/IoTPy

max_issues_repo_name: AssembleSoftware/IoTPy

max_forks_repo_name: AssembleSoftware/IoTPy