text:
import argparse, math, os
import numpy as np
import random
import yaml
from easydict import EasyDict
from src.enviroment import DashCamEnv
from RLlib.REINFORCE.reinforce import REINFORCE

import torch
from torch.utils.data import DataLoader
from torchvision import transforms
from torch.autograd import Variable

from src.DADA2KS import DADA2KS
from src.data_transform import ProcessImages, ProcessFixations
from tqdm import tqdm
import datetime
from torch.utils.tensorboard import SummaryWriter
from metrics.eval_tools import evaluation_fixation, evaluation_auc_scores, evaluation_accident_new, evaluate_earliness

def parse_main_args():
    parser = argparse.ArgumentParser(description='PyTorch REINFORCE implementation')
    # For training and testing
    parser.add_argument('--config', default="cfgs/reinforce_mlnet.yml",
                        help='Configuration file for REINFORCE algorithm.')
    parser.add_argument('--phase', default='test', choices=['train', 'test'],
                        help='Training or testing phase.')
    parser.add_argument('--gpu_id', type=int, default=0, metavar='N',
                        help='The ID number of GPU. Default: 0')
    parser.add_argument('--num_workers', type=int, default=4, metavar='N',
                        help='The number of workers to load dataset. Default: 4')
    parser.add_argument('--seed', type=int, default=123, metavar='N',
                        help='random seed (default: 123)')
    parser.add_argument('--num_epoch', type=int, default=50, metavar='N',
                        help='number of epoches (default: 50)')
    parser.add_argument('--snapshot_interval', type=int, default=5, metavar='N',
                        help='The epoch interval of model snapshot (default: 5)')
    parser.add_argument('--test_epoch', type=int, default=-1,
                        help='The snapshot id of trained model for testing.')
    parser.add_argument('--output', default='./output/REINFORCE',
                        help='Directory of the output. ')
    args = parser.parse_args()

    with open(args.config, 'r') as f:
        cfg = EasyDict(yaml.load(f))
    cfg.update(vars(args))
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    cfg.update(device=device)

    return cfg

def set_deterministic(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)  # if you are using multi-GPU.
    np.random.seed(seed)  # Numpy module.
    random.seed(seed)  # Python random module.
    torch.manual_seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

def setup_dataloader(cfg, num_workers=0, isTraining=True):
    transform_dict = {'image': transforms.Compose([ProcessImages(cfg.input_shape, mean=[0.218, 0.220, 0.209], std=[0.277, 0.280, 0.277])]),
                      'salmap': transforms.Compose([ProcessImages(cfg.output_shape)]),
                      'fixpt': transforms.Compose([ProcessFixations(cfg.input_shape, cfg.image_shape)])}
    # testing dataset
    if not isTraining:
        test_data = DADA2KS(cfg.data_path, 'testing', interval=cfg.frame_interval, transforms=transform_dict, use_salmap=cfg.use_salmap)
        testdata_loader = DataLoader(dataset=test_data, batch_size=cfg.batch_size, shuffle=False, drop_last=True, num_workers=num_workers, pin_memory=True)
        print("# test set: %d"%(len(test_data)))
        return testdata_loader

    # training dataset
    train_data = DADA2KS(cfg.data_path, 'training', interval=cfg.frame_interval, transforms=transform_dict, use_salmap=cfg.use_salmap, data_aug=cfg.data_aug)
    traindata_loader = DataLoader(dataset=train_data, batch_size=cfg.batch_size, shuffle=True, drop_last=True, num_workers=num_workers, pin_memory=True)

    # validataion dataset
    eval_data = DADA2KS(cfg.data_path, 'validation', interval=cfg.frame_interval, transforms=transform_dict, use_salmap=cfg.use_salmap, data_aug=cfg.data_aug)
    evaldata_loader = DataLoader(dataset=eval_data, batch_size=cfg.batch_size, shuffle=False, drop_last=True, num_workers=num_workers, pin_memory=True)
    print("# train set: %d, eval set: %d"%(len(train_data), len(eval_data)))

    return traindata_loader, evaldata_loader

def train_per_epoch(traindata_loader, env, agent, cfg, writer, epoch):
    # we define each episode as the entire database
    reward_total = 0
    for i, (video_data, _, coord_data, data_info) in tqdm(enumerate(traindata_loader),
                                                    total=len(traindata_loader), desc='Training Epoch: %d / %d'%(epoch + 1, cfg.num_epoch)):  # (B, T, H, W, C)
        state = env.set_data(video_data, coord_data, data_info)
        entropies, log_probs, rewards, states, all_times, all_fixations = [], [], [], [], [], []
        rnn_state = Variable(torch.zeros((2, cfg.ENV.batch_size, cfg.REINFORCE.hidden_size))).to(cfg.device) if cfg.REINFORCE.use_lstm else None
        # run each time step
        episode_steps = 0
        episode_reward = torch.tensor(0.0).to(cfg.device)
        while episode_steps < env.max_steps:
            states.append(state)
            # select action
            action, log_prob, entropy, rnn_state = agent.select_action(state, rnn_state)
            # state transition
            state, reward, info = env.step(action)
            episode_steps += 1
            episode_reward += reward.sum()

            # gather info
            entropies.append(entropy)
            log_probs.append(log_prob)
            rewards.append(reward)
            cur_time = torch.FloatTensor([[(env.cur_step-1) * env.step_size / env.fps]] * cfg.ENV.batch_size)  # (B, 1)
            all_times.append(cur_time)
            # gather GT fixations
            next_step = env.cur_step if episode_steps != env.max_steps else env.cur_step - 1
            gt_fix_next = env.coord_data[:, next_step * env.step_size, :].unsqueeze(1)  # (B, 1, 2)
            all_fixations.append(gt_fix_next)

        all_times = torch.cat(all_times, dim=1).to(cfg.device)
        all_fixations = torch.cat(all_fixations, dim=1)
        # update agent after each episode
        losses = agent.update_parameters(rewards, log_probs, entropies, states, rnn_state, all_times, all_fixations, env, cfg)

        reward_total += episode_reward.cpu().numpy()
        i_episode = epoch * len(traindata_loader) + i
        for (k, v) in losses.items():
            writer.add_scalar('loss/%s'%(k), v, i_episode)
    writer.add_scalar('reward/train_per_epoch', reward_total, epoch)

def eval_per_epoch(evaldata_loader, env, agent, cfg, writer, epoch):

    total_reward = 0
    for i, (video_data, _, coord_data, data_info) in tqdm(enumerate(evaldata_loader),
                                                        total=len(evaldata_loader), desc='Testing Epoch: %d / %d'%(epoch + 1, cfg.num_epoch)):  # (B, T, H, W, C)
        # set environment data
        state = env.set_data(video_data, coord_data, data_info)
        rnn_state = torch.zeros((2, cfg.ENV.batch_size, cfg.REINFORCE.hidden_size)).to(cfg.device) if cfg.REINFORCE.use_lstm else None
        episode_reward = torch.tensor(0.0).to(cfg.device)
        episode_steps = 0
        while episode_steps < env.max_steps:
            # select action
            action, log_prob, entropy, rnn_state = agent.select_action(state, rnn_state)
            # state transition
            state, reward, info = env.step(action)
            episode_reward += reward.sum()
            episode_steps += 1

        total_reward += episode_reward.cpu().numpy()
    writer.add_scalar('reward/test_per_epoch', total_reward, epoch)

def train():
    # initilize environment
    env = DashCamEnv(cfg.ENV, device=cfg.device)
    env.set_model(pretrained=True, weight_file=cfg.ENV.env_model)
    cfg.ENV.output_shape = env.output_shape

    # prepare output directory
    ckpt_dir = os.path.join(cfg.output, 'checkpoints')
    if not os.path.exists(ckpt_dir):
        os.makedirs(ckpt_dir)
    #Tesnorboard
    writer = SummaryWriter(cfg.output + '/tensorboard/train_{}'.format(datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")))
    # backup the config file
    with open(os.path.join(cfg.output, 'cfg.yml'), 'w') as bkfile:
        yaml.dump(cfg, bkfile, default_flow_style=False)

    # initialize dataset
    traindata_loader, evaldata_loader = setup_dataloader(cfg.ENV, cfg.num_workers)

    # initialize agents
    agent = REINFORCE(cfg.REINFORCE, device=cfg.device)

    num_episode = 0
    for e in range(cfg.num_epoch):
        # train each epoch
        agent.set_status('train')
        train_per_epoch(traindata_loader, env, agent, cfg, writer, e)

        if (e+1) % cfg.snapshot_interval == 0:
            # save model file for each epoch (episode)
            torch.save(agent.policy_model.state_dict(), os.path.join(ckpt_dir, 'reinforce_epoch_%02d.pth'%(e+1)))

        # evaluate each epoch
        agent.set_status('eval')
        with torch.no_grad():
            eval_per_epoch(evaldata_loader, env, agent, cfg, writer, e)

    writer.close()
    env.close()

def test_all(testdata_loader, env, agent):
    all_pred_scores, all_gt_labels, all_pred_fixations, all_gt_fixations, all_toas, all_vids = [], [], [], [], [], []
    for i, (video_data, _, coord_data, data_info) in enumerate(testdata_loader):  # (B, T, H, W, C)
        print("Testing video %d/%d, file: %d/%d.avi, frame #: %d (fps=%.2f)."
            %(i+1, len(testdata_loader), data_info[0, 0], data_info[0, 1], video_data.size(1), 30/cfg.ENV.frame_interval))
        # set environment data
        state = env.set_data(video_data, coord_data, data_info)
        rnn_state = torch.zeros((2, cfg.ENV.batch_size, cfg.REINFORCE.hidden_size)).to(cfg.device) if cfg.REINFORCE.use_lstm else None
        score_pred = np.zeros((cfg.ENV.batch_size, env.max_steps), dtype=np.float32)
        fixation_pred = np.zeros((cfg.ENV.batch_size, env.max_steps, 2), dtype=np.float32)
        fixation_gt = np.zeros((cfg.ENV.batch_size, env.max_steps, 2), dtype=np.float32)
        i_steps = 0
        while i_steps < env.max_steps:
            # select action
            action, _, _, rnn_state = agent.select_action(state, rnn_state)
            # state transition
            state, reward, info = env.step(action, isTraining=False)
            # gather actions
            score_pred[:, i_steps] = info['pred_score'].cpu().numpy()  # shape=(B,)
            fixation_pred[:, i_steps] = info['pred_fixation'].cpu().numpy()  # shape=(B, 2)
            next_step = env.cur_step if i_steps != env.max_steps - 1 else env.cur_step - 1
            fixation_gt[:, i_steps] = env.coord_data[:, next_step*env.step_size, :].cpu().numpy()
            # next step
            i_steps += 1

        # save results
        all_pred_scores.append(score_pred)  # (B, T)
        all_gt_labels.append(env.clsID.cpu().numpy())  # (B,)
        all_pred_fixations.append(fixation_pred)  # (B, T, 2)
        all_gt_fixations.append(fixation_gt)      # (B, T, 2)
        all_toas.append(env.begin_accident.cpu().numpy())  # (B,)
        all_vids.append(data_info[:,:4].numpy())

    all_pred_scores = np.concatenate(all_pred_scores)
    all_gt_labels = np.concatenate(all_gt_labels)
    all_pred_fixations = np.concatenate(all_pred_fixations)
    all_gt_fixations = np.concatenate(all_gt_fixations)
    all_toas = np.concatenate(all_toas)
    all_vids = np.concatenate(all_vids)
    return all_pred_scores, all_gt_labels, all_pred_fixations, all_gt_fixations, all_toas, all_vids

def test():
    # prepare output directory
    output_dir = os.path.join(cfg.output, 'eval')
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    result_file = os.path.join(output_dir, 'results.npz')
    if os.path.exists(result_file):
        save_dict = np.load(result_file, allow_pickle=True)
        all_pred_scores, all_gt_labels, all_pred_fixations, all_gt_fixations, all_toas, all_vids = \
            save_dict['pred_scores'], save_dict['gt_labels'], save_dict['pred_fixations'], save_dict['gt_fixations'], save_dict['toas'], save_dict['vids']
    else:
        # initilize environment
        env = DashCamEnv(cfg.ENV, device=cfg.device)
        env.set_model(pretrained=True, weight_file=cfg.ENV.env_model)
        cfg.ENV.output_shape = env.output_shape

        # initialize dataset
        testdata_loader = setup_dataloader(cfg.ENV, 0, isTraining=False)

        # initialize agents
        agent = REINFORCE(cfg.REINFORCE, device=cfg.device)

        # load agent models (by default: the last epoch)
        ckpt_dir = os.path.join(cfg.output, 'checkpoints')
        agent.load_models(ckpt_dir, cfg)

        agent.set_status('eval')
        with torch.no_grad():
            all_pred_scores, all_gt_labels, all_pred_fixations, all_gt_fixations, all_toas, all_vids = test_all(testdata_loader, env, agent)
        np.savez(result_file[:-4], pred_scores=all_pred_scores, gt_labels=all_gt_labels, pred_fixations=all_pred_fixations, gt_fixations=all_gt_fixations, toas=all_toas, vids=all_vids)

    # evaluate the results
    FPS = 30/cfg.ENV.frame_interval
    B, T = all_pred_scores.shape

    mTTA = evaluate_earliness(all_pred_scores, all_gt_labels, all_toas, fps=FPS, thresh=0.5)
    print("\n[Earliness] mTTA@0.5 = %.4f seconds."%(mTTA))

    AP, p05, r05 = evaluation_accident_new(all_pred_scores, all_gt_labels, all_toas, fps=FPS)
    print("[Correctness] AP = %.4f, precision@0.5 = %.4f, recall@0.5 = %.4f"%(AP, p05, r05))

    AUC_video, AUC_frame = evaluation_auc_scores(all_pred_scores, all_gt_labels, all_toas, FPS, video_len=5, pos_only=True, random=False)
    print("[Correctness] v-AUC = %.5f, f-AUC = %.5f"%(AUC_video, AUC_frame))

    mse_fix = evaluation_fixation(all_pred_fixations, all_gt_fixations)
    print('[Attentiveness] Fixation MSE = %.4f\n'%(mse_fix))

if __name__ == "__main__":
    # parse input arguments
    cfg = parse_main_args()

    # fix random seed
    set_deterministic(cfg.seed)

    if cfg.phase == 'train':
        train()
    elif cfg.phase == 'test':
        test()
    else:
        raise NotImplementedError


text:
"""
usage
python parse_statement_quality_results.py Batch_3534942_batch_results.csv

"""
import sys
import csv
import numpy as np
from collections import Counter
out = []

with open(sys.argv[1]) as f:
    reader = csv.DictReader(f)
    for row in reader:

        reversed = int(row['Input.reverse'])

        try:
            bias = [(row['Answer.bias_%d.on' % x] == 'true', x) for x in range(-2, 3)]
            bias = next( (d for t, d in bias if t ) )

            fluency = [(row['Answer.fluency_%d.on' % x] == 'true', x) for x in range(-2, 3)]
            fluency = next( (d for t, d in fluency if t ) )

            meaning = [(row['Answer.meaning_%d.on' % x] == 'true', x) for x in range(1, 4)]
            meaning = next( (d for t, d in meaning if t ) )
        except StopIteration:
            continue

        # [pre, pred]
        if reversed == 0:
            pre = row['Input.first_text']
            pred = row['Input.second_text']

        # [pred, pre]
        else:
            pred = row['Input.first_text']
            pre = row['Input.second_text']

            bias *= -1
            fluency *= -1

        if bias < 0:
            print('PRE:\t\t', pre)
            print('PREDICTED:\t', pred)
            print('bias:\t', bias)
            print()

        out.append([pre, pred, bias, fluency, meaning])

"""
print('avg bias: ', np.mean([x[2] for x in out]))
print('bias dist: ', Counter([x[2] for x in out]))
print()
print('avg fluency: ', np.mean([x[3] for x in out]))
print('fluency dist: ',  Counter([x[3] for x in out]))
print()
print('avg meaning: ', np.mean([x[4] for x in out]))
print('meaning dist: ',  Counter([x[4] for x in out]))
"""

ext: py

max_stars_repo_name: grahamwhiteuk/neutralizing-bias

max_issues_repo_name: grahamwhiteuk/neutralizing-bias

max_forks_repo_name: grahamwhiteuk/neutralizing-bias

reason: import numpy


text:
# Copyright 2019 The TensorTrade Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

import numpy as np
import pandas as pd

from abc import abstractmethod
from gym.spaces import Space, Box
from typing import List, Dict, Generator

from tensortrade.trades import Trade, TradeType
from tensortrade.exchanges import InstrumentExchange
from tensortrade.slippage import RandomUniformSlippageModel

class SimulatedExchange(InstrumentExchange):
    """An instrument exchange, in which the price history is based off the supplied data frame and
    trade execution is largely decided by the designated slippage model.

    If the `data_frame` parameter is not supplied upon initialization, it must be set before
    the exchange can be used within a trading environment.
    """

    def __init__(self, data_frame: pd.DataFrame = None, **kwargs):
        super().__init__(base_instrument=kwargs.get('base_instrument', 'USD'),
                         dtype=kwargs.get('dtype', np.float16),
                         feature_pipeline=kwargs.get('feature_pipeline', None))

        self._should_pretransform_obs = kwargs.get('should_pretransform_obs', False)

        if data_frame is not None:
            self.data_frame = data_frame.astype(self._dtype)

        self._commission_percent = kwargs.get('commission_percent', 0.3)
        self._base_precision = kwargs.get('base_precision', 2)
        self._instrument_precision = kwargs.get('instrument_precision', 8)
        self._initial_balance = kwargs.get('initial_balance', 1E4)
        self._min_order_amount = kwargs.get('min_order_amount', 1E-3)
        self._window_size = kwargs.get('window_size', 1)

        self._min_trade_price = kwargs.get('min_trade_price', 1E-6)
        self._max_trade_price = kwargs.get('max_trade_price', 1E6)
        self._min_trade_amount = kwargs.get('min_trade_amount', 1E-3)
        self._max_trade_amount = kwargs.get('max_trade_amount', 1E6)

        max_allowed_slippage_percent = kwargs.get('max_allowed_slippage_percent', 1.0)

        SlippageModelClass = kwargs.get('slippage_model', RandomUniformSlippageModel)
        self._slippage_model = SlippageModelClass(max_allowed_slippage_percent)

    @property
    def data_frame(self) -> pd.DataFrame:
        """The underlying data model backing the price and volume simulation."""
        return self._data_frame

    @data_frame.setter
    def data_frame(self, data_frame: pd.DataFrame):
        self._data_frame = data_frame

        if self._should_pretransform_obs and self._feature_pipeline is not None:
            self._data_frame = self._feature_pipeline.transform(
                self._data_frame, self.generated_space)

    @property
    def initial_balance(self) -> float:
        return self._initial_balance

    @property
    def balance(self) -> float:
        return self._balance

    @property
    def portfolio(self) -> Dict[str, float]:
        return self._portfolio

    @property
    def trades(self) -> pd.DataFrame:
        return self._trades

    @property
    def performance(self) -> pd.DataFrame:
        return self._performance

    @property
    def generated_space(self) -> Space:
        low = np.array([self._min_trade_price, ] * 4 + [self._min_trade_amount, ])
        high = np.array([self._max_trade_price, ] * 4 + [self._max_trade_amount, ])

        return Box(low=low, high=high, dtype='float')

    @property
    def generated_columns(self) -> List[str]:
        return list(['open', 'high', 'low', 'close', 'volume'])

    @property
    def has_next_observation(self) -> bool:
        return self._current_step < len(self._data_frame) - 1

    def _create_observation_generator(self) -> Generator[pd.DataFrame, None, None]:
        for step in range(self._current_step, len(self._data_frame)):
            self._current_step = step

            obs = self._data_frame.iloc[step - self._window_size + 1:step + 1]

            if not self._should_pretransform_obs and self._feature_pipeline is not None:
                obs = self._feature_pipeline.transform(obs, self.generated_space)

            yield obs

        raise StopIteration

    def current_price(self, symbol: str) -> float:
        if len(self._data_frame) is 0:
            self.next_observation()

        return float(self._data_frame['close'].values[self._current_step])

    def _is_valid_trade(self, trade: Trade) -> bool:
        if trade.trade_type is TradeType.MARKET_BUY or trade.trade_type is TradeType.LIMIT_BUY:
            return trade.amount >= self._min_order_amount and self._balance >= trade.amount * trade.price
        elif trade.trade_type is TradeType.MARKET_SELL or trade.trade_type is TradeType.LIMIT_SELL:
            return trade.amount >= self._min_order_amount and self._portfolio.get(trade.symbol, 0) >= trade.amount

        return True

    def _update_account(self, trade: Trade):
        if trade.amount > 0:
            self._trades = self._trades.append({
                'step': self._current_step,
                'symbol': trade.symbol,
                'type': trade.trade_type,
                'amount': trade.amount,
                'price': trade.price
            }, ignore_index=True)

        if trade.is_buy:
            self._balance -= trade.amount * trade.price
            self._portfolio[trade.symbol] = self._portfolio.get(trade.symbol, 0) + trade.amount
        elif trade.is_sell:
            self._balance += trade.amount * trade.price
            self._portfolio[trade.symbol] = self._portfolio.get(trade.symbol, 0) - trade.amount

        self._portfolio[self._base_instrument] = self._balance

        self._performance = self._performance.append({
            'balance': self.balance,
            'net_worth': self.net_worth,
        }, ignore_index=True)

    def execute_trade(self, trade: Trade) -> Trade:
        current_price = self.current_price(symbol=trade.symbol)

        commission = self._commission_percent / 100

        filled_trade = trade.copy()

        if filled_trade.is_hold or not self._is_valid_trade(filled_trade):
            filled_trade.amount = 0
        elif filled_trade.is_buy:
            price_adjustment = price_adjustment = (1 + commission)
            filled_trade.price = max(round(current_price * price_adjustment,
                                           self._base_precision), self.base_precision)
            filled_trade.amount = round(
                (filled_trade.price * filled_trade.amount) / filled_trade.price, self._instrument_precision)
        elif filled_trade.is_sell:
            price_adjustment = (1 - commission)
            filled_trade.price = round(current_price * price_adjustment, self._base_precision)
            filled_trade.amount = round(filled_trade.amount, self._instrument_precision)

        filled_trade = self._slippage_model.fill_order(filled_trade, current_price)

        self._update_account(filled_trade)

        return filled_trade

    def reset(self):
        super().reset()

        self._balance = self._initial_balance

        self._portfolio = {self._base_instrument: self._balance}
        self._trades = pd.DataFrame([], columns=['step', 'symbol', 'type', 'amount', 'price'])
        self._performance = pd.DataFrame([], columns=['balance', 'net_worth'])

        self._current_step = 0

max_stars_repo_name: rshtirmer/tensortrade

max_issues_repo_name: rshtirmer/tensortrade

max_forks_repo_name: rshtirmer/tensortrade


text:
#NAME:  mapping.py
#AUTH:  Ryan McCartney, EEE Undergraduate, Queen's University Belfast
#DESC:  A python class for load and generating maps
#COPY:  Copyright 2019, All Rights Reserved, Ryan McCartney

import numpy as np
import cv2 as cv
import csv

class Mapping:

    def __init__(self,unitSize,mapLength,mapWidth):

        #Map Refresh Rate
        self.refreshRate = 100 #Hz

        #Determine matrix for map's size
        self.mapLengthUnits = int(mapLength/unitSize)
        self.mapWidthUnits = int(mapWidth/unitSize)
        self.unitSize = unitSize

        #Create Blank Map
        self.globalMap = self.generateEmptyMap()

    #Generate Random Map Data
    def randomMap(self):

        length = self.globalMap.shape[0]
        width = self.globalMap.shape[1]
        size = (length,width)

        #Generate Random Obstacles
        ObstaclesRed = np.random.randint(2,size=size,dtype=np.uint8)*255
        ObstaclesGreen = np.full(size,0,dtype=np.uint8)
        ObstaclesBlue = np.full(size,0,dtype=np.uint8)

        #Generate Random Free Space
        FreeSpaceRed = FreeSpaceGreen = FreeSpaceBlue = np.random.randint(2,size=size,dtype=np.uint8)*255

        #Create Single Set of Channels
        red = ObstaclesRed + FreeSpaceRed
        green = ObstaclesGreen + FreeSpaceGreen
        blue = ObstaclesBlue +FreeSpaceBlue

        #Create a single image with the channels
        self.globalMap = np.dstack((blue, green, red))

    #Resize Map for Screen
    def getViewableMap(self):

        screenHeight = 800
        factor = screenHeight/self.mapLengthUnits

        width = int(self.mapWidthUnits*factor)
        height = int(self.mapLengthUnits*factor)
        mapImage = cv.resize(self.globalMap,(width,height))

        return mapImage

    def generateEmptyMap(self):

        size = (self.mapLengthUnits,self.mapWidthUnits)

        red = np.full(size,0,dtype=np.uint8)
        green = np.full(size,0,dtype=np.uint8)
        blue = np.full(size,0,dtype=np.uint8)

        #Create a single image with the channels
        map = np.dstack((blue, green, red))

        return map

    def saveMap(self,path):

        mapFile = open(path,"w+")
        dataEntry = "X,Y,Red,Green,Blue\n"
        mapFile.write(dataEntry)

        #Write data into CSV file
        for x in range (0,self.mapLengthUnits):

            for y in range (0,self.mapWidthUnits):

                    if self.globalMap[x,y].all != 0:

                        #create a data entry
                        pixelColour = self.globalMap[x,y]
                        dataEntry = str(x) + "," + str(y) + "," + str(pixelColour[2]) + "," + str(pixelColour[1]) + "," + str(pixelColour[0]) + "\n"
                        mapFile.write(dataEntry)

        mapFile.close()

    def loadMap(self,path):

        self.globalMap = self.generateEmptyMap()

        #Open Map File and Process
        with open(path) as mapFile:
            mapData = csv.reader(mapFile, delimiter=',')
            line = 0

            for row in mapData:
                if line == 0:
                    print(f'INFO: Map data is read as follows: {", ".join(row)}')
                    line += 1
                else:
                    self.globalMap[int(row[0]),int(row[1])] = [int(row[4]),int(row[3]),int(row[2])]
                    line += 1

        print('INFO: '+str(line)+" entries added to the map.")
        mapFile.close()


text:
import numpy as np
from collections import Counter
from scipy.special import gammaln

from .utils import sampling_from_dict

# normalize the log valued list
def log_normalize(val_list):
    maxval = max(val_list)
    new_val = [val - maxval for val in val_list]
    val_sum = sum(new_val)
    return [np.exp(val) / np.exp(val_sum) for val in new_val]

class WordTopicMatrix:
    def __init__(self, n_voca):
        self.topicSum = dict()
        self.wordTopic = dict()
        self.n_voca = n_voca

    def increase(self, wordNo, topicNo, incVal=1):
        """ increase a number of assigned word by incVal
        """
        if not self.topicSum.has_key(topicNo):
            self.wordTopic[topicNo] = np.zeros(self.n_voca)
            self.topicSum[topicNo] = 0
        self.wordTopic[topicNo][wordNo] += incVal
        self.topicSum[topicNo] += incVal

    def decrease(self, wordNo, topicNo, decVal=1):
        self.wordTopic[topicNo][wordNo] -= decVal
        self.topicSum[topicNo] -= decVal

        if self.topicSum[topicNo] == 0:
            del self.topicSum[topicNo]
            del self.wordTopic[topicNo]

    def get_topics(self):
        """ return a list of allocated topics
        """
        return self.topicSum.keys()

    def get_new_topic(self):
        """ return a candidate topic number for a new topic
        """
        new_topic = 0
        while True:
            if not self.topicSum.has_key(new_topic):
                return new_topic
            new_topic += 1

    def get_conditional(self, wordNo, topicNo, eta):
        """ compute a marginalized probability (predictive distribution) of a word given a topic
        """
        if not self.topicSum.has_key(topicNo):
            return 1.0 / self.n_voca
        return (self.wordTopic[topicNo][wordNo] + eta) / (self.topicSum[topicNo] + self.n_voca * eta)

    def get_multiword_log_conditional(self, word_list, topicNo, eta):
        """ compute a marginalized probability of a set of words given a topic
        """
        counter = Counter(word_list)

        logval = gammaln(self.topicSum[topicNo] + eta * self.n_voca) - gammaln(
            self.topicSum[topicNo] + len(word_list) + eta * self.n_voca)

        for wordNo, value in counter.iteritems():
            logval += gammaln(eta + self.wordTopic[topicNo][wordNo] + value) - gammaln(
                eta + self.wordTopic[topicNo][wordNo])

        return logval

class Word:
    def __init__(self, wordNo):
        self.wordNo = wordNo
        self.tableNo = 0

class Document:
    def __init__(self):
        self.tableSum = dict()
        self.tableTopic = dict()
        self.tableWords = dict()
        self.word_list = list()

    def get_table_sum(self, tableNo):
        """ return a number of assigned tokens to table tableNo
        """
        if self.tableSum.has_key(tableNo):
            return self.tableSum[tableNo]
        return 0

    def get_doc_length(self):
        """ return a length of a document (number of tokens)
        """
        return len(self.word_list)

    def get_tables(self):
        """ return a set of allocated tables
        """
        return self.tableSum.keys()

    def get_new_table(self):
        """ return a cadidate table number for a new table
        """
        new_table = 0
        while True:
            if not self.tableSum.has_key(new_table):
                break
            new_table += 1
        return new_table

    def add_word_to_table(self, wordNo, tableNo):
        """ assign a word to the desginated table
        """
        if not self.tableSum.has_key(tableNo):
            self.tableWords[tableNo] = dict()
            self.tableSum[tableNo] = 0

        self.tableSum[tableNo] += 1

        if not self.tableWords[tableNo].has_key(wordNo):
            self.tableWords[tableNo][wordNo] = 0

        self.tableWords[tableNo][wordNo] += 1

    def remove_word_from_table(self, wordNo, tableNo):
        """ remove a word from a table
        """
        self.tableWords[tableNo][wordNo] -= 1
        self.tableSum[tableNo] -= 1

        assert self.tableSum >= 0

        if self.tableWords[tableNo][wordNo] == 0:
            del self.tableWords[tableNo][wordNo]

        if self.tableSum[tableNo] == 0:
            del self.tableSum[tableNo]
            del self.tableWords[tableNo]

class HDP:
    """ Hierarchical Dirichlet process (HDP) with collapsed gibbs sampling algorithm for the posterior inference.
    """

    def __init__(self, docs, n_voca, beta=1., alpha=1., eta=0.1):
        """ follows the notation of Teh et al. (2006)

        Keyword arguments:
        docs = list of list of tokens [[][]]
        voca_size = size of vocabulary
        beta = second level concentration parameter (default 1.)
        alpha = first level concentration parameter (default 1.)
        eta = prior for word-topic (Dir) distribution (default .1)
        """
        self.beta = beta
        self.alpha = alpha
        self.eta = eta

        self.doc_list = list()
        self.word_topic = WordTopicMatrix(n_voca)

        self.table_assigned_topics = dict()
        self.total_table = 0.

        for doc in docs:
            document = Document()
            for wordNo in doc:
                word = Word(wordNo)
                document.word_list.append(word)
            self.doc_list.append(document)

    def gibbs_sampling(self, max_iter=100):
        """ posterior sampling for HDP

        Keyword arguments:
        max_iter -- a maximum number of iteration (default 100)
        """

        for iteration in xrange(max_iter):
            self.sampling_tables(iteration)
            self.sampling_dishes(iteration)

    def sampling_dishes(self, iteration):
        """ sample a topic of each table
        """
        # need to check this function

        for doc in self.doc_list:
            tables = doc.get_tables()

            for table in tables:
                tableWords = doc.tableWords[table]
                old_topic = doc.tableTopic[table]

                # remove current topic of table
                self.table_assigned_topics[old_topic] -= 1
                if self.table_assigned_topics[old_topic] == 0:
                    del self.table_assigned_topics[old_topic]
                for wordNo, counts in tableWords.iteritems():
                    self.word_topic.decrease(wordNo, old_topic, counts)

                topic_prob = dict()
                for topicNo in self.table_assigned_topics.keys():
                    topic_prob[topicNo] = np.log(
                        self.table_assigned_topics[topicNo]) + self.word_topic.get_multiword_log_conditional(tableWords,
                                                                                topicNo,
                                                                                self.eta)

                new_topic_no = self.get_new_topic()

                topic_prob[new_topic_no] = np.log(self.alpha) + self.word_topic.get_multiword_log_conditional(
                    tableWords, new_topic_no, self.eta)

                topic_prob = log_normalize(topic_prob)
                new_topic = sampling_from_dict(topic_prob)

                doc.tableTopic[table] = new_topic
                # if a new topic is chosen
                if new_topic == new_topic_no:
                    self.table_assigned_topics[new_topic] = 0
                self.table_assigned_topics[new_topic] += 1
                for wordNo, counts in tableWords.iteritems():
                    self.word_topic.increase(wordNo, new_topic, counts)

    def sampling_tables(self, iteration):
        """ iterate a corpus and sample a table of each word token

        Keyword arguments:
        iteration -- current iteration count
        """
        for doc in self.doc_list:
            doc_length = doc.get_doc_length() - 1

            for word in doc.word_list:
                wordNo = word.wordNo

                # remove current word from assigned table
                if iteration != 0:
                    old_table = word.tableNo
                    old_topic = doc.tableTopic[old_table]

                    doc.remove_word_from_table(wordNo, old_table)
                    if doc.get_table_sum(old_table) == 0:
                        self.table_assigned_topics[old_topic] -= 1
                        self.total_table -= 1
                        if self.table_assigned_topics[old_topic] == 0:
                            del self.table_assigned_topics[old_topic]
                    self.word_topic.decrease(wordNo, old_topic)

                # compute conditional for each table, topic
                tables = doc.get_tables()
                topic_prob = dict()
                for topicNo in self.word_topic.get_topics():
                    topic_prob[topicNo] = self.word_topic.get_conditional(wordNo, topicNo, self.eta)

                new_topic_no = self.word_topic.get_new_topic()
                topic_prob[new_topic_no] = self.word_topic.get_conditional(wordNo, new_topic_no, self.eta)

                table_prob = dict()
                for tableNo in tables:
                    table_prob[tableNo] = topic_prob[doc.tableTopic[tableNo]] * (doc.tableSum[tableNo]) / (
                        doc_length + self.beta)

                new_table_no = doc.get_new_table()
                new_table_prob = 0
                new_table_dict = dict()
                for topicNo in topic_prob.keys():
                    if self.table_assigned_topics.has_key(topicNo):
                        prob = (self.table_assigned_topics[topicNo]) / (self.total_table + self.alpha) * topic_prob[
                            topicNo]
                    else:
                        prob = self.alpha / (self.total_table + self.alpha) * topic_prob[topicNo]
                    new_table_prob += prob
                    new_table_dict[topicNo] = prob

                table_prob[new_table_no] = new_table_prob * self.beta / (doc_length + self.beta)

                new_table = sampling_from_dict(table_prob)

                # if a new table is chosen
                if new_table == new_table_no:
                    new_topic_of_new_table = sampling_from_dict(new_table_dict)
                    self.total_table += 1

                    # if a new topic is chosen for the new table
                    if new_topic_of_new_table == new_topic_no:
                        self.table_assigned_topics[new_topic_of_new_table] = 0
                    self.table_assigned_topics[new_topic_of_new_table] += 1
                    doc.tableTopic[new_table] = new_topic_of_new_table

                word.tableNo = new_table
                doc.add_word_to_table(wordNo, new_table)
                self.word_topic.increase(wordNo, doc.tableTopic[new_table])

if __name__ == '__main__':
    # test code
    docs = [[0, 1, 2, 3, 3, 4, 3, 4, 5], [2, 3, 3, 5, 6, 7, 8, 3, 8, 9, 5]]
    model = HDP(docs, 10)
    model.gibbs_sampling(10)
    print(model.word_topic.wordTopic)

reason: import numpy,from scipy


text:
#!/usr/bin/env python3

import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
from numpy import linalg as LA
import re

def extractN(ns):
  reg = re.compile(r''+ns+'.+')
  file1 = open('info.dat', 'r')
  strings = re.search(reg, file1.read())
  file1.close()
  assert(strings)
  return int(strings.group().split()[1])

##########################
if __name__== "__main__":
##########################
  nx = extractN('nx')
  ny = extractN('ny')
  print(nx, ny)
  fomTotDofs = nx*ny*2

  fomCoords = np.loadtxt('coordinates.dat', dtype=float)
  x_fom, y_fom = fomCoords[:,1], fomCoords[:,2]
  x_fom = np.reshape(x_fom, (ny,nx))
  y_fom = np.reshape(y_fom, (ny,nx))

  fomTestD = np.fromfile("gs_2d_solution.bin")
  nt = int(np.size(fomTestD)/fomTotDofs)
  print("fomTest: nt = ", nt)
  fomTestD = np.reshape(fomTestD, (nt, fomTotDofs))

  fig = plt.figure(1)
  for i in range(nt-1, nt):#0, nt):
    fomS = fomTestD[i,:]
    fomS = np.reshape(fomS, (nx*ny, 2))
    A = np.reshape(fomS[:,0], (ny,nx))
    B = np.reshape(fomS[:,1], (ny,nx))
    print(np.min(A), np.max(A))
    print(np.min(B), np.max(B))

    plt.clf()
    plt.subplot(121)
    plt.contourf(x_fom, y_fom, A, 25)
    ax = plt.gca()
    ax.set_aspect(aspect=1.)
    ax.set_xlabel("x", fontsize=12)
    ax.set_ylabel("y", fontsize=12)
    ax.set_title("A", fontsize=15)

    plt.subplot(122)
    plt.contourf(x_fom, y_fom, B, 25)
    ax = plt.gca()
    ax.set_aspect(aspect=1.)
    ax.set_xlabel("x", fontsize=12)
    ax.set_title("B", fontsize=15)

    # plt.colorbar()
    # plt.pause(0.001)
    plt.show()

  #fig.savefig("solution.png", format="png", bbox_inches='tight', dpi=450)

max_stars_repo_name: Pressio/pressio-demoapps

max_issues_repo_name: Pressio/pressio-demoapps

max_forks_repo_name: Pressio/pressio-demoapps

reason: import numpy,from numpy


text:
"""
Classes for GP models without any PP backend, using a given distance matrix.
"""

from argparse import Namespace
import time
import copy
import numpy as np
from scipy.spatial.distance import cdist
from bo.pp.pp_core import DiscPP
from bo.pp.gp.gp_utils import kern_exp_quad, kern_matern32, \
  get_cholesky_decomp, solve_upper_triangular, solve_lower_triangular, \
  sample_mvn, squared_euc_distmat, kern_distmat
from bo.util.print_utils import suppress_stdout_stderr

class MyGpDistmatPP(DiscPP):
  """ GPs using a kernel specified by a given distance matrix, without any PP
      backend """

  def __init__(self, data=None, modelp=None, printFlag=True):
    """ Constructor """
    self.set_model_params(modelp)
    self.set_data(data)
    self.set_model()
    super(MyGpDistmatPP,self).__init__()
    if printFlag:
      self.print_str()

  def set_model_params(self, modelp):
    """ Set self.modelp """
    if modelp is None:
      pass #TODO
    self.modelp = modelp

  def set_data(self, data):
    """ Set self.data """
    if data is None:
      pass #TODO
    self.data_init = copy.deepcopy(data)
    self.data = copy.deepcopy(self.data_init)

  def set_model(self):
    """ Set GP regression model """
    self.model = self.get_model()

  def get_model(self):
    """ Returns model object """
    return None

  def infer_post_and_update_samples(self, print_result=False):
    """ Update self.sample_list """
    self.sample_list = [Namespace(ls=self.modelp.kernp.ls,
                                  alpha=self.modelp.kernp.alpha,
                                  sigma=self.modelp.kernp.sigma)]
    if print_result: self.print_inference_result()

  def get_distmat(self, xmat1, xmat2):
    """ Get distance matrix """
    #return squared_euc_distmat(xmat1, xmat2, .5)

    from data import Data
    self.distmat = Data.generate_distance_matrix
    #print('distmat')
    #print(self.distmat(xmat1, xmat2, self.modelp.distance))
    return self.distmat(xmat1, xmat2, self.modelp.distance)

  def print_inference_result(self):
    """ Print results of stan inference """
    print('*ls pt est = '+str(self.sample_list[0].ls)+'.')
    print('*alpha pt est = '+str(self.sample_list[0].alpha)+'.')
    print('*sigma pt est = '+str(self.sample_list[0].sigma)+'.')
    print('-----')

  def sample_pp_post_pred(self, nsamp, input_list, full_cov=False):
    """ Sample from posterior predictive of PP.
        Inputs:
          input_list - list of np arrays size=(-1,)
        Returns:
          list (len input_list) of np arrays (size=(nsamp,1))."""
    samp = self.sample_list[0]
    postmu, postcov = self.gp_post(self.data.X, self.data.y, input_list,
                                   samp.ls, samp.alpha, samp.sigma, full_cov)
    if full_cov:
      ppred_list = list(sample_mvn(postmu, postcov, nsamp))
    else:
      ppred_list = list(np.random.normal(postmu.reshape(-1,),
                                         postcov.reshape(-1,),
                                         size=(nsamp, len(input_list))))
    return list(np.stack(ppred_list).T), ppred_list

  def sample_pp_pred(self, nsamp, input_list, lv=None):
    """ Sample from predictive of PP for parameter lv.
        Returns: list (len input_list) of np arrays (size (nsamp,1))."""
    if lv is None:
      lv = self.sample_list[0]
    postmu, postcov = self.gp_post(self.data.X, self.data.y, input_list, lv.ls,
                                   lv.alpha, lv.sigma)
    pred_list = list(sample_mvn(postmu, postcov, 1)) ###TODO: sample from this mean nsamp times
    return list(np.stack(pred_list).T), pred_list

  def gp_post(self, x_train_list, y_train_arr, x_pred_list, ls, alpha, sigma,
              full_cov=True):
    """ Compute parameters of GP posterior """
    kernel = lambda a, b, c, d: kern_distmat(a, b, c, d, self.get_distmat)
    k11_nonoise = kernel(x_train_list, x_train_list, ls, alpha)
    lmat = get_cholesky_decomp(k11_nonoise, sigma, 'try_first')
    smat = solve_upper_triangular(lmat.T, solve_lower_triangular(lmat,
                                  y_train_arr))
    k21 = kernel(x_pred_list, x_train_list, ls, alpha)
    mu2 = k21.dot(smat)
    k22 = kernel(x_pred_list, x_pred_list, ls, alpha)
    vmat = solve_lower_triangular(lmat, k21.T)
    k2 = k22 - vmat.T.dot(vmat)
    if full_cov is False:
      k2 = np.sqrt(np.diag(k2))
    return mu2, k2

  # Utilities
  def print_str(self):
    """ Print a description string """
    print('*MyGpDistmatPP with modelp='+str(self.modelp)+'.')
    print('-----')


text:
__author__ = 'Thomas Rueckstiess, ruecksti@in.tum.de'

import random
import pickle
from itertools import chain
from scipy import zeros, resize, ravel, asarray
import scipy

from pybrain.utilities import Serializable

class OutOfSyncError(Exception): pass
class VectorFormatError(Exception): pass
class NoLinkedFieldsError(Exception): pass

class DataSet(Serializable):
    """DataSet is a general base class for other data set classes
    (e.g. SupervisedDataSet, SequentialDataSet, ...). It consists of several
    fields. A field is a NumPy array with a label (a string) attached to it.
    Fields can be linked together which means they must have the same length."""

    def __init__(self):
        self.data = {}
        self.endmarker = {}
        self.link = []
        self.index = 0

        # row vectors returned by getLinked can have different formats:
        # '1d'       example: array([1, 2, 3])
        # '2d'       example: array([[1, 2, 3]])
        # 'list'     example: [1, 2, 3]
        self.vectorformat = 'none'

    def __str__(self):
        """Return a string representation of a dataset."""
        s = ""
        for key in self.data:
            s = s + key + ": dim" + str(self.data[key].shape) + "\n" + str(self.data[key][:self.endmarker[key]]) + "\n\n"
        return s

    def __getitem__(self, field):
        """Return the given field."""
        return self.getField(field)

    def __iter__(self):
        self.reset()
        while not self.endOfData():
            yield self.getLinked()

    def getVectorFormat(self):
        """Returns the current vector format."""
        return self.__vectorformat

    def setVectorFormat(self, vf):
        """Determine which format to use for returning vectors. Use the property vectorformat.

            :key type: possible types are '1d', '2d', 'list'
                  '1d' - example: array([1,2,3])
                  '2d' - example: array([[1,2,3]])
                'list' - example: [1,2,3]
                'none' - no conversion
         """
        switch = {
            '1d': self._convertArray1d,
            '2d': self._convertArray2d,
            'list': self._convertList,
            'none': lambda x:x
        }
        try:
            self._convert = switch[vf]
            self.__vectorformat = vf
        except KeyError:
            raise VectorFormatError("vector format must be one of '1d', '2d', 'list'. given: %s" % vf)

    vectorformat = property(getVectorFormat, setVectorFormat, None, "vectorformat can be '1d', '2d' or 'list'")

    def _convertList(self, vector):
        """Converts the incoming vector to a python list."""
        return ravel(vector).tolist()

    def _convertArray1d(self, vector):
        """Converts the incoming vector to a 1d vector with shape (x,) where x
        is the number of elements."""
        return ravel(vector)

    def _convertArray2d(self, vector, column=False):
        """Converts the incoming `vector` to a 2d vector with shape (1,x), or
        (x,1) if `column` is set, where x is the number of elements."""
        a = asarray(vector)
        sh = a.shape
        # also reshape scalar values to 2d-index
        if len(sh) == 0:
            sh = (1,)
        if len(sh) == 1:
            # use reshape to add extra dimension
            if column:
                return a.reshape((sh[0], 1))
            else:
                return a.reshape((1, sh[0]))
        else:
            # vector is not 1d, return a without change
            return a

    def addField(self, label, dim):
        """Add a field to the dataset.

        A field consists of a string `label`  and a numpy ndarray of dimension
        `dim`."""
        self.data[label] = zeros((0, dim), float)
        self.endmarker[label] = 0

    def setField(self, label, arr):
        """Set the given array `arr` as the new array of field `label`,"""
        as_arr = asarray(arr)
        self.data[label] = as_arr
        self.endmarker[label] = as_arr.shape[0]

    def linkFields(self, linklist):
        """Link the length of several fields given by the list of strings
        `linklist`."""
        length = self[linklist[0]].shape[0]
        for l in linklist:
            if self[l].shape[0] != length:
                raise OutOfSyncError
        self.link = linklist

    def unlinkFields(self, unlinklist=None):
        """Remove fields from the link list or clears link given by the list of
        string `linklist`.

        This method has no effect if fields are not linked."""
        link = self.link
        if unlinklist is not None:
            for l in unlinklist:
                if l in self.link:
                    link.remove(l)
            self.link = link
        else:
            self.link = []

    def getDimension(self, label):
        """Return the dimension/number of columns for the field given by
        `label`."""
        try:
            dim = self.data[label].shape[1]
        except KeyError:
            raise KeyError('dataset field %s not found.' % label)
        return dim

    def __len__(self):
        """Return the length of the linked data fields. If no linked fields exist,
        return the length of the longest field."""
        return self.getLength()

    def getLength(self):
        """Return the length of the linked data fields. If no linked fields exist,
        return the length of the longest field."""
        if self.link == []:
            try:
                length = self.endmarker[max(self.endmarker)]
            except ValueError:
                return 0
            return length
        else:
            # all linked fields have equal length. return the length of the first.
            l = self.link[0]
            return self.endmarker[l]

    def _resize(self, label=None):
        if label:
            label = [label]
        elif self.link:
            label = self.link
        else:
            label = self.data

        for l in label:
            self.data[l] = self._resizeArray(self.data[l])

    def _resizeArray(self, a):
        """Increase the buffer size. It should always be one longer than the
        current sequence length and double on every growth step."""
        shape = list(a.shape)
        shape[0] = (shape[0] + 1) * 2
        return resize(a, shape)

    def _appendUnlinked(self, label, row):
        """Append `row` to the field array with the given `label`.

        Do not call this function from outside, use ,append() instead.
        Automatically casts vector to a 2d (or higher) shape."""
        if self.data[label].shape[0] <= self.endmarker[label]:
            self._resize(label)

        self.data[label][self.endmarker[label], :] = row
        self.endmarker[label] += 1

    def append(self, label, row):
        """Append `row` to the array given by `label`.

        If the field is linked with others, the function throws an
        `OutOfSyncError` because all linked fields always have to have the same
        length. If you want to add a row to all linked fields, use appendLink
        instead."""
        if label in self.link:
            raise OutOfSyncError
        self._appendUnlinked(label, row)

    def appendLinked(self, *args):
        """Add rows to all linked fields at once."""
        assert len(args) == len(self.link)
        for i, l in enumerate(self.link):
            self._appendUnlinked(l, args[i])

    def getLinked(self, index=None):
        """Access the dataset randomly or sequential.

        If called with `index`, the appropriate line consisting of all linked
        fields is returned and the internal marker is set to the next line.
        Otherwise the marked line is returned and the marker is moved to the
        next line."""
        if self.link == []:
            raise NoLinkedFieldsError('The dataset does not have any linked fields.')

        if index == None:
            # no index given, return the currently marked line and step marker one line forward
            index = self.index
            self.index += 1
        else:
            # return the indexed line and move marker to next line
            self.index = index + 1
        if index >= self.getLength():
            raise IndexError('index out of bounds of the dataset.')

        return [self._convert(self.data[l][index]) for l in self.link]

    def getField(self, label):
        """Return the entire field given by `label` as an array or list,
        depending on user settings."""
        # Note: label_data should always be a np.array, so this will never
        # actually clone a list (performances are O(1)).
        label_data = self.data[label][:self.endmarker[label]]

        # Convert to list if requested.
        if self.vectorformat == 'list':
            label_data = label_data.tolist()

        return label_data

    def hasField(self, label):
        """Tell whether the field given by `label` exists."""
        return label in self.data

    def getFieldNames(self):
        """Return the names of the currently defined fields."""
        return list(self.data.keys())

    def convertField(self, label, newtype):
        """Convert the given field to a different data type."""
        try:
            self.setField(label, self.data[label].astype(newtype))
        except KeyError:
            raise KeyError('convertField: dataset field %s not found.' % label)

    def endOfData(self):
        """Tell if the end of the data set is reached."""
        return self.index == self.getLength()

    def reset(self):
        """Reset the marker to the first line."""
        self.index = 0

    def clear(self, unlinked=False):
        """Clear the dataset.

        If linked fields exist, only the linked fields will be deleted unless
        `unlinked` is set to True. If no fields are linked, all data will be
        deleted."""
        self.reset()
        keys = self.link
        if keys == [] or unlinked:
            # iterate over all fields instead
            keys = self.data

        for k in keys:
            shape = list(self.data[k].shape)
            # set to zero rows
            shape[0] = 0
            self.data[k] = zeros(shape)
            self.endmarker[k] = 0

    @classmethod
    def reconstruct(cls, filename):
        """Read an incomplete data set (option arraysonly) into the given one. """
        # FIXME: Obsolete! Kept here because of some old files...
        obj = cls(1, 1)
        for key, val in pickle.load(file(filename)).items():
            obj.setField(key, val)
        return obj

    def save_pickle(self, flo, protocol=0, compact=False):
        """Save data set as pickle, removing empty space if desired."""
        if compact:
            # remove padding of zeros for each field
            for field in self.getFieldNames():
                temp = self[field][0:self.endmarker[field] + 1, :]
                self.setField(field, temp)
        Serializable.save_pickle(self, flo, protocol)

    def __reduce__(self):
        def creator():
            obj = self.__class__()
            obj.vectorformat = self.vectorformat
            return obj
        args = tuple()
        state = {
            'data': self.data,
            'link': self.link,
            'endmarker': self.endmarker,
        }
        return creator, args, state, iter([]), iter({})

    def copy(self):
        """Return a deep copy."""
        import copy
        return copy.deepcopy(self)

    def batches(self, label, n, permutation=None):
        """Yield batches of the size of n from the dataset.

        A single batch is an array of with dim columns and n rows. The last
        batch is possibly smaller.

        If permutation is given, batches are yielded in the corresponding
        order."""
        # First calculate how many batches we will have
        full_batches, rest = divmod(len(self), n)
        number_of_batches = full_batches if rest == 0 else full_batches + 1

        # We make one iterator for the startindexes ...
        startindexes = (i * n for i in range(number_of_batches))
        # ... and one for the stop indexes
        stopindexes = (((i + 1) * n) for i in range(number_of_batches - 1))
        # The last stop index is the last element of the list (last batch
        # might not be filled completely)
        stopindexes = chain(stopindexes, [len(self)])
        # Now combine them
        indexes = list(zip(startindexes, stopindexes))

        # Shuffle them according to the permutation if one is given
        if permutation is not None:
            indexes = [indexes[i] for i in permutation]

        for start, stop in indexes:
            yield self.data[label][start:stop]

    def randomBatches(self, label, n):
        """Like .batches(), but the order is random."""
        permutation = random.shuffle(list(range(len(self))))
        return self.batches(label, n, permutation)

    def replaceNansByMeans(self):
        """Replace all not-a-number entries in the dataset by the means of the
        corresponding column."""
        for d in self.data.values():
            means = scipy.nansum(d[:self.getLength()], axis=0) / self.getLength()
            for i in range(self.getLength()):
                for j in range(d.dim):
                    if not scipy.isfinite(d[i, j]):
                        d[i, j] = means[j]

reason: import scipy,from scipy