#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <err.h>

#include <gsl/gsl_rng.h>
#include <tskit/tables.h>

#define check_tsk_error(val) if (val < 0) {\
    errx(EXIT_FAILURE, "line %d: %s", __LINE__, tsk_strerror(val));\
}

void
simulate(tsk_table_collection_t *tables, int N, int T, int simplify_interval, gsl_rng *rng)
{
    tsk_id_t *buffer, *parents, *children, child, left_parent, right_parent;
    double breakpoint;
    int ret, j, t, b;

    assert(simplify_interval != 0); // leads to division by zero
    buffer = malloc(2 * N * sizeof(tsk_id_t));
    if (buffer == NULL) {
        errx(EXIT_FAILURE, "Out of memory");
    }
    tables->sequence_length = 1.0;
    parents = buffer;
    for (j = 0; j < N; j++) {
        parents[j] = tsk_node_table_add_row(&tables->nodes, 0, T,
                TSK_NULL, TSK_NULL, NULL, 0);
        check_tsk_error(parents[j]);
    }
    b = 0;
    for (t = T - 1; t >= 0; t--) {
        /* Alternate between using the first and last N values in the buffer */
        parents = buffer + (b * N);
        b = (b + 1) % 2;
        children = buffer + (b * N);
        for (j = 0; j < N; j++) {
            child = tsk_node_table_add_row(&tables->nodes, 0, t,
                    TSK_NULL, TSK_NULL, NULL, 0);
            check_tsk_error(child);
            left_parent = parents[gsl_rng_uniform_int(rng, N)];
            right_parent = parents[gsl_rng_uniform_int(rng, N)];
            do {
                breakpoint = gsl_rng_uniform(rng);
            } while (breakpoint == 0); /* tiny proba of breakpoint being 0 */
            ret = tsk_edge_table_add_row(&tables->edges, 0, breakpoint, left_parent, child);
            check_tsk_error(ret);
            ret = tsk_edge_table_add_row(&tables->edges, breakpoint, 1, right_parent, child);
            check_tsk_error(ret);
            children[j] = child;
        }
        if (t % simplify_interval == 0) {
            printf("Simplify at generation %d: (%d nodes %d edges)", t,
                    tables->nodes.num_rows, tables->edges.num_rows);
            /* Note: Edges must be sorted for simplify to work, and we use a brute force
             * approach of sorting each time here for simplicity. This is inefficient. */
            ret = tsk_table_collection_sort(tables, NULL, 0);
            check_tsk_error(ret);
            ret = tsk_table_collection_simplify(tables, children, N, 0, NULL);
            check_tsk_error(ret);
            printf(" -> (%d nodes %d edges)\n", tables->nodes.num_rows,
                    tables->edges.num_rows);
            for (j = 0; j < N; j++) {
                children[j] = j;
            }
        }
    }
    free(buffer);
}

int
main(int argc, char **argv)
{
    int ret;
    tsk_table_collection_t tables;
    gsl_rng *rng = gsl_rng_alloc(gsl_rng_default);

    if (argc != 5) {
        errx(EXIT_FAILURE, "usage: N T simplify-interval output-file");
    }
    ret = tsk_table_collection_init(&tables, 0);
    check_tsk_error(ret);
    simulate(&tables, atoi(argv[1]), atoi(argv[2]), atoi(argv[3]), rng);
    ret = tsk_table_collection_dump(&tables, argv[4], 0);
    check_tsk_error(ret);

    tsk_table_collection_free(&tables);
    gsl_rng_free(rng);
    return 0;
}


#ifndef OPS_REDUCERS_H
#define OPS_REDUCERS_H

#include <fundamental/tensor.h>
#include <nn/container.h>

#ifdef TENSOR_USE_CUBLAS
  #include <cublas_v2.h>
#endif

#ifdef TENSOR_USE_OPENBLAS
  #include <cblas.h>
#endif

#include <ops/transform.h>

namespace operators{

  template<typename xpu, index_t stream_id, typename T>
  void asum_backward(nn::TensorContainer<xpu, stream_id, T>& x,
                     nn::TensorContainer<xpu, stream_id, T>& y,
                     int offset, int inc, int n){

     auto bwd = [](nn::TensorContainer<xpu, stream_id, float> x,
                   nn::TensorContainer<xpu, stream_id, float> y,
                   int offset, int inc, int n)
     {

       auto strided_x = expression::transform::stride(x, offset, inc, n);

       core::Shape shape = strided_x.self().shape();

       //auto repeated_y = expression::transform::repeat(shape.size(), yexp);
       core::Constant<xpu, T, stream_id> repeated_y(y.gradient().scalar(),
                                                    core::Shape(x.shape().size()));

       expression::Exceturer<xpu>::backward(repeated_y, strided_x);

     };

     STREAM_BACKWARD(xpu, stream_id).put(bwd, x, y, offset, inc, n);

   }

  template<index_t stream_id>
  void asum(nn::TensorContainer<cpu, stream_id, float>& x,
            nn::TensorContainer<cpu, stream_id, float>& y,
            int offset, int inc, int n){

    //y.conditional_override(core::Shape(1));

    ASSERT_ERROR( ((n+offset) - (x.data().shape().size())) <= 0 , " Out of bound ");

    auto fwd = [](nn::TensorContainer<cpu, stream_id, float> x,
                  nn::TensorContainer<cpu, stream_id, float> y,
                  int offset, int inc, int n)
    {
      float* ptr_offseted = x.data().ptr() + offset;
      float result = cblas_sasum(n, ptr_offseted, inc);
      y.data().set(0, result);
    };

    STREAM_FORWARD(cpu, stream_id).put(fwd, x, y, offset, inc, n);

    if (x.require_grad() && y.require_grad())
      asum_backward(x, y, offset, inc, n);

  }

  #ifdef TENSOR_USE_CUBLAS

    template<index_t stream_id>
    void asum(nn::TensorContainer<gpu, stream_id, float>& x,
              nn::TensorContainer<gpu, stream_id, float>& y,
              int offset, int inc, int n){

      DEBUG_ASSERT(y.data().shape().size() == 1);
      ASSERT_ERROR( ((n+offset) - (x.data().shape().size())) <= 0 , " Out of bound ");

      auto fwd = [](nn::TensorContainer<gpu, stream_id, float> x,
                    nn::TensorContainer<gpu, stream_id, float> y,
                    int offset, int inc, int n)
      {
        auto cublasHandle = STREAM_FORWARD(gpu, stream_id).cublasHandle();
        float result;
        float* ptr_offseted = x.data().ptr() + offset;
        CUDBLAS_CHECK_ERROR_ENFORCE(cublasSasum(cublasHandle, n,
                                                ptr_offseted, inc,
                                                &result));
        y.data().set(0, result);

      };

      STREAM_FORWARD(gpu, stream_id).put(fwd, x, y, offset, inc, n);

      if (x.require_grad() && y.require_grad())
        asum_backward(x, y, offset, inc, n);

    }

  #endif

}

#ifdef TENSOR_USE_CUDNN

  namespace cudnn {

    template<cudnnReduceTensorOp_t OPERATION, index_t stream_id>
    void reduce_cudnn_i(core::Tensor<gpu, stream_id, float>& tensorA,
                        core::Tensor<gpu, stream_id, float>& tensorB,
                        core::Tensor<gpu, stream_id, indices_t>& indicies,
                        float alpha = 1, float beta = 0){

      DEBUG_ASSERT( indicies.shape() == tensorB.shape() );

      cudnnReduceTensorDescriptor_t reduce;
      cudnnCreateReduceTensorDescriptor(&reduce);

      CHECK_CUDNN( cudnnSetReduceTensorDescriptor(
                   reduce,
                   OPERATION,
                   CUDNN_DATA_FLOAT,
                   CUDNN_NOT_PROPAGATE_NAN,
                   CUDNN_REDUCE_TENSOR_FLATTENED_INDICES,
                   CUDNN_32BIT_INDICES) );

      size_t workspaces_sz;

      auto cudnnhandle = STREAM_FORWARD(gpu, stream_id).cudnnHandle();

      CHECK_CUDNN( cudnnGetReductionWorkspaceSize(cudnnhandle,
                                                  reduce,
                                                  tensorA.descriptor(),
                                                  tensorB.descriptor(),
                                                  &workspaces_sz) );

      auto workspace_ptr = cuda::generate_workspace<void>(workspaces_sz);

      CHECK_CUDNN( cudnnReduceTensor(
                                     cudnnhandle,
                                     reduce, // reduceTensorDesc
                                     indicies.ptr(), // indicies
                                     sizeof(indices_t) * indicies.shape().size(),  // indicesSizeInBytes,
                                     workspace_ptr, // workspace,
                                     workspaces_sz, //workspaceSizeInBytes,
                                     (void*) &alpha,
                                     tensorA.descriptor(),
                                     (void*) tensorA.ptr(),
                                     (void*) &beta,
                                     tensorB.descriptor(),
                                     (void*) tensorB.ptr()) ) ;

      for (int i = 0; i < indicies.shape()[0];  ++i)
        printf(" %f\n", tensorB.at(i) );

    }

    template<cudnnReduceTensorOp_t OPERATION, index_t stream_id>
    void reduce_cudnn(core::Tensor<gpu, stream_id, float>& tensorA,
                      core::Tensor<gpu, stream_id, float>& tensorB,
                      float alpha = 1, float beta = 0){

      cudnnReduceTensorDescriptor_t reduce;
      cudnnCreateReduceTensorDescriptor(&reduce);

      CHECK_CUDNN( cudnnSetReduceTensorDescriptor(
                   reduce,
                   OPERATION,
                   CUDNN_DATA_FLOAT,
                   CUDNN_NOT_PROPAGATE_NAN,
                   CUDNN_REDUCE_TENSOR_NO_INDICES,
                   CUDNN_32BIT_INDICES) );

      size_t workspaces_sz;

      auto cudnnhandle = STREAM_FORWARD(gpu, stream_id).cudnnHandle();

      CHECK_CUDNN( cudnnGetReductionWorkspaceSize(cudnnhandle,
                                                  reduce,
                                                  tensorA.descriptor(),
                                                  tensorB.descriptor(),
                                                  &workspaces_sz) );

      auto workspace_ptr = cuda::generate_workspace<void>(workspaces_sz);

      float dummy;

      CHECK_CUDNN( cudnnReduceTensor(
                                     cudnnhandle,
                                     reduce, // reduceTensorDesc
                                     &dummy, // indicies
                                     0,  // indicesSizeInBytes,
                                     workspace_ptr, // workspace,
                                     workspaces_sz, //workspaceSizeInBytes,
                                     (void*) &alpha,
                                     tensorA.descriptor(),
                                     (void*) tensorA.ptr(),
                                     (void*) &beta,
                                     tensorB.descriptor(),
                                     (void*) tensorB.ptr()) ) ;

    }

  }

#endif

namespace operators{

  #ifdef TENSOR_USE_CUDNN

    template<index_t stream_id>
    void reduce_sum(core::Tensor<gpu, stream_id, float>& tensorA,
                    core::Tensor<gpu, stream_id, float>& tensorB,
                    float alpha = 1, float beta = 0){
      cudnn::reduce_cudnn<CUDNN_REDUCE_TENSOR_ADD, stream_id>
              (tensorA, tensorB, alpha, beta);
    }

    template<index_t stream_id>
    void reduce_max(core::Tensor<gpu, stream_id, float>& tensorA,
                    core::Tensor<gpu, stream_id, float>& tensorB,
                    float alpha = 1, float beta = 0){
      cudnn::reduce_cudnn<CUDNN_REDUCE_TENSOR_MAX, stream_id>
              (tensorA, tensorB, alpha, beta);
    }

    template<index_t stream_id>
    void reduce_min(core::Tensor<gpu, stream_id, float>& tensorA,
                    core::Tensor<gpu, stream_id, float>& tensorB,
                    float alpha = 1, float beta = 0){
      cudnn::reduce_cudnn<CUDNN_REDUCE_TENSOR_MIN, stream_id>
              (tensorA, tensorB, alpha, beta);
    }

    template<index_t stream_id>
    void reduce_max_i(core::Tensor<gpu, stream_id, float>& tensorA,
                      core::Tensor<gpu, stream_id, float>& tensorB,
                      core::Tensor<gpu, stream_id, indices_t>& indices,
                      float alpha = 1, float beta = 0){
      cudnn::reduce_cudnn_i<CUDNN_REDUCE_TENSOR_MAX, stream_id>
              (tensorA, tensorB, indices, alpha, beta);
    }

    template<index_t stream_id>
    void reduce_min_i(core::Tensor<gpu, stream_id, float>& tensorA,
                      core::Tensor<gpu, stream_id, float>& tensorB,
                      core::Tensor<gpu, stream_id, indices_t>& indices,
                      float alpha = 1, float beta = 0){
      cudnn::reduce_cudnn_i<CUDNN_REDUCE_TENSOR_MIN, stream_id>
              (tensorA, tensorB, indices, alpha, beta);
    }

    template<index_t stream_id>
    TENSOR_INLINE_HOST void reduce_min(
        std::shared_ptr<nn::TensorContainer<gpu, stream_id, float>> A,
        std::shared_ptr<nn::TensorContainer<gpu, stream_id, float>> B,
        index_t axis){

        B->conditional_override(A->shape().reduce_axis(axis));

        if (B->require_grad()){
          // Create tensor for incicies
          core::Tensor<gpu, stream_id, float> I(B->shape()); I.allocate();

          auto fwd = [](core::Tensor<gpu, stream_id, float> A,
                        core::Tensor<gpu, stream_id, float> B,
                        core::Tensor<gpu, stream_id, float> I){
            reduce_min_i(A, B, I);
          };

          STREAM_FORWARD(gpu, stream_id).put(fwd, A->data(), B->data(), I);

        }else{

          auto fwd = [](core::Tensor<gpu, stream_id, float> A,
                        core::Tensor<gpu, stream_id, float> B){
            reduce_min(A, B);
          };

          STREAM_FORWARD(gpu, stream_id).put(fwd, A->data(), B->data());

        }

    }

  #endif

}

#endif //OPS_REDUCERS_H


#ifdef __cplusplus
extern "C" {
#endif

#include "src/allvars.h"
#include "src/proto.h"
#include "src/tags.h"
#include <gsl/gsl_rng.h>

typedef struct {
    double mass;                                        /// mass
    double x, y, z;                                     /// position
    double vx, vy, vz;                                  /// velocity
} dynamics_state;

typedef struct {
    double mass;                                        /// mass
    double x, y, z;                                     /// position
    double vx, vy, vz;                                  /// velocity
    double u;                                           /// entropy
#ifdef MORRIS97VISC
    double alpha, dalphadt;				///viscosity
#endif
} sph_state;

void   begrun(void);
double second(void);

int found_particle(int index_of_the_particle, int *local_index);
void update_particle_map(void);

void hydro_state_at_point(FLOAT pos[3], FLOAT vel[3], FLOAT *h_out,
  FLOAT *ngb_out, FLOAT *dhsml_out, FLOAT *rho_out, FLOAT *rhov_out,
  FLOAT *rhov2_out, FLOAT *rhoe_out);

#ifdef __cplusplus
}
#endif


 // MIT License

 // Copyright (c) [2017] [Vinay Yuvashankar]

 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
 // in the Software without restriction, including without limitation the rights
 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 // copies of the Software, and to permit persons to whom the Software is
 // furnished to do so, subject to the following conditions:

 // The above copyright notice and this permission notice shall be included in all
 // copies or substantial portions of the Software.

 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.

#include <assert.h>
#include "wavelet.h"
#include <inttypes.h>
#include <gsl/gsl_statistics.h>
#include <omp.h>

int main(int argc, char const *argv[])
{
    if (argc < 2)
    {
        printf("Not enough input arguments please provide a BDF file\n");
        return(-1);
    }

    double t = omp_get_wtime();

    //Variable Declarations
    int handle,
        numberOfChannels,
        openFlag,
        channel,
        filteredTriggerNumber,
        readFlag,
        writeFlag,
        samplesToRead,
        numberOfTriggers,
        read_channel,
        min_i,
        max_i,
        J; //needed by Wavelet

    int *rawStatus,
        *buffer,
        *value,
        *filteredBuffer;

    double  sampleFrequency,
            dt;

    int64_t numberOfRecords;

    double  *tempBuffer,
            *data,
            *result, //The Final Result
            *scales,
            *frequency; //The Corrosponding Frequencies

    char    graph_title[256],
            file_name[256];

    int64_t * triggerList;

    struct edf_hdr_struct edfHeader;

    // Debug File ... just in case.
    // FILE *debug_file = fopen("Main_debug.log", "w");
    // assert(debug_file != NULL);

    //Opening and reading the file into the edfHeader input the first argument is the input file.
    openFlag = OpenFile(argv[2], &edfHeader);
    read_channel = atoi(argv[1]);
    assert( openFlag == 0 );

    //Get File Information
    handle           = edfHeader.handle;
    sampleFrequency  = (( double ) edfHeader.signalparam[1].smp_in_datarecord /
                        ( double ) edfHeader.datarecord_duration              ) * EDFLIB_TIME_DIMENSION;
    numberOfChannels = edfHeader.edfsignals;
    numberOfRecords  = edfHeader.signalparam[numberOfChannels - 1].smp_in_file;
    channel          = numberOfChannels - 1; //The status channel.
    samplesToRead    = (PRE_EVENT_TIME + POST_EVENT_TIME) * sampleFrequency;

    //Generate Constants for the CWT
    dt               = 1.0/sampleFrequency;
    min_i            = floor( ( log2( (W_0) / (dt * 2 * M_PI * sampleFrequency/2.0) ) )/D_J);
    max_i            = floor( ( log2( (W_0) / (dt * 2 * M_PI * MIN_FREQUENCY) ) )      /D_J);
    assert(min_i > 0); assert(max_i > 0);

    J     = (int) max_i - min_i;

    /*Allocate Necessary Memory*/
    rawStatus   =    (int*)     malloc( numberOfRecords  * sizeof   (int)     );
    triggerList =    (int64_t*) malloc( MAXIMUM_TRIGGERS * sizeof   (int64_t) );
    tempBuffer  =    (double*)  malloc( samplesToRead    * sizeof   (double)  );
    assert(rawStatus != NULL); assert(triggerList!= NULL); assert(tempBuffer!= NULL);

    //Read the status Signal --> Output to rawStatus
    readFlag = edfread_digital_samples(handle, channel, numberOfRecords, rawStatus);
    assert (readFlag != 1);

    //Find and parse the file and find the triggers -->  Output to TriggerList
    numberOfTriggers = FindTriggers(rawStatus, numberOfRecords, triggerList);
    assert (numberOfTriggers != -1);

    //Allocating Read Buffer
    buffer         = (int *) malloc( numberOfTriggers * sizeof(int) );
    filteredBuffer = (int *) malloc( numberOfTriggers * sizeof(int) );
    assert (buffer!= NULL); assert(filteredBuffer != NULL);

    scales         = (double*) malloc ( J * sizeof(double) );
    frequency      = (double*) malloc ( J * sizeof(double) );
    assert(scales != NULL); assert(frequency != NULL);

    //Wavelet Memory Allocations
    data           = (double*) malloc( samplesToRead * filteredTriggerNumber * sizeof(double));
    result         = (double*) malloc( J * samplesToRead * sizeof(double));
    assert(result != NULL); assert(data != NULL);

    /*Main Program Begins*/
    GenerateScalesAndFrequency(min_i, max_i, dt, scales, frequency);

    //Read the status channel when there is a trigger and put it in the buffer
    value = (int*) malloc( sizeof(int) );
    for (int i = 0; i < numberOfTriggers; ++i)
    {
        edfseek(handle, channel, triggerList[i], EDFSEEK_SET);

        readFlag = edfread_digital_samples(handle, channel, 1, value);
        assert(readFlag != -1);
        buffer[i] = *value;
    }

    //Filter the Triggers to what you want.
    filteredTriggerNumber = FilterTriggers(1, 2, numberOfTriggers, triggerList,
                                            buffer, filteredBuffer);
    printf("Number of Filtered Triggers Found: %d\n", filteredTriggerNumber);

    printf("Beginning Wavelet Analysis\n");
    for (int i = 0; i < filteredTriggerNumber; ++i)
    {
        edfseek(handle, 0, filteredBuffer[i], EDFSEEK_SET);

        readFlag = edfread_physical_samples(handle, read_channel, samplesToRead, tempBuffer);
        // TestCases( tempBuffer, 5, 16.0 , sampleFrequency, samplesToRead);

        // Preform a Z-Score on the read data.
        CleanData(tempBuffer, samplesToRead);

        for (int j = 0; j < samplesToRead; ++j)
        {
            data[i * samplesToRead + j] = tempBuffer[j];
        }
    }

    ERSP (data, scales, sampleFrequency, samplesToRead,
                J, filteredTriggerNumber, 1,
                result);

    printf("Writing to File\n");
    sprintf(graph_title, "Channel %d", read_channel);
    sprintf(file_name,   "Channel_%d.log", read_channel);

    writeFlag = WriteFile(result, frequency, J, samplesToRead, sampleFrequency,
                          file_name);
    WriteGnuplotScript(graph_title, file_name);

    assert(writeFlag != -1);

    printf("Freeing Memory and closing files\n");
    edfclose_file(handle);

    free(data);   free(result);
    free(scales); free(frequency);
    free(buffer); free(filteredBuffer);
    free(rawStatus); free(triggerList); free(tempBuffer);

    t = omp_get_wtime() - t;

    printf("Execution Time = %f\n", t);

    return 0;
}


/*

   Copyright 2016  Amazon.com, Inc. or its affiliates. All Rights Reserved.

   Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at

   http://aws.amazon.com/apache2.0/

   or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 */

#ifndef NNTYPES_H
#define NNTYPES_H
#include <vector>
#include <set>
#include <string>
#include <iostream>
#include <iomanip>
#include <fstream>
#include <algorithm>
#include <netcdf>
#ifndef __NVCC__
#include <tuple>
#include <json/json.h>
#endif
#include <cmath>
#include <memory>

class NNDataSetBase;
class NNLayer;
class NNNetwork;
class NNWeight;

// Activates step by step CPU validation
#define VALIDATION
#ifdef VALIDATION
extern "C"
{
    #include <cblas.h>
}
#endif

static const float NN_VERSION       = 0.9f;
static const float MIN_ERROR        = 1.0e-12f;
static const float MIN_ACTIVATION   = 0.000001f;
static const float MAX_ACTIVATION   = 0.999999f;
static const float MAX_VALUE        = 999999999999999.0f;

template <typename T> struct GpuBuffer;

enum
{
    DefaultBatch    = 512
};

enum Mode {
    Prediction = 0,
    Training = 1,
    Validation = 2,
    Unspecified = 3
};

enum TrainingMode
{
    SGD = 0,
    Momentum = 1,
    AdaGrad = 2,
    Nesterov = 3,
    RMSProp = 4,
    AdaDelta = 5,
    Adam = 6,
};

ostream& operator<< (ostream& out, const TrainingMode& e);

enum ErrorFunction
{
    L1,
    L2,
    CrossEntropy,
    ScaledMarginalCrossEntropy,
    DataScaledMarginalCrossEntropy,
    Hinge,
    L2Hinge,
};

ostream& operator<< (ostream& out, const ErrorFunction& e);

enum Activation {
    Sigmoid,
    Tanh,
    RectifiedLinear,
    Linear,
    ParametricRectifiedLinear,
    SoftPlus,
    SoftSign,
    SoftMax,
    RELUMax,
    LinearMax,
    ExponentialLinear,
    LeakyRectifiedLinear,
    ScaledExponentialLinear,
};

ostream& operator<< (ostream& out, const Activation& a);

enum WeightInitialization
{
    Xavier,
    CaffeXavier,
    Gaussian,
    Uniform,
    UnitBall,
    Constant,
    SELU,
};

ostream& operator<< (ostream& out, const WeightInitialization& w);

enum PoolingFunction {
    None,
    Max,
    Average,
    LRN,
    Maxout,
    DotProduct,
    Cosine,
    Stochastic,
    LCN,
    GlobalTemporal,
};

ostream& operator<< (ostream& out, const PoolingFunction& p);

#include "kernels.h"
#include "GpuSort.h"
#include "NNEnum.h"
#include "NNWeight.h"
#include "NNLayer.h"
#include "NNNetwork.h"

int MPI_Bcast_string(string& s);

struct NNDataSetDimensions
{
    uint32_t _dimensions;
    uint32_t _width;
    uint32_t _height;
    uint32_t _length;

    NNDataSetDimensions();

    NNDataSetDimensions(uint32_t width, uint32_t height = 1, uint32_t length = 1);
};

struct NNDataSetDescriptor
{
    string _name;                       // dataset name
    NNDataSetEnums::DataType _dataType; // dataset type
    uint32_t _attributes;               // dataset attributes
    NNDataSetDimensions _dim;           // dataset dimensions
    uint32_t _examples;                 // number of examples in this dataset
    float _sparseDensity;               // sparseness density of this dataset

    /**
     * Only support vanilla sparse and dense datasets for now
     */
    static bool isSupported(uint32_t attributes)
    {
        using NNDataSetEnums::Attributes;

        static const vector<Attributes> SUPPORTED_ATTRIBUTES(Attributes::Sparse);
        for (auto mask : SUPPORTED_ATTRIBUTES)
        {
            if (attributes & mask)
            {
                attributes -= mask;
            }
        }
        return attributes == 0;
    }
};

/**
 * Creates a new NNDatSetBase object given the descriptor.
 * Only attributes specified by the NNDataSetDescriptor::isSupported
 * is valid. If any other attributes are specified, a std::runtime_error
 * is thrown. It is the caller's responsibility to delete the returned object
 * (not wrapping the return value with a unique_ptr to be consistent with
 * the rest of the code base).
 */
NNDataSetBase* createNNDataSet(const NNDataSetDescriptor &descriptor);

struct NNDataSetBase {

    string                          _name;                          // Dataset name
    NNDataSetEnums::DataType        _dataType;                      // Dataset type (see above enum)
    uint32_t                        _attributes;                    // Dataset characteristics (see NNDataSetEnum::Attributes in NNEnum.h)
    uint32_t                        _examples;                      // Number of examples
    uint32_t                        _uniqueExamples;                // Number of unique examples for indexed data (set to examples if unindexed)
    uint32_t                        _localExamples;                 // Number of local examples when data sharded
    uint32_t                        _dimensions;                    // Dimensionality of data set
    uint32_t                        _width;                         // Dataset x dimension
    uint32_t                        _height;                        // Dataset y dimension
    uint32_t                        _length;                        // Dataset z dimension
    uint32_t                        _stride;                        // Stride between examples
    NNDataSetEnums::Sharding        _sharding;                      // Sharding of dataset for parallel execution
    uint32_t                        _minX;                          // Beginning of local X sharding for model parallel execution
    uint32_t                        _maxX;                          // End of local X sharding for model parallel execution
    uint64_t                        _sparseDataSize;                // Total sparse datapoints
    NNFloat                         _sparseDensity;                 // Overall sparse density (0.0 - 1.0)
    vector<uint64_t>                _vSparseStart;                  // Vector of sparse datapoint starts per example
    unique_ptr<GpuBuffer<uint64_t>> _pbSparseStart;                 // GPU copy of _vSparseStart
    vector<uint64_t>                _vSparseEnd;                    // Vector of sparse datapoint ends per example
    unique_ptr<GpuBuffer<uint64_t>> _pbSparseEnd;                   // GPU copy of _vSparseEnd
    vector<uint32_t>                _vSparseIndex;                  // Vector of sparse indices
    unique_ptr<GpuBuffer<uint32_t>> _pbSparseIndex;                 // GPU copy of _vSparseIndex
    vector<NNFloat>                 _vDataWeight;                   // Per example sparse index weights
    unique_ptr<GpuBuffer<NNFloat>>  _pbDataWeight;                  // GPU copy of _vDataWeight
    vector<uint32_t>                _vIndex;                        // Indexed data array
    unique_ptr<GpuBuffer<uint32_t>> _pbIndex;                       // GPU copy of _vIndex;
    unique_ptr<GpuBuffer<NNFloat>>  _pbDenoisingRandom;             // Denoising randoms

    // Transposed sparse lookup for sparse backpropagation
    vector<uint64_t>                _vSparseDatapointCount;
    vector<uint32_t>                _vSparseMaxDatapointCount;
    vector<uint32_t>                _vSparseMultiDatapointCount;
    vector<uint32_t>                _vSparseTransposedStart;
    uint64_t                        _sparseTransposedIndices;
    unique_ptr<GpuBuffer<uint32_t>> _pbSparseTransposedStart;
    unique_ptr<GpuBuffer<uint32_t>> _pbSparseTransposedEnd;
    unique_ptr<GpuBuffer<uint32_t>> _pbSparseTransposedIndex;
    unique_ptr<GpuBuffer<NNFloat>>  _pbSparseTransposedData;

    // States
    bool                            _bDenoising;
    bool                            _bDirty;
    bool                            _bStreaming;
    bool                            _bIndexed;
    uint32_t                        _batch;

    NNDataSetBase();
    NNDataSetDimensions GetDimensions();
    uint32_t GetExamples() { return _examples; };
    uint32_t GetUniqueExamples() { return _uniqueExamples; };

    virtual bool SaveNetCDF(const string& fname) = 0;
    virtual bool WriteNetCDF(netCDF::NcFile& nfc, const string& fname, const uint32_t n) = 0;
    virtual ~NNDataSetBase() = 0;
    virtual void RefreshState(uint32_t batch) = 0;
    virtual bool Shard(NNDataSetEnums::Sharding sharding) = 0;
    virtual bool UnShard() = 0;
    virtual bool SetStreaming(bool flag) = 0;
    virtual bool GetStreaming() = 0;
    virtual vector<tuple<uint64_t, uint64_t> > getMemoryUsage() = 0;
    virtual bool CalculateSparseDatapointCounts() = 0;
    virtual bool GenerateSparseTransposedMatrix(uint32_t batch, NNLayer* pLayer) = 0;
    virtual bool CalculateSparseTransposedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer) = 0;
    virtual bool CalculateSparseTransposedDenoisedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer) = 0;
    virtual bool CalculateSparseTransposedWeightGradient(NNFloat alpha, NNFloat beta, uint32_t m, uint32_t n, NNFloat* pDelta, NNFloat* pWeightGradient) = 0;
    virtual bool SetDenoising(bool flag) = 0;
    virtual bool GenerateDenoisingData() = 0;
    virtual bool LoadInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool LoadSparseInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool LoadSparseDenoisedInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool CalculateSparseZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta = (NNFloat)0.0) = 0;
    virtual bool CalculateSparseDenoisedZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta = (NNFloat)0.0) = 0;
    virtual float CalculateL1Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateL2Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateL2HingeError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateMultinomialCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateMultinomialScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateDataScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateHingeError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool CalculateL1OutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda) = 0;
    virtual bool CalculateCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;
    virtual bool CalculateScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;
    virtual bool CalculateOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda) = 0;
    virtual bool CalculateL2HingeOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda) = 0;
    virtual bool CalculateDataScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;
    virtual bool CalculateHingeOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;

    /**
     * Copies data from srcData to this dataset and uploads to GPU.
     * The length to copy is determined by the example size and stride
     * of this dataset. Only valid for dense and dense indexed datasets.
     * If the dataset is indexed this sets the unique data
     * and NNDataSet::SetIndexedData sets the actual examples.
     * An exception is thrown if this dataset is not dense.
     */
    virtual void LoadDenseData(const void *srcData) = 0;

    /**
     * Copies data from srcData to this dataset but doesn't upload to GPU.
     */
    virtual void CopyDenseData(const void *srcData) = 0;

    /**
     * Copies sparse data points from the specified src to this NNDataSet
     * and uploads to GPU. Any existing data is overwritten.
     * An exception is thrown if this dataset is not sparse.
     */
    virtual void LoadSparseData(const uint64_t *srcSparseStart, const uint64_t *srcSparseEnd, const void *srcSparseData,
                                const uint32_t *srcSparseIndex) = 0;

    /**
     * Copies sparse data points from the specified src to this NNDataSet
     * but doesn't upload to GPU. Any existing data is overwritten.
     * An exception is thrown if this dataset is not sparse.
     */
    virtual void CopySparseData(const uint64_t *srcSparseStart, const uint64_t *srcSparseEnd, const void *srcSparseData,
                                const uint32_t *srcSparseIndex) = 0;

    /**
     * Same as LoadSparseData(const uint64_t*, const uint64_t*, const void*, const uint32_t*) except that
     * it takes long* for the sparse start, end and indexes and casts them into
     * uint64_t and uint32_t during load. It is up to the caller to ensure that the cast is bounds safe
     * (e.g. no negative indexes). This method is useful when writing language extensions for those languages
     * that do not have unsigned primitive types (e.g. Java).
     */
    virtual void LoadSparseData(const long *srcSparseStart, const long *srcSparseEnd, const void *srcSparseData,
                                const long *srcSparseIndex) = 0;

      /**
     * Same as CopySparseData(const uint64_t*, const uint64_t*, const void*, const uint32_t*) except that
     * it takes long* for the sparse start, end and indexes and casts them into
     * uint64_t and uint32_t during load. It is up to the caller to ensure that the cast is bounds safe
     * (e.g. no negative indexes). This method is useful when writing language extensions for those languages
     * that do not have unsigned primitive types (e.g. Java).
     */
    virtual void CopySparseData(const long *srcSparseStart, const long *srcSparseEnd, const void *srcSparseData,
                                const long *srcSparseIndex) = 0;

    /**
     * If this dataset is indexed, then sets the actual (unique) examples.
     * An exception is thrown if this dataset is not indexed.
     */
    virtual void LoadIndexedData(const uint32_t *srcIndexedData) = 0;

    /**
     * If this dataset is weighted, then sets the weights for each example.
     * An exception is thrown if this dataset is not weighted.
     */
    virtual void LoadDataWeight(const NNFloat *srcWeightData) = 0;

 protected:
    NNDataSetBase(const string &name, NNDataSetEnums::DataType dataType, uint32_t examples, uint32_t uniqueExamples,
                  const NNDataSetDimensions &datasetDim);

};

ostream& operator<< (ostream& out, NNDataSetEnums::Attributes& a);
ostream& operator<< (ostream& out, NNDataSetEnums::Kind& k);
ostream& operator<< (ostream& out, NNDataSetEnums::DataType& t);
ostream& operator<< (ostream& out, NNDataSetEnums::Sharding& s);

template<typename T> class NNDataSet : public NNDataSetBase {
public:
    friend class NNetwork;
    friend class NNLayer;
    friend vector<NNDataSetBase*> LoadNetCDF(const string& fname);
    friend bool SaveNetCDF(const string& fname, vector<NNDataSetBase*> vDataSet);

private:
    // Type-specific data
    vector<T>                   _vData;
    unique_ptr<GpuBuffer<T>>    _pbData;
    vector<T>                   _vSparseData;
    unique_ptr<GpuBuffer<T>>    _pbSparseData;

    // Force constructor private
    NNDataSet(const string& fname, uint32_t n);
    bool Rename(const string& name);
    bool SaveNetCDF(const string& fname);
    bool WriteNetCDF(netCDF::NcFile& nfc, const string& fname, const uint32_t n);
    void RefreshState(uint32_t batch) {}
    bool Shard(NNDataSetEnums::Sharding sharding);
    bool UnShard();
    vector<tuple<uint64_t, uint64_t> > getMemoryUsage();
    bool CalculateSparseDatapointCounts();
    bool GenerateSparseTransposedMatrix(uint32_t batch, NNLayer* pLayer);
    bool CalculateSparseTransposedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer);
    bool CalculateSparseTransposedDenoisedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer);
    bool CalculateSparseTransposedWeightGradient(NNFloat alpha, NNFloat beta, uint32_t m, uint32_t n, NNFloat* pDelta, NNFloat* pWeightGradient);
    bool SetStreaming(bool flag);
    bool GetStreaming();
    bool SetDenoising(bool flag);
    bool GenerateDenoisingData();
    bool LoadInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool LoadSparseInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool LoadSparseDenoisedInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool CalculateSparseZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta);
    bool CalculateSparseDenoisedZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta);
    float CalculateL1Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateL2Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateL2HingeError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateMultinomialCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateMultinomialScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateDataScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateHingeError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool CalculateL1OutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda);
    bool CalculateCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);
    bool CalculateScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);
    bool CalculateOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda);
    bool CalculateL2HingeOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda);
    bool CalculateDataScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);
    bool CalculateHingeOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);

public:
    /**
     * Creates dense dataset with the specified dimensions and name
     * with space allocated for the specified number of examples
     */
    NNDataSet(uint32_t examples, const NNDataSetDimensions &dim, const string &name = "");

    /**
     * Creates dense indexed dataset with the specified dimensions and name
     * with space allocated for the specified number of examples
     * and index data space allocated for the specified number of uniqueExamples.
     */
    NNDataSet(uint32_t examples, uint32_t uniqueExamples, const NNDataSetDimensions &dim, const string &name = "");

    /**
     * Creates sparse dataset for the layer with space allocated
     * for the specified number of examples. The isWeighted parameter
     * can be set to true to create a sparse weighted dataset.
     */
    NNDataSet(uint32_t examples, NNFloat sparseDensity, const NNDataSetDimensions &dim, bool isWeighted = false, const string &name = "");

    /**
     * Creates a sparse dataset with the specified dimensions
     * and name with space allocated for the specified number of
     * examples and index data (if isIndexed).
     * The isIndexed and isWeighted parameters can be set to
     * create a sparse indexed, weighted dataset.
     */
    NNDataSet(uint32_t examples, uint32_t uniqueExamples, size_t sparseDataSize, const NNDataSetDimensions &dim,
              bool isIndexed = false, bool isWeighted = false, const string &name = "");

    void LoadDenseData(const void *srcData) override;

    void CopyDenseData(const void *srcData) override;

    void LoadSparseData(const uint64_t *srcSparseStart, const uint64_t *srcSparseEnd, const void *srcSparseData,
                        const uint32_t *srcSparseIndex) override;

    void CopySparseData(const uint64_t *srcSparseStart, const uint64_t *srcSparseEnd, const void *srcSparseData,
                        const uint32_t *srcSparseIndex) override;

    void LoadSparseData(const long *srcSparseStart, const long *srcSparseEnd, const void *srcSparseData,
                        const long *srcSparseIndex) override;

    void CopySparseData(const long *srcSparseStart, const long *srcSparseEnd, const void *srcSparseData,
                        const long *srcSparseIndex) override;

    void LoadIndexedData(const uint32_t *srcIndexedData) override;

    void LoadDataWeight(const NNFloat *srcWeightData) override;

    ~NNDataSet();
    void Shuffle();
    T GetDataPoint(uint32_t n, uint32_t x, uint32_t y = 0, uint32_t z = 0);
    bool SetDataPoint(T v, uint32_t n, uint32_t x, uint32_t y = 0, uint32_t z = 0);
    uint64_t GetSparseDataPoints(uint32_t n);
    uint32_t GetSparseIndex(uint32_t n, uint32_t i);
    bool SetSparseIndex(uint32_t n, uint32_t i, uint32_t v);
    T GetSparseDataPoint(uint32_t n, uint32_t i);
    bool SetSparseDataPoint(uint32_t n, uint32_t i, T v);
};

template<typename T> bool NNDataSet<T>::LoadInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Indexed)
        kLoadIndexedInputUnit(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData);
    else
        kLoadInputUnit(position, batch, stride, pUnit, _pbData->_pDevData);
    return true;
}

template<typename T> bool NNDataSet<T>::LoadSparseInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Boolean)
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kLoadIndexedSparseInputUnit(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight);
        else
            kLoadSparseInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kLoadIndexedSparseAnalogInputUnit(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData);
        else
            kLoadSparseAnalogInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::LoadSparseDenoisedInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Boolean)
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kLoadIndexedSparseDenoisedInputUnit(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbDenoisingRandom->_pDevData);
        else
            kLoadSparseDenoisedInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbDenoisingRandom->_pDevData);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kLoadIndexedSparseAnalogDenoisedInputUnit(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData);
        else
            kLoadSparseAnalogDenoisedInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Boolean)
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseZ(position, batch, stride, pWeight, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, pUnit, beta);
        else
            kCalculateSparseZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, pUnit, beta);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseAnalogZ(position, batch, stride, pWeight, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, pUnit, beta);
        else
            kCalculateSparseAnalogZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, pUnit, beta);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseDenoisedZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Boolean)
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseDenoisedZ(position, batch, stride, pWeight, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbDenoisingRandom->_pDevData, pUnit, beta);
        else
            kCalculateSparseDenoisedZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbDenoisingRandom->_pDevData, pUnit, beta);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseAnalogDenoisedZ(position, batch, stride, pWeight, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData, pUnit, beta);
        else
            kCalculateSparseAnalogDenoisedZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData, pUnit, beta);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseTransposedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer)
{
    // Rebuild sparse data table if dataset changed
    if (_bDirty || (batch != _batch))
    {
        GenerateSparseTransposedMatrix(batch, pLayer);
    }

    // Initialize transposed sparse offsets
    _pbSparseTransposedEnd->Copy(_pbSparseTransposedStart->_pDevData);

    // Call appropriate matrix generation kernel
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    NNFloat* pSparseTransposedData = ((_attributes & NNDataSetEnums::Weighted) || !(_attributes & NNDataSetEnums::Boolean)) ? _pbSparseTransposedData->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Boolean)
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseTransposedMatrix(position, batch, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
        else
            kCalculateSparseTransposedMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseTransposedAnalogMatrix(position, batch, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
        else
            kCalculateSparseTransposedAnalogMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
    }

    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseTransposedDenoisedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer)
{

    // Rebuild sparse data table if dataset changed
    if (_bDirty || (batch != _batch))
    {
        GenerateSparseTransposedMatrix(batch, pLayer);
    }

    // Initialize transposed sparse offsets
    _pbSparseTransposedEnd->Copy(_pbSparseTransposedStart->_pDevData);

    // Call appropriate matrix generation kernel
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    NNFloat* pSparseTransposedData = ((_attributes & NNDataSetEnums::Weighted) || !(_attributes & NNDataSetEnums::Boolean)) ? _pbSparseTransposedData->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Boolean)
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseTransposedDenoisedMatrix(position, batch, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbDenoisingRandom->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
        else
            kCalculateSparseTransposedDenoisedMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbDenoisingRandom->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseTransposedAnalogDenoisedMatrix(position, batch, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
        else
            kCalculateSparseTransposedAnalogDenoisedMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pSparseTransposedData);
    }

#if 0
    vector<uint32_t> vSparseTransposedStart(53120);
    vector<uint32_t> vSparseTransposedEnd(53120);
    _pbSparseTransposedStart->Download(&vSparseTransposedStart[0]);
    _pbSparseTransposedEnd->Download(&vSparseTransposedEnd[0]);
    for (uint32_t i = 0; i < 53120; i++)
    printf("%6u %9u %9u %9u %9u\n", i, vSparseTransposedStart[i], vSparseTransposedEnd[i], vSparseTransposedEnd[i] - vSparseTransposedStart[i], (uint32_t)_vSparseDatapointCount[i]);
    exit(-1);
#endif
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseTransposedWeightGradient(NNFloat alpha, NNFloat beta, uint32_t m, uint32_t n, NNFloat* pDelta, NNFloat* pWeightGradient)
{
    if ((_attributes & NNDataSetEnums::Boolean) && !(_attributes & NNDataSetEnums::Weighted))
        kCalculateSparseTransposedWeightGradient(alpha, beta, m, n, _pbSparseTransposedStart->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pDelta, pWeightGradient);
    else
        kCalculateSparseTransposedAnalogWeightGradient(alpha, beta, m, n, _pbSparseTransposedStart->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, _pbSparseTransposedData->_pDevData, pDelta, pWeightGradient);
    return true;
}

template<typename T> float NNDataSet<T>::CalculateL1Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
        {
           if (_attributes & NNDataSetEnums::Indexed)
           {
                return kCalculateIndexedSparseL1Error(position, batch, stride, pUnit,
                       _pbIndex->_pDevData,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                       pDataWeight,
                       bSparseIgnoreZero);
           }
           else
           {
                return kCalculateSparseL1Error(position, batch, stride, pUnit,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                       pDataWeight,
                       bSparseIgnoreZero);
           }
        }
        else
        {
           if (_attributes & NNDataSetEnums::Indexed)
           {
                return kCalculateIndexedSparseAnalogL1Error(position, batch, stride, pUnit,
                       _pbIndex->_pDevData,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                        pDataWeight,
                       _pbSparseData->_pDevData,
                       bSparseIgnoreZero);
           }
           else
           {
                return kCalculateSparseAnalogL1Error(position, batch, stride, pUnit,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                        pDataWeight,
                       _pbSparseData->_pDevData,
                       bSparseIgnoreZero);
           }
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            return kCalculateIndexedL1Error(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            return kCalculateL1Error(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
    }
}

template<typename T> float NNDataSet<T>::CalculateL2Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseL2Error(position, batch, stride, pUnit,
                       _pbIndex->_pDevData,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                       pDataWeight,
                       bSparseIgnoreZero);
            }
            else
            {
                return kCalculateSparseL2Error(position, batch, stride, pUnit,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                        pDataWeight,
                       bSparseIgnoreZero);
            }
        }
        else
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseAnalogL2Error(position, batch, stride, pUnit,
                           _pbIndex->_pDevData,
                           _pbSparseStart->_pDevData,
                           _pbSparseEnd->_pDevData,
                           _pbSparseIndex->_pDevData,
                           pDataWeight,
                           _pbSparseData->_pDevData,
                           bSparseIgnoreZero);
            }
            else
            {
                return kCalculateSparseAnalogL2Error(position, batch, stride, pUnit,
                           _pbSparseStart->_pDevData,
                           _pbSparseEnd->_pDevData,
                           _pbSparseIndex->_pDevData,
                           pDataWeight,
                           _pbSparseData->_pDevData,
                           bSparseIgnoreZero);
            }
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            return kCalculateIndexedL2Error(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            return kCalculateL2Error(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
    }
}

template<typename T> float NNDataSet<T>::CalculateL2HingeError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseL2HingeError(position, batch, stride, pUnit,
                       _pbIndex->_pDevData,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                       pDataWeight,
                       bSparseIgnoreZero);
            }
            else
            {
                return kCalculateSparseL2HingeError(position, batch, stride, pUnit,
                       _pbSparseStart->_pDevData,
                       _pbSparseEnd->_pDevData,
                       _pbSparseIndex->_pDevData,
                        pDataWeight,
                       bSparseIgnoreZero);
            }
        }
        else
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseAnalogL2HingeError(position, batch, stride, pUnit,
                           _pbIndex->_pDevData,
                           _pbSparseStart->_pDevData,
                           _pbSparseEnd->_pDevData,
                           _pbSparseIndex->_pDevData,
                           pDataWeight,
                           _pbSparseData->_pDevData,
                           bSparseIgnoreZero);
            }
            else
            {
                return kCalculateSparseAnalogL2HingeError(position, batch, stride, pUnit,
                           _pbSparseStart->_pDevData,
                           _pbSparseEnd->_pDevData,
                           _pbSparseIndex->_pDevData,
                           pDataWeight,
                           _pbSparseData->_pDevData,
                           bSparseIgnoreZero);
            }
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            return kCalculateIndexedL2HingeError(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            return kCalculateL2HingeError(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
    }
}

template<typename T> float NNDataSet<T>::CalculateCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Indexed)
        {
            return kCalculateIndexedSparseCrossEntropyError(position, batch, stride, pUnit,
                   _pbIndex->_pDevData,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData,
                   pDataWeight,
                   bSparseIgnoreZero);
        }
        else
        {
            return kCalculateSparseCrossEntropyError(position, batch, stride, pUnit,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData,
                   pDataWeight,
                   bSparseIgnoreZero);
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            return kCalculateIndexedCrossEntropyError(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            return kCalculateCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
    }
}

template<typename T> float NNDataSet<T>::CalculateScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Indexed)
        {
            return kCalculateIndexedSparseScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                   _pbIndex->_pDevData,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData,
                   pDataWeight,
                   bSparseIgnoreZero);
        }
    else
        {
            return kCalculateSparseScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                    _pbSparseStart->_pDevData,
                    _pbSparseEnd->_pDevData,
                    _pbSparseIndex->_pDevData,
                    pDataWeight,
                    bSparseIgnoreZero);
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            return kCalculateIndexedScaledMarginalCrossEntropyError(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            return kCalculateScaledMarginalCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
    }
}

template<typename T> float NNDataSet<T>::CalculateMultinomialCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        if (_attributes & NNDataSetEnums::Boolean)
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseMultinomialCrossEntropyError(position, batch, stride, pUnit,
                        _pbIndex->_pDevData,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight);
            }
            else
            {
                return kCalculateSparseMultinomialCrossEntropyError(position, batch, stride, pUnit,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight);
            }
        }
        else
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseAnalogMultinomialCrossEntropyError(position, batch, stride, pUnit,
                        _pbIndex->_pDevData,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight,
                        _pbSparseData->_pDevData);
            }
            else
            {
                return kCalculateSparseAnalogMultinomialCrossEntropyError(position, batch, stride, pUnit,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight,
                        _pbSparseData->_pDevData);
            }
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            return kCalculateIndexedMultinomialCrossEntropyError(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            return kCalculateMultinomialCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
    }
}

template<typename T> float NNDataSet<T>::CalculateMultinomialScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        if (_attributes & NNDataSetEnums::Boolean)
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                        _pbIndex->_pDevData,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight);
            }
            else
            {
                return kCalculateSparseMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight);
            }
        }
        else
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseAnalogMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                        _pbIndex->_pDevData,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight,
                        _pbSparseData->_pDevData);
            }
            else
            {
                return kCalculateSparseAnalogMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight,
                        _pbSparseData->_pDevData);
            }
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            return kCalculateIndexedMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            return kCalculateMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
    }
}

template<typename T> float NNDataSet<T>::CalculateDataScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        // Scale by 1 if data is Boolean
        if (_attributes & NNDataSetEnums::Boolean)
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                        _pbIndex->_pDevData,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight,
                        bSparseIgnoreZero);
            }
            else
            {
                return kCalculateSparseScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                        _pbSparseStart->_pDevData,
                        _pbSparseEnd->_pDevData,
                        _pbSparseIndex->_pDevData,
                        pDataWeight,
                        bSparseIgnoreZero);
            }
        }
        else
        {
            if (_attributes & NNDataSetEnums::Indexed)
            {
                return kCalculateIndexedSparseDataScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                                _pbIndex->_pDevData,
                                _pbSparseStart->_pDevData,
                                _pbSparseEnd->_pDevData,
                                _pbSparseIndex->_pDevData,
                                _pbSparseData->_pDevData, bSparseIgnoreZero);
            }
            else
            {
                return kCalculateSparseDataScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                                _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData,
                                _pbSparseData->_pDevData, bSparseIgnoreZero);
            }
        }
    }
    else
    {
        // BUG BUG BUG unacceptable behavior, this should be caught at startup
        cout << "unsupported data format of this cost function" << endl;
        getGpu().Shutdown();
        exit(-1);
    }
}

template<typename T> float NNDataSet<T>::CalculateHingeError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Indexed)
        return kCalculateIndexedHingeError(position, batch, stride, pUnit, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
    else
        return kCalculateHingeError(position, batch, stride, pUnit, _pbData->_pDevData, pDataWeight);
}

template<typename T> bool NNDataSet<T>::CalculateL1OutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseL1OutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero, slope, alpha, lambda);
        else
            kCalculateSparseL1OutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero, slope, alpha, lambda);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedL1OutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight, slope, alpha, lambda);
        else
            kCalculateL1OutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData, pDataWeight, slope, alpha, lambda);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero);
        else
            kCalculateSparseCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero);

    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            kCalculateCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData, pDataWeight);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedSparseScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero);
        else
            kCalculateSparseScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero);
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
        else
            kCalculateScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData, pDataWeight);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse) {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
        {
            if (_attributes & NNDataSetEnums::Indexed)
                kCalculateIndexedSparseOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero, slope, alpha, lambda);
            else
                kCalculateSparseOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero, slope, alpha, lambda);
        }
        else
        {
            if (_attributes & NNDataSetEnums::Indexed)
                kCalculateIndexedSparseAnalogOutputDelta(activation, position, batch, stride, pUnit,  pDelta, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, bSparseIgnoreZero, slope, alpha, lambda);
            else
                kCalculateSparseAnalogOutputDelta(activation, position, batch, stride, pUnit,  pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, bSparseIgnoreZero, slope, alpha, lambda);
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight, slope, alpha, lambda);
        else
            kCalculateOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData, pDataWeight, slope, alpha, lambda);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateL2HingeOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta, NNFloat slope, NNFloat alpha, NNFloat lambda)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Sparse) {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
        {
            if (_attributes & NNDataSetEnums::Indexed)
                kCalculateIndexedSparseL2HingeOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero, slope, alpha, lambda);
            else
                kCalculateSparseL2HingeOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, bSparseIgnoreZero, slope, alpha, lambda);
        }
        else
        {
            if (_attributes & NNDataSetEnums::Indexed)
                kCalculateIndexedSparseAnalogL2HingeOutputDelta(activation, position, batch, stride, pUnit,  pDelta, _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, bSparseIgnoreZero, slope, alpha, lambda);
            else
                kCalculateSparseAnalogL2HingeOutputDelta(activation, position, batch, stride, pUnit,  pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pDataWeight, _pbSparseData->_pDevData, bSparseIgnoreZero, slope, alpha, lambda);
        }
    }
    else
    {
        if (_attributes & NNDataSetEnums::Indexed)
            kCalculateIndexedL2HingeOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight, slope, alpha, lambda);
        else
            kCalculateL2HingeOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData, pDataWeight, slope, alpha, lambda);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateDataScaledMarginalCrossEntropyOutputDelta(Activation activation,
                uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Indexed)
        {
            kCalculateIndexedSparseDataScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta,
                             _pbIndex->_pDevData, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData,
                            _pbSparseData->_pDevData, bSparseIgnoreZero);
        }
        else
        {
            kCalculateSparseDataScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta,
                            _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData,
                            _pbSparseData->_pDevData, bSparseIgnoreZero);
        }
    }
    else
    {
        cout << "unsupported data format of this cost function" << endl;
        getGpu().Shutdown();
        exit(-1);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateHingeOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    NNFloat* pDataWeight = (_attributes & NNDataSetEnums::Weighted) ? _pbDataWeight->_pDevData : NULL;
    if (_attributes & NNDataSetEnums::Indexed)
        kCalculateIndexedHingeOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbIndex->_pDevData, _pbData->_pDevData, pDataWeight);
    else
        kCalculateHingeOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData, pDataWeight);
    return true;
}

vector<NNDataSetBase*> LoadNetCDF(const string& fname);
bool SaveNetCDF(const string& fname, vector<NNDataSetBase*> vDataset);
vector<NNDataSetBase*> LoadImageData(const string& fname);
vector<NNDataSetBase*> LoadCSVData(const string& fname);
vector<NNDataSetBase*> LoadJSONData(const string& fname);
vector<NNDataSetBase*> LoadAudioData(const string& name);

#endif