/*

cp ../../../../native_ref/<target binary> libnetlib.so

gcc-mp-4.8 -O3 ddottest.c common.c -o ddottest -L. -lnetlib -I../../../../netlib/CBLAS
./ddottest  > ../../../results/mac_os_x-x86_64-ddot-CBLAS.csv

gcc-mp-4.8 -O3 ddottest.c common.c -o ddottest -I/System/Library/Frameworks/vecLib.framework/Headers -framework veclib
./ddottest  > ../../../results/mac_os_x-x86_64-ddot-veclib.csv

gcc-mp-4.8 -O3 ddottest.c common.c -o ddottest -I/opt/local/include /opt/local/lib/libatlas.a /opt/local/lib/libcblas.a /opt/local/lib/liblapack.a /opt/local/lib/libf77blas.a -lgfortran
./ddottest  > ../../../results/mac_os_x-x86_64-ddot-atlas.csv

gcc-mp-4.8 -O3 ddottest.c common.c -o ddottest -I../../../../netlib/CBLAS -L/opt/intel/composerxe/mkl/lib -lmkl_rt
export DYLD_LIBRARY_PATH=/opt/intel/composerxe/mkl/lib:/opt/intel/composerxe/lib/
./ddottest  > ../../../results/mac_os_x-x86_64-ddot-mkl.csv

gcc-mp-4.8 -O3 ddottest.c cudawrapper.c common.c -o ddottest -I../../../../netlib/CBLAS -I/usr/local/cuda/include/ -L/usr/local/cuda/lib -lcublas
export DYLD_LIBRARY_PATH=/usr/local/cuda/lib
./ddottest  > ../../../results/mac_os_x-x86_64-ddot-cuda.csv

*/

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <cblas.h>
#include "common.h"

long benchmark(int size) {
	long requestStart, requestEnd;

    double* a = random_array(size);
    double* b = random_array(size);

	requestStart = currentTimeNanos();

    cblas_ddot(size, a, 1, b, 1);

	requestEnd = currentTimeNanos();

    free(a);
    free(b);

    return (requestEnd - requestStart);
  }

main()
{
	srand(time(NULL));

    double factor = 6.0 / 100.0;
    int i, j;
    for (i = 0 ; i < 10 ; i++) {
        for (j = 1 ; j <= 100 ; j++) {
            int size = (int) pow(10.0, factor * j);
            if (size < 10) continue;
            long took = benchmark(size);
            printf("\"%d\",\"%lu\"\n", size, took);
        }
    }
}


/*
 */

#ifndef _aXe_GRISM_H
#define _aXe_GRISM_H

#include <gsl/gsl_matrix.h>
#include <gsl/gsl_vector.h>
#include "spc_trace_functions.h"
//#include "aXe_errors.h"

#define RELEASE "See github"

#define gsl_matrix              gsl_matrix_float
#define gsl_matrix_get          gsl_matrix_float_get
#define gsl_matrix_set          gsl_matrix_float_set
#define gsl_matrix_set_all      gsl_matrix_float_set_all
#define gsl_matrix_alloc        gsl_matrix_float_alloc
#define gsl_matrix_free         gsl_matrix_float_free
#define gsl_matrix_fprintf      gsl_matrix_float_fprintf
#define gsl_matrix_add_constant gsl_matrix_float_add_constant
#define gsl_matrix_scale        gsl_matrix_float_scale
#define gsl_matrix_min          gsl_matrix_float_min
#define gsl_matrix_max          gsl_matrix_float_max

/* WARNING: If you ever change PIXEL_T, make sure you also change
   the FITS loader */
#define PIXEL_T float           /* type of pixel data -- this should be in
                                   sync with the GSL definitions */
#define MAX_BEAMS 27            /* Max. number of spectrum orders we handle */
                                // now equals the alphabet...

#define BEAM(x) (char)((int)x+65)       /* Defines names for 3 different beams */

#define MAXCHAR 255             /* Number of characters in small strings and FITS keywords */

/**
        A set of two relative (pixel) offsets.
*/
typedef struct
{
  int dx0, dx1;
}
px_offset;

/**
        A set of two relative double offsets.
*/
typedef struct
{
  double dx0, dx1;
}
dx_offset;

/**
  A point with integer (pixel) coordinates.
*/
typedef struct
{
  int x, y;
}
px_point;

/**
  A point with double coordinates.
*/
typedef struct
{
  double x, y;
}
d_point;

/**
  An aggregation of the grism exposure, its errors, the background,
  and flatfields
*/
typedef struct
{
  gsl_matrix *grism;            /* The grism image */
  gsl_matrix *pixerrs;  /* (Absolute) errors associated with the grism image */
  gsl_matrix *dq;        /* Data quality associated with grism image */
}
observation;

/**
   A beam, i.e., a description of the image of a spectrum of a given order,
   including a reference point, the corners of a convex quadrangle
   bounding the image, and the parametrization of the spectrum trace
*/
typedef struct
{
  int ID;                       /* An integer ID for this beam
                                   (see enum beamtype */
  d_point  refpoint;            /* Pixel coordinates of the reference point */
  px_point corners[4];          /* Bounding box of spectrum as an arbitrary
                                   quadrangle */
  px_point bbox[2];             /* Bounding box of corners, will be filled
                                   in by spc_extract */
  trace_func *spec_trace;       /* Parametrization of the spectrum trace */
  double width;                 /* largest axis of object in pixels */
  double orient;                /* orientation of largest axis in radian,
                                   0==horizontal */
  int modspec;
  int modimage;

  double awidth;                /* the object width along the extraction direction */
  double bwidth;                /* the object width perpendicular to ewidth
                                   (which is closer to the dispersion direction */
  double aorient;               /* orientation of awidth (CCW) =0 along the x-axis */

  double slitgeom[4];          /* containst the values for the slit length, angle and
                                   so on */

  gsl_vector *flux;              /* the flux of the object at various wavelengths */

  int ignore;           /* This beam should be ignored if this is not set to 0 */
}
beam;

/**
  An object, describing the width and orientation of the object,
  the beams in which the spectrum images lie, and the images
  comprising an observation of the object.
*/
typedef struct
{
     int ID;                    /* An integer ID number for this object */
     beam beams[MAX_BEAMS];     /* table of beams (orders of spectrum) */
     int nbeams;                /* number of beams in beams member */
     observation *grism_obs;
}
object;

/**
  An aperture pixel, i.e., a pixel within a beam.  The structure
  contains the coordinates, the abscissa of the section point between
  the object axis and the spectrum trace, the distance between
  the pixel and the section point along the object axis, the path
  length of the section point along the trace, and the photon count for
  this pixel.  A table of these is produced by the
  make_spc_table function.
*/
typedef struct
{
  int p_x, p_y;         /* the absolute coordinates of the
                                   source pixel. A -1 in p_x signifies
                                   the end of a list of ap_pixels */
  double x, y;          /* the logical coordinates of this (logical)
                                   pixel relative to the beam's reference
                                   point */
  double dist;          /* distance between point and section */
  double xs;                    /* abscissa of section point relative to reference point */
  double ys;                    /* y coord of section point relative to reference point */

  // traditionally (at least aXe-1.3-1.6 "dxs" was called "Projected size of pixel along the trace".
  // however in the code (spc_extract.c, function "handle_one_pixel()" it becomes clear
  // that this quantity indeed is the local trace angle
  double dxs;

  double xi;                    /* path length of the section along the
                                   spectrum trace (computed from xs) */
  double lambda;                /* path length of the section along the
                                   spectrum trace (computed from xi) */
  double dlambda;               /* dpath length of the section along the
                                   spectrum trace */
  double count;         /* intensity for this pixel */
  double weight;     /* extraction weight for this pixel */
  double error;         /* absolute error of count */
  double contam;     /* flag whether this pixel is contaminated by another spectrum */
  double model;     /* the pixel flux computed for one of the emission models */
  long dq;           /* DQ bit information */
}
ap_pixel;

typedef struct
{
  ap_pixel *ap;                 /* A pointer to an ap_pixel array */
  object *obj;                  /* A pointer to the corresponding object
                                   structure */
  beam *b;                      /* A pointer to the beam in the object
                                   that this group of pixel correspond to */
}
PET_entry;

/* See comment to spectrum structure */
typedef enum
{
     SPC_W_CONTAM,
     SPC_W_BORDER
}
spec_warning;

/**
  A point within a spectrum, consisting of the minimum, maximum, and
  (weighted) mean lambda of the points that contributed to the
  bin, as well as an error estimate for the value (that's in there,
  too, of course)
*/
typedef struct
{
  double lambda_min, lambda_mean;
  double lambda_max, dlambda;/* wavelength information */
  double error;         /* error in counts in this bin */
  double count;         /* number of counts in this bin */
  double weight;                /* weight of this bin */
  double flux;          /* flux in physical units in this bin */
  double ferror;                /* error in physical units in this bin */
  double contam;     /* spectral contamination flag */
  //  int    contam;     /* spectral contamination flag */
  long   dq;        /* DQ information */
}
spc_entry;

/**
  A spectrum, consisting of a gsl_array containing the data, a start
  value (center wave length of lowest bin), a bin size, and warn flags,
  which may be

SPC_W_CONTAM -- Spectrum is contaminated

SPC_W_BORDER -- Spectrum lies partially beyond the image borders.

*/
typedef struct
{
  spc_entry *spec;      /* the actual spectrum */
  double lambdamin;
  double lambdamax;
  int spec_len;         /* Number of items in spec */
  spec_warning warning; /* problems with this spectrum */
}
spectrum;

#define DEBUG_FILE 0x01
#define DEBUG 0x0000

extern void
print_ap_pixel_table (const ap_pixel * ap_p);

extern ap_pixel *
make_spc_table (object * const ob,
                const int beamorder, int *const flags);

extern ap_pixel *
make_gps_table (object * const ob,
                const int beamorder, int *const flags,
                int xval, int yval);

#endif /* !_aXe_GRISM_H */


#pragma once

#include <gsl\gsl>
#include <winrt\Windows.Foundation.h>
#include <d3d11.h>
#include "DrawableGameComponent.h"
#include "MatrixHelper.h"

namespace Rendering
{
	class AmbientLightingMaterial;

	class AmbientLightingDemo final : public Library::DrawableGameComponent
	{
	public:
		AmbientLightingDemo(Library::Game& game, const std::shared_ptr<Library::Camera>& camera);

		bool AnimationEnabled() const;
		void SetAnimationEnabled(bool enabled);

		float AmbientLightIntensity() const;
		void SetAmbientLightIntensity(float intensity);

		virtual void Initialize() override;
		virtual void Update(const Library::GameTime& gameTime) override;
		virtual void Draw(const Library::GameTime& gameTime) override;

	private:
		inline static const float RotationRate{ DirectX::XM_PI };

		std::shared_ptr<AmbientLightingMaterial> mMaterial;
		DirectX::XMFLOAT4X4 mWorldMatrix{ Library::MatrixHelper::Identity };
		winrt::com_ptr<ID3D11Buffer> mVertexBuffer;
		winrt::com_ptr<ID3D11Buffer> mIndexBuffer;
		std::uint32_t mIndexCount{ 0 };
		float mModelRotationAngle{ 0.0f };
		bool mAnimationEnabled{ true };
		bool mUpdateMaterial{ true };
	};
}


/*

   Copyright 2016  Amazon.com, Inc. or its affiliates. All Rights Reserved.

   Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at

   http://aws.amazon.com/apache2.0/

   or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 */

#ifndef NNTYPES_H
#include <vector>
#include <set>

#include <string>
#include <iostream>
#include <iomanip>
#include <fstream>
#include <algorithm>
#include <netcdf>
#ifndef __NVCC__
#include <tuple>
#include <json/json.h>
#endif
#include <sys/time.h>
#include <cmath>

class NNDataSetBase;
class NNLayer;
class NNNetwork;
class NNWeight;

// Activates step by step CPU validation
#define VALIDATION
#ifdef VALIDATION
extern "C"
{
    #include <cblas.h>
}
#endif

static const float NN_VERSION       = 0.81f;
static const float MIN_ERROR        = 1.0e-12f;
static const float MIN_ACTIVATION   = 0.000001f;
static const float MAX_ACTIVATION   = 0.999999f;
static const float MAX_VALUE        = 999999999999999.0f;

template <typename T> struct GpuBuffer;

enum
{
    DefaultBatch    = 512
};

enum TrainingMode
{
    SGD = 0,
    Momentum = 1,
    AdaGrad = 2,
    Nesterov = 3,
    RMSProp = 4,
    AdaDelta = 5,
};

ostream& operator<< (ostream& out, const TrainingMode& e);

enum ErrorFunction
{
    L1,
    L2,
    CrossEntropy,
    ScaledMarginalCrossEntropy,
};

ostream& operator<< (ostream& out, const ErrorFunction& e);

enum Activation {
    Sigmoid,
    Tanh,
    RectifiedLinear,
    Linear,
    ParametricRectifiedLinear,
    SoftPlus,
    SoftSign,
    SoftMax,
    ReluMax,
    LinearMax,
};

ostream& operator<< (ostream& out, const Activation& a);

enum WeightInitialization
{
    Xavier,
    CaffeXavier,
    Gaussian,
    Uniform,
    UnitBall,
    Constant
};

ostream& operator<< (ostream& out, const WeightInitialization& w);

enum PoolingFunction {
    None,
    Max,
    Average,
    Stochastic,
    LocalContrastNormalization,
    LocalResponseNormalization,
    GlobalTemporal,
};

ostream& operator<< (ostream& out, const PoolingFunction& p);

#include "kernels.h"
#include "GpuSort.h"
#include "NNEnum.h"
#include "NNWeight.h"
#include "NNLayer.h"
#include "NNNetwork.h"

int MPI_Bcast_string(string& s);

struct NNDataSetDimensions
{
    uint32_t _dimensions;
    uint32_t _width;
    uint32_t _height;
    uint32_t _length;
};

struct NNDataSetBase {

    string                   _name;                          // Dataset name
    NNDataSetEnums::DataType _dataType;                      // Dataset type (see above enum)
    uint32_t                 _attributes;                    // Dataset characteristics (see NNDataSetEnum::Attributes in NNEnum.h)
    uint32_t                 _examples;                      // Number of examples
    uint32_t                 _dimensions;                    // Dimensionality of data set
    uint32_t                 _width;                         // Dataset x dimension
    uint32_t                 _height;                        // Dataset y dimension
    uint32_t                 _length;                        // Dataset z dimension
    uint32_t                 _stride;                        // Stride between examples
    NNDataSetEnums::Sharding _sharding;                      // Sharding of dataset for parallel execution
    uint32_t                 _minX;                          // Beginning of local X sharding for model parallel execution
    uint32_t                 _maxX;                          // End of local X sharding for model parallel execution
    uint64_t                 _sparseDataSize;                // Total sparse datapoints
    uint32_t                 _maxSparseDatapoints;           // Maximum observed sparse datapoints per example
    NNFloat                  _sparseDensity;                 // Overall sparse density (0.0 - 1.0)
    vector<uint64_t>         _vSparseStart;                  // Vector of sparse datapoint starts per example
    GpuBuffer<uint64_t>*     _pbSparseStart;                 // GPU copy of _vSparseStart
    vector<uint64_t>         _vSparseEnd;                    // Vector of sparse datapoint ends per example
    GpuBuffer<uint64_t>*     _pbSparseEnd;                   // GPU copy of _vSparseEnd
    vector<uint32_t>         _vSparseIndex;                  // Vector of sparse indices
    GpuBuffer<uint32_t>*     _pbSparseIndex;                 // GPU copy of _vSparseIndex
    GpuBuffer<NNFloat>*      _pbDenoisingRandom;             // Denoising randoms

    // Transposed sparse lookup for sparse backpropagation
    vector<uint64_t>         _vSparseDatapointCount;
    vector<uint32_t>         _vSparseTransposedStart;
    uint32_t                 _sparseTransposedIndices;
    GpuBuffer<uint32_t>*     _pbSparseTransposedStart;
    GpuBuffer<uint32_t>*     _pbSparseTransposedEnd;
    GpuBuffer<uint32_t>*     _pbSparseTransposedIndex;

    // States
    bool                     _bDenoising;
    bool                     _bDirty;
    uint32_t                 _batch;

    NNDataSetBase();
    NNDataSetDimensions GetDimensions();
    uint32_t GetExamples() { return _examples; };

    virtual bool SaveNetCDF(const string& fname) = 0;
    virtual bool WriteNetCDF(netCDF::NcFile& nfc, const string& fname, const uint32_t n) = 0;
    virtual ~NNDataSetBase() = 0;
    virtual void RefreshState(uint32_t batch) = 0;
    virtual bool Shard(NNDataSetEnums::Sharding sharding) = 0;
    virtual bool UnShard() = 0;
    virtual vector<tuple<uint64_t, uint64_t> > getMemoryUsage() = 0;
    virtual bool CalculateSparseDatapointCounts() = 0;
    virtual bool GenerateSparseTransposedMatrix(uint32_t batch, NNLayer* pLayer) = 0;
    virtual bool CalculateSparseTransposedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer) = 0;
    virtual bool CalculateSparseTransposedDenoisedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer) = 0;
    virtual bool CalculateSparseTransposedWeightGradient(NNFloat alpha, NNFloat beta, uint32_t m, uint32_t n, NNFloat* pDelta, NNFloat* pWeightGradient) = 0;
    virtual bool SetDenoising(bool flag) = 0;
    virtual bool GenerateDenoisingData() = 0;
    virtual bool LoadInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool LoadSparseInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool LoadSparseDenoisedInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool CalculateSparseZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta = (NNFloat)0.0) = 0;
    virtual bool CalculateSparseDenoisedZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta = (NNFloat)0.0) = 0;
    virtual float CalculateL1Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateL2Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateMultinomialCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual float CalculateMultinomialScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit) = 0;
    virtual bool CalculateL1OutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;
    virtual bool CalculateCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;
    virtual bool CalculateScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;
    virtual bool CalculateOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta) = 0;
};

ostream& operator<< (ostream& out, NNDataSetEnums::Attributes& a);
ostream& operator<< (ostream& out, NNDataSetEnums::Kind& k);
ostream& operator<< (ostream& out, NNDataSetEnums::DataType& t);
ostream& operator<< (ostream& out, NNDataSetEnums::Sharding& s);

template<typename T> class NNDataSet : public NNDataSetBase {
public:
    friend class NNetwork;
    friend class NNLayer;
    friend vector<NNDataSetBase*> LoadNetCDF(const string& fname);
    friend bool SaveNetCDF(const string& fname, vector<NNDataSetBase*> vDataSet);

private:

    vector<T>               _vData;
    GpuBuffer<T>*           _pbData;
    vector<T>               _vSparseData;
    GpuBuffer<T>*           _pbSparseData;
    GpuBuffer<T>*           _pbSparseTransposedData;

    // Force constructor private
    NNDataSet(const string& fname, uint32_t n);
    bool Rename(const string& name);
    bool SaveNetCDF(const string& fname);
    bool WriteNetCDF(netCDF::NcFile& nfc, const string& fname, const uint32_t n);
    void RefreshState(uint32_t batch) {}
    bool Shard(NNDataSetEnums::Sharding sharding);
    bool UnShard();
    vector<tuple<uint64_t, uint64_t> > getMemoryUsage();
    bool CalculateSparseDatapointCounts();
    bool GenerateSparseTransposedMatrix(uint32_t batch, NNLayer* pLayer);
    bool CalculateSparseTransposedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer);
    bool CalculateSparseTransposedDenoisedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer);
    bool CalculateSparseTransposedWeightGradient(NNFloat alpha, NNFloat beta, uint32_t m, uint32_t n, NNFloat* pDelta, NNFloat* pWeightGradient);
    bool SetDenoising(bool flag);
    bool GenerateDenoisingData();
    bool LoadInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool LoadSparseInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool LoadSparseDenoisedInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool CalculateSparseZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta);
    bool CalculateSparseDenoisedZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta);
    float CalculateL1Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateL2Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateMultinomialCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    float CalculateMultinomialScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit);
    bool CalculateL1OutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);
    bool CalculateCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);
    bool CalculateScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);
    bool CalculateOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta);

public:

    ~NNDataSet();
    void Shuffle();
    T GetDataPoint(uint32_t n, uint32_t x, uint32_t y = 0, uint32_t z = 0);
    bool SetDataPoint(T v, uint32_t n, uint32_t x, uint32_t y = 0, uint32_t z = 0);
    uint32_t GetSparseDataPoints(uint32_t n);
    uint32_t GetSparseIndex(uint32_t n, uint32_t i);
    bool SetSparseIndex(uint32_t n, uint32_t i, uint32_t v);
    T GetSparseDataPoint(uint32_t n, uint32_t i);
    bool SetSparseDataPoint(uint32_t n, uint32_t i, T v);
};

template<typename T> bool NNDataSet<T>::LoadInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    kLoadInputUnit(position, batch, stride, pUnit, _pbData->_pDevData);
    return true;
}

template<typename T> bool NNDataSet<T>::LoadSparseInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Boolean)
        kLoadSparseInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData);
    else
        kLoadSparseAnalogInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseData->_pDevData);
    return true;
}

template<typename T> bool NNDataSet<T>::LoadSparseDenoisedInputUnit(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Boolean)
        kLoadSparseDenoisedInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbDenoisingRandom->_pDevData);
    else
        kLoadSparseAnalogDenoisedInputUnit(position, batch, stride, pUnit, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData);
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta)
{
    if (_attributes & NNDataSetEnums::Boolean)
        kCalculateSparseZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, pUnit, beta);
    else
        kCalculateSparseAnalogZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseData->_pDevData, pUnit, beta);
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseDenoisedZ(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pWeight, NNFloat* pUnit, NNFloat beta)
{
    if (_attributes & NNDataSetEnums::Boolean)
        kCalculateSparseDenoisedZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbDenoisingRandom->_pDevData, pUnit, beta);
    else
        kCalculateSparseAnalogDenoisedZ(position, batch, stride, pWeight, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData, pUnit, beta);
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseTransposedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer)
{
    // Rebuild sparse data table if dataset changed
    if (_bDirty || (batch != _batch))
    {
        GenerateSparseTransposedMatrix(batch, pLayer);
    }

    // Initialize transposed sparse offsets
    _pbSparseTransposedEnd->Copy(_pbSparseTransposedStart->_pDevData);

    // Call appropriate matrix generation kernel
    if (_attributes & NNDataSetEnums::Boolean)
        kCalculateSparseTransposedMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData);
    else
        kCalculateSparseTransposedAnalogMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseData->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, _pbSparseTransposedData->_pDevData);

    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseTransposedDenoisedMatrix(uint32_t position, uint32_t batch, NNLayer* pLayer)
{

    // Rebuild sparse data table if dataset changed
    if (_bDirty || (batch != _batch))
    {
        GenerateSparseTransposedMatrix(batch, pLayer);
    }

    // Initialize transposed sparse offsets
    _pbSparseTransposedEnd->Copy(_pbSparseTransposedStart->_pDevData);

    // Call appropriate matrix generation kernel
    if (_attributes & NNDataSetEnums::Boolean)
        kCalculateSparseTransposedDenoisedMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbDenoisingRandom->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData);
    else
        kCalculateSparseTransposedAnalogDenoisedMatrix(position, batch, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseData->_pDevData, _pbDenoisingRandom->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, _pbSparseTransposedData->_pDevData);

#if 0
    vector<uint32_t> vSparseTransposedStart(53120);
    vector<uint32_t> vSparseTransposedEnd(53120);
    _pbSparseTransposedStart->Download(&vSparseTransposedStart[0]);
    _pbSparseTransposedEnd->Download(&vSparseTransposedEnd[0]);
    for (uint32_t i = 0; i < 53120; i++)
    printf("%6u %9u %9u %9u %9u\n", i, vSparseTransposedStart[i], vSparseTransposedEnd[i], vSparseTransposedEnd[i] - vSparseTransposedStart[i], (uint32_t)_vSparseDatapointCount[i]);
    exit(-1);
#endif
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateSparseTransposedWeightGradient(NNFloat alpha, NNFloat beta, uint32_t m, uint32_t n, NNFloat* pDelta, NNFloat* pWeightGradient)
{
    if (_attributes & NNDataSetEnums::Boolean)
        kCalculateSparseTransposedWeightGradient(alpha, beta, m, n, _pbSparseTransposedStart->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, pDelta, pWeightGradient);
    else
        kCalculateSparseTransposedAnalogWeightGradient(alpha, beta, m, n, _pbSparseTransposedStart->_pDevData, _pbSparseTransposedEnd->_pDevData, _pbSparseTransposedIndex->_pDevData, _pbSparseTransposedData->_pDevData, pDelta, pWeightGradient);
    return true;
}

template<typename T> float NNDataSet<T>::CalculateL1Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
           return kCalculateSparseL1Error(position, batch, stride, pUnit,
                  _pbSparseStart->_pDevData,
                  _pbSparseEnd->_pDevData,
                  _pbSparseIndex->_pDevData,
                  bSparseIgnoreZero);
        else
           return kCalculateSparseAnalogL1Error(position, batch, stride, pUnit,
                  _pbSparseStart->_pDevData,
                  _pbSparseEnd->_pDevData,
                  _pbSparseIndex->_pDevData,
                  _pbSparseData->_pDevData,
                  bSparseIgnoreZero);
    }
    else
        return kCalculateL1Error(position, batch, stride, pUnit, _pbData->_pDevData);
}

template<typename T> float NNDataSet<T>::CalculateL2Error(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
            return kCalculateSparseL2Error(position, batch, stride, pUnit,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData,
                   bSparseIgnoreZero);
        else
            return kCalculateSparseAnalogL2Error(position, batch, stride, pUnit,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData,
                   _pbSparseData->_pDevData,
                   bSparseIgnoreZero);
    }
    else
        return kCalculateL2Error(position, batch, stride, pUnit, _pbData->_pDevData);
}

template<typename T> float NNDataSet<T>::CalculateCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        return kCalculateSparseCrossEntropyError(position, batch, stride, pUnit,
               _pbSparseStart->_pDevData,
               _pbSparseEnd->_pDevData,
               _pbSparseIndex->_pDevData,
               bSparseIgnoreZero);
    }
    else
        return kCalculateCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData);
}

template<typename T> float NNDataSet<T>::CalculateScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        return kCalculateSparseScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
               _pbSparseStart->_pDevData,
               _pbSparseEnd->_pDevData,
               _pbSparseIndex->_pDevData,
               bSparseIgnoreZero);
    }
    else
        return kCalculateScaledMarginalCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData);
}

template<typename T> float NNDataSet<T>::CalculateMultinomialCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        if (_attributes & NNDataSetEnums::Boolean)
        {
            return kCalculateSparseMultinomialCrossEntropyError(position, batch, stride, pUnit,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData);
        }
        else
            return kCalculateSparseAnalogMultinomialCrossEntropyError(position, batch, stride, pUnit,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData,
                   _pbSparseData->_pDevData);
    }
    else
        return kCalculateMultinomialCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData);
}

template<typename T> float NNDataSet<T>::CalculateMultinomialScaledMarginalCrossEntropyError(uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        if (_attributes & NNDataSetEnums::Boolean)
            return kCalculateSparseMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData);
        else
            return kCalculateSparseAnalogMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit,
                   _pbSparseStart->_pDevData,
                   _pbSparseEnd->_pDevData,
                   _pbSparseIndex->_pDevData,
                   _pbSparseData->_pDevData);
    }
    else
        return kCalculateMultinomialScaledMarginalCrossEntropyError(position, batch, stride, pUnit, _pbData->_pDevData);
}

template<typename T> bool NNDataSet<T>::CalculateL1OutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        kCalculateSparseL1OutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, bSparseIgnoreZero);
    }
    else
        kCalculateL1OutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData);
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        kCalculateSparseCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, bSparseIgnoreZero);
    }
    else
        kCalculateCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData);
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateScaledMarginalCrossEntropyOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    if (_attributes & NNDataSetEnums::Sparse)
    {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        kCalculateSparseScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, bSparseIgnoreZero);
    }
    else
    {
        kCalculateScaledMarginalCrossEntropyOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData);
    }
    return true;
}

template<typename T> bool NNDataSet<T>::CalculateOutputDelta(Activation activation, uint32_t position, uint32_t batch, uint32_t stride, NNFloat* pUnit, NNFloat* pDelta)
{
    if (_attributes & NNDataSetEnums::Sparse) {
        bool bSparseIgnoreZero = _attributes & NNDataSetEnums::SparseIgnoreZero;
        if (_attributes & NNDataSetEnums::Boolean)
        {
            kCalculateSparseOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, bSparseIgnoreZero);
        }
        else
        {
            kCalculateSparseAnalogOutputDelta(activation, position, batch, stride, pUnit,  pDelta, _pbSparseStart->_pDevData, _pbSparseEnd->_pDevData, _pbSparseIndex->_pDevData, _pbSparseData->_pDevData, bSparseIgnoreZero);
        }
    }
    else
    {
        kCalculateOutputDelta(activation, position, batch, stride, pUnit, pDelta, _pbData->_pDevData);
    }
    return true;
}

vector<NNDataSetBase*> LoadNetCDF(const string& fname);
bool SaveNetCDF(const string& fname, vector<NNDataSetBase*> vDataset);
vector<NNDataSetBase*> LoadImageData(const string& fname);
vector<NNDataSetBase*> LoadCSVData(const string& fname);
vector<NNDataSetBase*> LoadJSONData(const string& fname);
vector<NNDataSetBase*> LoadAudioData(const string& name);

#define NNTYPES_H
#endif


#include <gsl/gsl_sf_erf.h>
#include <gsl/gsl_vector.h>
#include <math.h>
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
//#include <gsl/gsl_ran_gaussian.h>

int main (int argc, char *argv[]){

    double a = gsl_sf_erf(1);
    printf("The random number we get is %f.", a);
    exit(0);
}


// matrix/snowboy-blas.h

// Copyright 2016  KITT.AI (author: Guoguo Chen)

#ifndef SNOWBOY_MATRIX_SNOWBOY_BLAS_H_
#define SNOWBOY_MATRIX_SNOWBOY_BLAS_H_

#ifdef HAVE_ATLAS
  extern "C" {
    #include <cblas.h>
    #include <clapack.h>
  }
#elif defined(HAVE_CLAPACK)
  #ifdef __APPLE__
    #ifndef __has_extension
    #define __has_extension(x) 0
    #endif
    #define vImage_Utilities_h
    #define vImage_CVUtilities_h
    #include <Accelerate/Accelerate.h>
    typedef __CLPK_integer          integer;
    typedef __CLPK_logical          logical;
//    typedef __CLPK_real             real;
    typedef __CLPK_doublereal       doublereal;
    typedef __CLPK_complex          complex;
    typedef __CLPK_doublecomplex    doublecomplex;
    typedef __CLPK_ftnlen           ftnlen;
  #else
    extern "C" {
      #include <cblas.h>
      #include <f2c.h>
      #include <clapack.h>

      // Removes dangerous macros from f2c.h.
      #undef abs
      #undef dabs
      #undef min
      #undef max
      #undef dmin
      #undef dmax
      #undef bit_test
      #undef bit_clear
      #undef bit_set
    }
  #endif
  // Defines SnowboyBlasInt type that will be used in SVD.
  typedef integer SnowboyBlasInt;
#elif defined(HAVE_OPENBLAS)
  #include "cblas.h"
#ifndef NO_FORTRAN
  #include "lapacke.h"
#endif
  #undef I
  #undef complex
  // get rid of macros from f2c.h -- these are dangerous.
  #undef abs
  #undef dabs
  #undef min
  #undef max
  #undef dmin
  #undef dmax
  #undef bit_test
  #undef bit_clear
  #undef bit_set
#else
  #error "You have to define HAVE_CLAPACK or HAVE_ATLAS or HAVE_OPENBLAS."
#endif

#endif  // SNOWBOY_MATRIX_SNOWBOY_BLAS_H_


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <time.h>
#include <assert.h>
#include <gsl/gsl_math.h>
#include <gsl/gsl_integration.h>
#include <stdbool.h>

#include "core_allvars.h"
#include "core_proto.h"

#define IMF_CONSTANT IMF_norm/(IMF_slope + 1.0)

// Local Proto-Types //

void  update_SN_stars_array(int p, double stars, double dt, int tree, int ngal);
void update_stellar_tracking(int p, double stars, double dt, int tree, int ngal);

// External Functions //

void update_from_SN_feedback(int p, int centralgal, double reheated_mass, double ejected_mass, double mass_stars_recycled, double mass_metals_new, double NSN, double dt)
{

  double metallicity, metallicityHot, dust_fraction_cold, dust_fraction_hot;
  double dust_fraction_coldhot, dust_fraction_hotcold; // These are the dust fractions defined by M_colddust / (Mcold + Mhot) and M_hotdust / (Mcold + Mhot).
  // These second dust fractions are required because when we annihilate dust from SN we want the TOTAL amount to be NSN * eta * Ms.

  // Dust constants, taken mainly from Dayal, Ferrara and Saro 2011.
  double yd = 0.5 / 1.0e10 * Hubble_h; // Amount of dust mass created from each supernova event. Units of Msun.
  double eta = 0.4; // Coupling between shock energy and annihilation of dust.
  double Ms = 6.8e3 / 1.0e10 * Hubble_h; // Amount of mass that is accelerated to 100km/s by SN blastwave. Units of Msun.
  double dust_added = 0.0, dust_removed = 0.0;

  XASSERT((reheated_mass >= 0.0) && (ejected_mass >= 0.0) && (mass_stars_recycled >= 0.0) && (mass_metals_new >= 0.0), "When trying to update from the supernova feedback we got negative masses.\nReheated Mass = %.4e (1.0e10 Msun/h) \t Ejected Mass = %.4e (1.0e10 Msun/h) \t Mass of Stars Recycled = %.4e (1.0e10 Msun/h) \t Mass of Metals Added = %.4e (1.0e10 Msun/h)\n", reheated_mass, ejected_mass, mass_stars_recycled, mass_metals_new);

  Gal[p].StellarMass -= mass_stars_recycled; // The supernova remnants are instantly moved back into the cold ISM.
  Gal[p].Total_Stellar_Stars -= mass_stars_recycled;

  if (Gal[p].Total_Stellar_Stars < 0.0)
    Gal[p].Total_Stellar_Stars = 0.0;

  Gal[p].ColdGas += mass_stars_recycled;

  Gal[p].DustColdGas += yd * NSN; // Dust is created by supernova (something) on cold gas.
  Gal[p].ColdGas -= yd * NSN; // Hence the created dust is subtracted from the cold gas reservoir.

  dust_added += yd * NSN;

  if(Gal[p].ColdGas > 1.0e-10)
      Gal[p].MetalsColdGas += mass_metals_new; // ISM is enriched by new metals.
  else
      Gal[centralgal].MetalsHotGas += mass_metals_new;

  // Here we do checks to see if we need to rescale the amount of mass that has been ejected/reheated. //

  if(reheated_mass > Gal[p].ColdGas) // Just perform this check again to be sure.
      reheated_mass = Gal[p].ColdGas; // Because SF takes gas out of cold resevoir.

  if(ejected_mass > Gal[centralgal].HotGas)
      ejected_mass = Gal[centralgal].HotGas;

  metallicity = get_metallicity(Gal[p].ColdGas, Gal[p].MetalsColdGas);
  dust_fraction_cold = get_dust_fraction(Gal[p].ColdGas, Gal[p].DustColdGas);
  dust_fraction_coldhot = get_dust_fraction(Gal[p].ColdGas + Gal[p].HotGas + Gal[p].EjectedMass, Gal[p].DustColdGas);

  Gal[p].ColdGas -= reheated_mass;
  Gal[p].MetalsColdGas -= reheated_mass * metallicity;
  Gal[p].DustColdGas -= reheated_mass * dust_fraction_cold; // Dust that has been reheated to the hot reservoir.
  Gal[p].DustColdGas -= NSN * eta * Ms * dust_fraction_coldhot; // Dust that has been annihilated by shocks.

  dust_removed += NSN * eta * Ms * dust_fraction_coldhot;

  Gal[centralgal].HotGas += reheated_mass;
  Gal[centralgal].MetalsHotGas += reheated_mass * metallicity;
  Gal[centralgal].DustHotGas += reheated_mass * dust_fraction_cold;

  metallicityHot = get_metallicity(Gal[centralgal].HotGas, Gal[centralgal].MetalsHotGas);
  dust_fraction_hot = get_dust_fraction(Gal[p].HotGas, Gal[p].DustHotGas);
  dust_fraction_hotcold = get_dust_fraction(Gal[p].ColdGas + Gal[p].HotGas + Gal[p].EjectedMass, Gal[p].DustHotGas);
  // When dust_fraction_hotcold is calculated is irrelevant (i.e., before or after mass is added/subtracted from ColdGas).
  // This is because mass is conserved between ColdGas and HotGas during this step.

  Gal[centralgal].HotGas -= ejected_mass;
  Gal[centralgal].MetalsHotGas -= ejected_mass * metallicityHot;
  Gal[centralgal].DustHotGas -= ejected_mass * dust_fraction_hot; // Dust that has been ejected from the galaxy.
  Gal[centralgal].DustHotGas -= NSN * eta * Ms * dust_fraction_hotcold; // Dust that has been annihilated by shocks.
  dust_removed += NSN * eta * Ms * dust_fraction_hotcold;

  Gal[centralgal].EjectedMass += ejected_mass;
  Gal[centralgal].MetalsEjectedMass += ejected_mass * metallicityHot;
  Gal[centralgal].DustEjectedMass += ejected_mass * dust_fraction_hot;

  Gal[centralgal].EjectedMassSN += ejected_mass;

  Gal[p].GridOutflowRate[Halo[Gal[centralgal].HaloNr].SnapNum] += ejected_mass / dt;

  Gal[p].GrandSum -= mass_stars_recycled;

  if(Gal[p].ColdGas < 0.0) // Some final checks.
    Gal[p].ColdGas = 0.0;
  if(Gal[p].MetalsColdGas < 0.0)
    Gal[p].MetalsColdGas = 0.0;
  if(Gal[p].StellarMass < 0.0)
    Gal[p].StellarMass = 0.0;
  if(Gal[centralgal].HotGas < 0.0)
    Gal[centralgal].HotGas = 0.0;
  if(Gal[p].MetalsHotGas < 0.0)
    Gal[p].MetalsHotGas = 0.0;
  if(Gal[p].GrandSum < 0.0)
    Gal[p].GrandSum = 0.0;

  if (Gal[p].DustColdGas < 0.0)
    Gal[p].DustColdGas = 0.0;
  if (Gal[p].DustHotGas < 0.0)
    Gal[p].DustHotGas = 0.0;
}

void update_from_star_formation(int p, double stars, double dt, int step, bool ismerger, int tree, int ngal)
{

  double metallicity = get_metallicity(Gal[p].ColdGas, Gal[p].MetalsColdGas);
  double dust_fraction_cold = get_dust_fraction(Gal[p].ColdGas, Gal[p].DustColdGas);
  double time_spanned;

  // If the SF episode was from a merger the stars are placed in different reservoirs.
  if(!ismerger)
  {
    Gal[p].SfrDisk[step] += stars / dt;
    Gal[p].SfrDiskColdGas[step] = Gal[p].ColdGas;
    Gal[p].SfrDiskColdGasMetals[step] = Gal[p].MetalsColdGas;
  } else
  {

    Gal[p].SfrBulge[step] += stars / dt;
    Gal[p].SfrBulgeColdGas[step] += Gal[p].ColdGas;
    Gal[p].SfrBulgeColdGasMetals[step] += Gal[p].MetalsColdGas;

    Gal[p].BulgeMass += stars;
    Gal[p].MetalsBulgeMass += stars;

  }

  // If we're doing delayed SN we need to update the tracking
  if (ismerger == true)
  {
    time_spanned = 0.0;
  }
  else
  {
    time_spanned = dt;
  }

  //time_spanned = dt;

  if (IRA == 0)
  {
    update_SN_stars_array(p, stars, time_spanned, tree, ngal);
  }

  if (PhotonPrescription == 1)
  {
    update_stellar_tracking(p, stars, dt, tree, ngal);
  }

  // update gas and metals from star formation
  Gal[p].ColdGas -= stars;
  Gal[p].MetalsColdGas -= metallicity * stars;
  Gal[p].StellarMass += stars;
  Gal[p].MetalsStellarMass += metallicity * stars;
  Gal[p].DustColdGas -= dust_fraction_cold * stars; // When stars form they eat up dust.
  if (Gal[p].ColdGas < 0.0)
    Gal[p].ColdGas = 0.0;
  if (Gal[p].MetalsColdGas < 0.0)
    Gal[p].MetalsColdGas = 0.0;

}

void starformation_and_feedback(int p, int centralgal, double time, double dt, int halonr, int step, int tree, int ngal)
{
  double reff, tdyn, strdot, stars;
  double cold_crit;
  double reheated_mass = 0.0, mass_metals_new = 0.0, mass_stars_recycled = 0.0, ejected_mass = 0.0, NSN = 0.0;

  // Initialise variables
  strdot = 0.0;

  // First we check to see if we need to do delayed supernova feedback.

  if(IRA == 0)
  {
    do_previous_recycling(p, centralgal, step, dt);

    // The total amount that the reservoirs need to be adjusted by for this evolution step (i.e., ALL steps) has already been determined.
    // Within each substep (i.e., each `step` iteration) we ask 'how many iterations are left?' and then adjusts the reservoirs by this amount.
    // After the reservoirs have been updated, we subtract the amount of mass that we added from the running totals.
    //
    // These running totals are required because contemporaneous SN could cause extra mass to be added/removed within the next step iteration.

    update_from_SN_feedback(p, centralgal, Gal[p].reheated_mass / (STEPS-step), Gal[p].ejected_mass / (STEPS-step), Gal[p].mass_stars_recycled / (STEPS-step), Gal[p].mass_metals_new / (STEPS-step), Gal[p].NSN / (STEPS-step), dt);

    Gal[p].reheated_mass -= Gal[p].reheated_mass / (STEPS-step);
    Gal[p].ejected_mass -= Gal[p].ejected_mass / (STEPS-step);
    Gal[p].mass_stars_recycled -= Gal[p].mass_stars_recycled / (STEPS-step);
    Gal[p].mass_metals_new -= Gal[p].mass_metals_new / (STEPS-step);
    Gal[p].NSN -= Gal[p].NSN / (STEPS-step);

  }

  // star formation recipes
  if(SFprescription == 0)
  {
    // we take the typical star forming region as 3.0*r_s using the Milky Way as a guide
    reff = 3.0 * Gal[p].DiskScaleRadius;
    tdyn = reff / Gal[p].Vvir;

    // from Kauffmann (1996) eq7 x piR^2, (Vvir in km/s, reff in Mpc/h) in units of 10^10Msun/h
    cold_crit = 0.19 * Gal[p].Vvir * reff;

    if(Gal[p].ColdGas > cold_crit && tdyn > 0.0)
    	strdot = SfrEfficiency * (Gal[p].ColdGas - cold_crit) / tdyn;
    else
    {
    	strdot = 0.0;
    }
  }
  else
  {
    printf("No star formation prescription selected!\n");
    ABORT(0);
  }

  stars = strdot * dt;
  if(stars < 0.0)
    stars = 0.0;

  if(SupernovaRecipeOn == 1)
  {
    if(IRA == 0)
      do_contemporaneous_SN(p, centralgal, dt, &stars, &reheated_mass, &mass_metals_new, &mass_stars_recycled, &ejected_mass, &NSN);
    else if(IRA == 1)
    {
      do_IRA_SN(p, centralgal, &stars, &reheated_mass, &mass_metals_new, &mass_stars_recycled, &ejected_mass, &NSN);
    }

  }

  if(stars > Gal[p].ColdGas) // we do this check in 'do_current_sn()' but if supernovarecipeon == 0 then we don't do the check.
  {
    double factor = Gal[p].ColdGas / stars;
    stars *= factor;
  }

  update_from_star_formation(p, stars, dt, step, false, tree, ngal);

  update_from_SN_feedback(p, centralgal, reheated_mass, ejected_mass, mass_stars_recycled, mass_metals_new, NSN, dt);

  // check for disk instability
  if(DiskInstabilityOn)
    check_disk_instability(p, centralgal, halonr, time, dt, step, tree, ngal);

}

// This function will calculate the fraction of stars formed in snapshot i that go nova in snapshot j.
// Will also calculate the mass fraction of stars formed in snapshot i that go nova in snapshot j.
// Based off equation (17) and (24) of Mutch et al. (2016).
//
// INPUT: The bounds for the integral (m_low and m_high). ## UNITS: Msun.
// 	: Pointers to modify the values for number and mass fraction of stars that go nova (*Delta_Eta and *Delta_m). ## UNITS: Msun^-1 (Delta_Eta) and unitless (Delta_m).
//
// OUTPUT: None but delta_Eta, the number fraction of stars, and delta_m, the mass fraction of stars, will be modified.

void calculate_Delta_Eta(double m_low, double m_high, double *Delta_Eta, double *Delta_m)
{

  if (m_low < 8.0)
    m_low = 8.0;

  if (m_high < 8.0)
    m_high = 8.0;

  int32_t bin_idx_low = round((m_low - m_IMFbins_low) / (m_IMFbins_delta));
  int32_t bin_idx_high = round((m_high - m_IMFbins_low) / (m_IMFbins_delta));

  if (bin_idx_high == N_massbins)
    --bin_idx_high;

  if (bin_idx_low == N_massbins)
    --bin_idx_low;

  *Delta_Eta = IMF_CONSTANT * (IMF_massgrid_eta[bin_idx_high] - IMF_massgrid_eta[bin_idx_low]);
  *Delta_m = IMF_CONSTANT * (IMF_massgrid_m[bin_idx_high] - IMF_massgrid_m[bin_idx_low]);

}

// This function determines the amount of mass reheated for a supernova event.
//
// INPUT: The number fraction of stars that have gone nova (Delta_Eta) ## UNITS: Msun^-1.
// 	: The mass of stars formed (stars). ## UNITS: 1.0e10Msun/h (code units).
// 	: The maximum velocity of the galaxy disc (Vmax). ## UNITS: km/s.
//
// OUTPUT: The amount of mass reheated by the supernova event. ## UNITS: 1.0e10Msun/h (code units).

double calculate_reheated_mass(double Delta_Eta, double stars, double Vmax)
{

 // if(stars < 1e-10)
 //   return 0.0;

  double epsilon_mass;

  if(RescaleSN == 0)
  {
    epsilon_mass = FeedbackReheatingEpsilon;
  } else
  {
    epsilon_mass = alpha_mass * (0.5 + pow(Vmax/V_mass, -beta_mass));
  }

  if (epsilon_mass > epsilon_mass_max)
  {
    epsilon_mass = epsilon_mass_max; // We enforce a maximum value for the mass loading factor.
  }

  double reheated_mass = Delta_Eta/Eta_SNII * stars * epsilon_mass;

  return reheated_mass;

}

// This function determines the amount of energy injected from a supernova event.
//
// INPUT: The number fraction of stars that have gone nova (Delta_Eta) ## UNITS: Msun^-1.
// 	: The mass of stars formed (stars). ## UNITS: 1.0e10Msun/h (code units).
// 	: The maximum velocity of the galaxy disc (Vmax). ## UNITS: km/s.
//
// OUTPUT: The amount of energy from a supernova event. ## UNITS: erg.

double calculate_reheated_energy(double Delta_Eta, double stars, double Vmax)
{

  double epsilon_energy;

  if(RescaleSN == 0)
  {
    epsilon_energy = FeedbackEjectionEfficiency;
  } else
  {
    epsilon_energy = alpha_energy * (0.5 + pow(Vmax/V_energy, -beta_energy));
  }

  double reheated_energy = 0.5 * Delta_Eta * stars * epsilon_energy * EnergySN; // Delta_Eta was in Msun so need to convert to stars to Msun.

  return reheated_energy;

}

// This function calculates the mass of stars that would have gone supernova in time t.
// Function and fit taken from Portinari et al. (1998).
//
// INPUT: Time (in Myr).
//
// OUTPUT: Mass of stars (in Msun) that would have gone Nova in time t.

double calculate_coreburning(double t)
{

  /*
  double a = 0.7473; // Fits from Portinari et al. (1998).
  double b = -2.6979;
  double c = -4.7659;
  double d = 0.5934;

  double m = pow(10, a/log10(t) + b * exp(c/log10(t)) + d);

  return m;
  */

  int32_t bin_idx = (t - coreburning_tbins_low) / (coreburning_tbins_delta);

  if (bin_idx < 0) // Time is so short that only stars with mass greater than 120Msun can go nova.
    return 120.0;

  if (bin_idx > N_tbins) // Time is so long that all stars within the IMF range can go nova.
    return 8.0;

  return coreburning_times[bin_idx];

}

// If the cold gas that has been reheated has enough energy, it is possible to unbind some of the gas in the hot halo.
// This function determines this threshhold and how much mass is unbound and ejected.
//
// INPUT: Amount of cold gas reheated by supernova events (reheated_mass). ## UNITS: 1.0e10Msun/h (code units).
// 	: Amount of energy injected from the supernova events (reheated_energy). ## UNITS: erg.
// 	: Virial velocity of the background FoF halo (Vvir). ## UNITS: km/s.
//
// OUTPUT: Mass of ejected material. ## UNITS: 1.0e10Msun/h (code units).
//
// Note: Be aware here that a number of unit changes have occurred.
// This is necessary because of how Joules are defined.

double calculate_ejected_mass(double *reheated_mass, double reheated_energy, double Vvir)
{
  double ejected_mass;
  double Delta_Ehot;
  const double Joule_to_erg = 0.5 * SOLAR_MASS / 1.0e3 * 1.0e3 * 1.0e3 * 1.0e7; // Note: This is ergs assuming the mass used is in kg.
                                                                                // As we use 1.0e10 Msun/h our energy isn't quite ergs.
                                                                                // However we have kept this consistent throughout allowing the factor to be cancelled.
  const double inv_Joule_to_erg = 1.0 / Joule_to_erg;

  if(Vvir > 0.0)
  {
    Delta_Ehot = *reheated_mass * Vvir * Vvir * Joule_to_erg; // Change in the thermal energy of the hot resevoir in erg..
    // Note the units here. Want final answer in Ergs (which is 1e-7 Joules) which has SI units of kg m^2 s^-2.
    // We change the mass from 1.0e10Msun/h to kg.  Change virial velocity from km/s to m/s. Finally change Joules to erg.

    if(reheated_energy > Delta_Ehot) // Have enough energy to have some ejected mass.
    {
    	ejected_mass = (reheated_energy - Delta_Ehot) * inv_Joule_to_erg / (Vvir * Vvir); // Balance between the excess thermal energy and the thermal energy of the hot gas.
      // Energy changed from erg to Joules (kg m^2 s^-2) then divided by virial velocity (converted to m/s) giving final units of kg.

    }
    else // Not enough energy to eject mass.
    {
      ejected_mass = 0.0;
      *reheated_mass = reheated_energy * inv_Joule_to_erg / (Vvir * Vvir); // Amount of mass that can be reheated to virial temperature of the hot halo.
    }

  }
  else
    ejected_mass = 0.0;

  if(ejected_mass < 0.0)
    ejected_mass = 0.0;

  return ejected_mass;

}

// This function answers the question, "How many stars formed in the previous time steps will explode during the current supernova timestep."
// Note: Since the time scale on which we calculate SN feedback is variable (defined by the user in 'TimeResolutionSN') we use the terminology 'supernova timestep'.
// Double note: This function is different to "do_contemporaneous_SN" because that looks at the stars formed during the CURRENT STAR FORMATION time step, not previous ones.
void do_previous_SN(int p, int centralgal, double dt)
{
  double reheated_mass = 0.0;
  double reheated_energy = 0.0;
  double mass_stars_recycled = 0.0;
  double mass_metals_new = 0.0;
  double ejected_mass = 0.0;
  double NSN = 0.0; // This is the number of supernova events during this time step.

  int i;
  double m_low = -1.0, m_high = -1.0, t_low = -1.0, t_high = -1.0;
  double Delta_Eta, Delta_m;
  double time_until_next_SN;

  if(dt * UnitTime_in_Megayears / Hubble_h < TimeResolutionSN) // If the star formation time scale is smaller than the time scale on which we do SN feedback
    time_until_next_SN = TimeResolutionSN; // Then the time that we next calculate delayed SN feedback would be given dictated by the time resolution of SN.
  else // Otherwise the star formation time scale is larger than the SN feedback time scale.
  {
    time_until_next_SN = dt * UnitTime_in_Megayears / Hubble_h;  // Then the time that we next calculate delayed SN feedback is in the next SF timestep.
  }

  if(SupernovaRecipeOn == 1) // Calculating the ejected mass due to previous star formation episodes.
  {
    for(i = 1; i < SN_Array_Len; ++i)
    {
      if(Gal[p].SN_Stars[i] < 1e-10)
	    continue;

      // First calculate the smallest star which would have expended its H and He and gone supernova.
      // This defines the mass boundary below which a star will not go nova in the current SN step.

      t_low = (i * TimeResolutionSN) / 2.0 + time_until_next_SN; // (i * TimeResolutionSN) is the number of stars that were formed that many megayears ago.  time_until_next_SN allows us to ask how many of stars will explode in this SN step.
      if(t_low < 2)
  	    t_low = 2;
      if (t_low > 45)
        m_low = 120.0;

      m_low = calculate_coreburning(t_low);

      if (m_low < 8.0) // We enforce that SN-II only occur in stars M > 8Msun.
        m_low = 8.0;

      if (m_low > 120.0)
        continue;

      // Next we calculate the largest stellar mass which would have expended its H and He and gone supernova.
      // This defines the mass boundary beyond which a star would have gone nova BEFORE the current SN step.

      t_high = (i * TimeResolutionSN) / 2.0; // (i * TimeResolutionSN) is the number of stars that were formed that many Megayears ago.  The / 2.0 factor arises from the fact that we assume stars were formed in a single co-eval burst in the middle of this period.
      if(t_high < 2)
        t_high = 2;

      m_high = calculate_coreburning(t_high);

      if (m_high < 8) // In this instance every star that has formed in i Myr ago has already gone supernova and accounted for.
          continue;

      if (m_high > 120.0)
          continue;

      calculate_Delta_Eta(m_low, m_high, &Delta_Eta, &Delta_m); // Calculate the number and mass fraction of stars i Myr ago that go supernova in the current SF step.
      reheated_mass += calculate_reheated_mass(Delta_Eta, Gal[p].SN_Stars[i], Gal[centralgal].Vmax); // Update the amount of mass reheated from previous stars that have gone nova.

      reheated_energy += calculate_reheated_energy(Delta_Eta, Gal[p].SN_Stars[i], Gal[centralgal].Vmax); // Update the energy injected from previous stars that have gone nova.
      mass_stars_recycled += Delta_m * Gal[p].SN_Stars[i]; // Update the amount of stellar mass recycled from previous stars that have gone nova.

      // Since stars have gone supernova, need to update the number of stars remaining at that time.
      if (PhotonPrescription == 1)
      {
        Gal[p].Stellar_Stars[i] -= Delta_m * Gal[p].SN_Stars[i];
        if (Gal[p].Stellar_Stars[i] < 0.0)
          Gal[p].Stellar_Stars[i] = 0.0;
      }

      Gal[p].SN_Stars[i] -= Delta_m * Gal[p].SN_Stars[i];
      if (Gal[p].SN_Stars[i] < 0.0)
        Gal[p].SN_Stars[i] = 0.0;

      mass_metals_new += Delta_m / m_SNII * Yield * Gal[p].SN_Stars[i]; // Update the amount of new metals that the supernova has enriched the ISM with.

      NSN += Delta_Eta * Gal[p].SN_Stars[i] * 1.0e10 / Hubble_h; // The number of supernova events will be simply given by the number fraction of stars that exploded times the mass of stars.

      XASSERT(reheated_mass >= 0.0, "i = %d \t Reheated mass = %.4e \t t_low = %.4e Myr \t m_low = %.4e \t t_high = %.4e Myr \t m_high = %.4e\n", i, reheated_mass, t_low, m_low, t_high, m_high); // Just make sure we're doing this right.

    }

    if(reheated_mass > Gal[p].ColdGas) // Can't reheated more cold gas than we currently have.
        reheated_mass = Gal[p].ColdGas;

    XASSERT(reheated_mass >= 0.0, "Reheated mass = %.4e \t t_low = %.4e Myr \t m_low = %.4e \t t_high = %.4e Myr \t m_high = %.4e\n", reheated_mass, t_low, m_low, t_high, m_high); // Just make sure we're doing this right.
    assert(reheated_energy >= 0.0);
    assert(mass_stars_recycled >= 0.0);
    assert(mass_metals_new >= 0.0);

    ejected_mass = calculate_ejected_mass(&reheated_mass, reheated_energy, Gal[centralgal].Vmax); // Calculate the amount of mass ejected from supernova events.

    XASSERT(ejected_mass >= 0.0, "For galaxy %d the ejected mass was %.4e \t reheated_mass = %.4e\n", p, ejected_mass, reheated_mass);
  }

  Gal[p].reheated_mass = reheated_mass;
  Gal[p].ejected_mass = ejected_mass;
  Gal[p].mass_stars_recycled = mass_stars_recycled;
  Gal[p].mass_metals_new = mass_metals_new;
  Gal[p].NSN = NSN;

}

/*
void do_current_SN(int p, int centralgal, int halonr, double *stars, double *reheated_mass, double *mass_metals_new, double *mass_stars_recycled, double *ejected_mass)
{

  *reheated_mass = 0.0;
  double reheated_energy = 0.0;
  *mass_stars_recycled = 0.0;
  *mass_metals_new = 0.0;
  *ejected_mass = 0.0;

  double Delta_Eta = Eta_SNII;
  double Delta_m = RecycleFraction;
  double mass_fraction = 1.0;

  *mass_stars_recycled = Delta_m * (*stars);
  *reheated_mass = calculate_reheated_mass(Delta_Eta, *stars, Gal[centralgal].Vmax); // Update the amount of mass reheated from stars that have gone nova.

  if(*reheated_mass > Gal[p].ColdGas) // Can only reheat as much cold gas we have available.
      *reheated_mass = Gal[p].ColdGas;
  if((*reheated_mass + *stars > Gal[p].ColdGas) && (*reheated_mass + *stars > 0.0))
  {
    double factor = Gal[p].ColdGas / (*reheated_mass + *stars);
    *reheated_mass *= factor;
    *stars *= factor;
  }

  *mass_metals_new = mass_fraction * Yield * (*stars); // Update the amount of new metals that the supernova has enriched the ISM with.
  reheated_energy = calculate_reheated_energy(Delta_Eta, *stars, Gal[centralgal].Vmax); // Update the energy injected from previous stars that have gone nova.

  assert(*reheated_mass >= 0.0); // Just make sure we're doing this right.
  assert(reheated_energy >= 0.0);
  assert(*mass_stars_recycled >= 0.0);
  assert(*mass_metals_new >= 0.0);

  *ejected_mass = calculate_ejected_mass(&(*reheated_mass), reheated_energy, Gal[centralgal].Vvir); // Calculate the amount of mass ejected from supernova events.

}
*/

void do_previous_recycling(int p, int centralgal, int step, double dt)
{

  if(Gal[p].StellarAge_Numerator > 0.0)
  {
    double t_low, t_high;
    double m_low, m_high;
    double Delta_Eta, Delta_m;
    double mass_stars_recycled;
    double NSN;

    double mwmsa = Gal[p].StellarAge_Numerator / Gal[p].StellarAge_Denominator; // The numerator is weighted by the age of the stars with the denominator simply being the number of stars.
    double time_into_snap = dt * step;

    t_high = (mwmsa - ((Age[Gal[p].SnapNum - 1]) - time_into_snap)) * UnitTime_in_Megayears / Hubble_h;
    if (t_high < 2.0)
      t_high = 2.0;
    if (t_high > 45.0)
      m_high = 8.0;
    else
      m_high = calculate_coreburning(t_high);

    t_low = (mwmsa - (Age[Gal[p].SnapNum] - time_into_snap)) * UnitTime_in_Megayears / Hubble_h;
    if (t_low < 2.0)
      t_low = 2.0;

    if (t_low > 45.0)
      m_low = 8.0;
    else
      m_low = calculate_coreburning(t_low);

    calculate_Delta_Eta(m_low, m_high, &Delta_Eta, &Delta_m); // Calculate the number and mass fraction of stars from snapshot i that go nova in the snapshot we are evolving FROM.
    mass_stars_recycled = Delta_m * Gal[p].StellarAge_Denominator; // Update the amount of stellar mass recycled from previous stars that have gone nova.
    NSN = Delta_Eta * Gal[p].StellarAge_Denominator * 1.0e10 / Hubble_h;
    update_from_SN_feedback(p, centralgal, 0.0, 0.0, mass_stars_recycled, 0.0, NSN, dt);
  }
}

// In this function we answer the question, "How many stars formed in the current star formation time step will explode by the time we next calculate our supernova feedback?"
// Note: This function only concerns itself with stars formed in the CURRENT STAR FORMATION time step, unlike "do_previous_SN" which focuses on calculated SN feedback from PREVIOUS STAR FORMATION time steps.

void do_contemporaneous_SN(int p, int centralgal, double dt, double *stars, double *reheated_mass, double *mass_metals_new, double *mass_stars_recycled, double *ejected_mass, double *NSN)
{

  *reheated_mass = 0.0;
  double reheated_energy = 0.0;
  *mass_stars_recycled = 0.0;
  *mass_metals_new = 0.0;
  *ejected_mass = 0.0;

  double m_low, m_high, t_low;
  double Delta_Eta, Delta_m;
  double fac;

  if(dt * UnitTime_in_Megayears / Hubble_h / 2.0 < TimeResolutionSN) // If the star formation time scale is smaller than the time scale on which we do SN feedback
    t_low = (TimeResolutionSN - Gal[p].Total_SN_SF_Time); // Then our 'sub-grid' SN feedback time will be the time from this SF episode until we next calculate SN feedback (divided by 2 as we assume the stars are formed in the middle of the interval).
  else // Otherwise the star formation time scale is larger than the SN feedback time scale.
    t_low  = (dt * UnitTime_in_Megayears / Hubble_h) / 2.0; // Then the feedback time will be the time from this SF event to the next star formation event. This is because SN feedback is only calculated when star formation occurs regardless of the actual value of 'TimeResolutionSN'.

  if (t_low < 2.0)
    t_low = 2.0;

  m_low = calculate_coreburning(t_low);
  if(m_low < 8.0)
    m_low = 8.0;

  if(m_low > 120.0)
    return;

  m_high = 120.0;

  calculate_Delta_Eta(m_low, m_high, &Delta_Eta, &Delta_m); // Calculate the number and mass fraction of stars i Myr ago that go supernova in the current SF step.
  *reheated_mass = calculate_reheated_mass(Delta_Eta, *stars, Gal[centralgal].Vmax); // Update the amount of mass reheated from previous stars that have gone nova.

  if((*stars + *reheated_mass) > Gal[p].ColdGas && (*stars + *reheated_mass) > 0.0)
  {
    fac = Gal[p].ColdGas / (*stars + *reheated_mass);
    *stars *= fac;
    *reheated_mass *= fac;
  }

  reheated_energy += calculate_reheated_energy(Delta_Eta, *stars, Gal[centralgal].Vmax); // Update the energy injected from previous stars that have gone nova.
  *mass_stars_recycled += Delta_m * (*stars); // Update the amount of stellar mass recycled from previous stars that have gone nova.
  *mass_metals_new += Delta_m / m_SNII * Yield * (*stars); // Update the amount of new metals that the supernova has enriched the ISM with.

  if(*reheated_mass > Gal[p].ColdGas) // Can't reheated more cold gas than we currently have.
    *reheated_mass = Gal[p].ColdGas;

  *ejected_mass = calculate_ejected_mass(&(*reheated_mass), reheated_energy, Gal[centralgal].Vmax); // Calculate the amount of mass ejected from supernova events.
  *NSN = Delta_Eta * (*stars * 1.0e10 / Hubble_h);

  assert(*reheated_mass >= 0.0); // Just make sure we're doing this right.
  assert(reheated_energy >= 0.0);
  assert(*mass_stars_recycled >= 0.0);
  assert(*mass_metals_new >= 0.0);
  assert(*ejected_mass >= 0.0);

}

void do_IRA_SN(int p, int centralgal, double *stars, double *reheated_mass, double *mass_metals_new, double *mass_stars_recycled, double *ejected_mass, double *NSN)
{

  double fac;
  *reheated_mass = 0.0;
  *mass_metals_new = 0.0;
  *mass_stars_recycled = 0.0;
  *ejected_mass = 0.0;

  *reheated_mass = calculate_reheated_mass(Eta_SNII, *stars, Gal[centralgal].Vmax);
  *NSN = Eta_SNII * (*stars);

  if((*stars + *reheated_mass) > Gal[p].ColdGas && (*stars + *reheated_mass) > 0.0)
  {
    fac = Gal[p].ColdGas / (*stars + *reheated_mass);
    *stars *= fac;
    *reheated_mass *= fac;
  }

  double reheated_energy = calculate_reheated_energy(Eta_SNII, *stars, Gal[centralgal].Vmax);

  *ejected_mass = calculate_ejected_mass(&(*reheated_mass), reheated_energy, Gal[centralgal].Vmax);

  if(*ejected_mass < 0.0)
    *ejected_mass = 0.0;

  *mass_metals_new = Yield * (*stars);
  *mass_stars_recycled = RecycleFraction * (*stars);

}

// Local Functions //

// This function updates the array which holds the amount of stars formed in the past 50 Myr.
// As we only need this array for doing delayed SN, if we are using the IRA then this function will never be called.
// If the SF timestep is less than the resolution on which we do supernova feedback then we will keep track of the total stars formed over these small timesteps and then update them all in one bin.
// If the SF timestep is larger than the resolution on which we do supernova feedback we will spread the stars formed evenly over a number of elements (>= 1).
//
// INPUT: The index of the galaxy (p).
// 	: The number of stars formed in the current SF episode (stars). ## UNITS: 1.0e10 Msun/h (Code Units).
// 	: The timestep over which the SF is occuring (dt). ## UNITS: Code Units, multiply by 'UnitTime_in_Megayears / Hubble_h' for Myr.
// 	: The tree currently being used (tree); currently used for debugging purposes.
//
// OUTPUT: None.

void update_SN_stars_array(int p, double stars, double dt, int tree, int ngal)
{

  double time_spanned = dt * UnitTime_in_Megayears / Hubble_h; // The time spanned by this star formation event.

  Gal[p].Total_SN_SF_Time += time_spanned; // How long it has been since we've updated the array?
  Gal[p].Total_SN_Stars += stars; // How many stars we will need to bin once we do update the array?

  if(Gal[p].Total_SN_SF_Time > TimeResolutionSN * SN_Array_Len) // This handles cases in which the time spanned is greater than 50Myr.  In this case we wipe the array clean and push the star formation into a 50Myr bin.
    Gal[p].Total_SN_SF_Time = TimeResolutionSN * SN_Array_Len;

  if(Gal[p].Total_SN_SF_Time < TimeResolutionSN) // If it hasn't been long enough yet, don't update the array.
    return;

  int num_shuffled = round(Gal[p].Total_SN_SF_Time/TimeResolutionSN); // How many cells will this SF event span.

  double stars_spread = Gal[p].Total_SN_Stars/num_shuffled; // We spread the stars evenly over time.

  int i;

  XASSERT(Gal[p].IsMalloced == 1, "We are attempting to update the stars array but this galaxy has already had its arrays freed.\nGalaxy %d \t Halo %d \t Tree %d \t Time spanned %.4eMyr\n", p, Gal[p].HaloNr, tree, time_spanned);

  for(i = SN_Array_Len - 1; i > num_shuffled - 1; --i)
  {
      Gal[p].SN_Stars[i] = Gal[p].SN_Stars[i-num_shuffled]; // Shuffle the current elements of the array far enough along so we can store the new stars.
  }
    XPRINT(p < ngal, "We have the case where the galaxy p is greater than the number of galaxies.  p = %d \t ngal = %d\n", p, ngal);
  for(i = SN_Array_Len - 1; i > (SN_Array_Len - num_shuffled - 1); --i)
  {
    Gal[p].StellarAge_Numerator += Gal[p].SN_Stars[i] * (Age[Gal[p].SnapNum - 4]);
    Gal[p].StellarAge_Denominator += Gal[p].SN_Stars[i];
  }

  for(i = 0; i < num_shuffled; ++i)
  {
    Gal[p].StellarAge_Numerator += Gal[p].SN_Stars[i] * (Age[Gal[p].SnapNum - 4]);
    Gal[p].SN_Stars[i] = stars_spread; // Update the vacated elements with the new stars.
    Gal[p].GrandSum += stars_spread;
  }

  Gal[p].Total_SN_SF_Time = 0.0; // We've updated so reset our variables.
  Gal[p].Total_SN_Stars = 0.0;
}

void update_stellar_tracking(int p, double stars, double dt, int tree, int ngal)
{

  double time_spanned = dt * UnitTime_in_Megayears / Hubble_h; // The time spanned by this star formation event.

  Gal[p].Total_Stellar_SF_Time += time_spanned; // How long it has been since we've updated the array?
  Gal[p].Total_Stellar_Stars += stars; // How many stars we will need to bin once we do update the array?

  if(Gal[p].Total_Stellar_SF_Time > TimeResolutionStellar * StellarTracking_Len)
    Gal[p].Total_Stellar_SF_Time = TimeResolutionStellar * StellarTracking_Len;

  if(Gal[p].Total_Stellar_SF_Time < TimeResolutionStellar) // If it hasn't been long enough yet, don't update the array.
    return;

  int num_shuffled = round(Gal[p].Total_Stellar_SF_Time/TimeResolutionStellar); // How many cells will this SF event span.
  double stars_spread = Gal[p].Total_Stellar_Stars/num_shuffled; // We spread the stars evenly over time.

  int i;

  XASSERT(Gal[p].IsMalloced == 1, "We are attempting to update the Stellar Tracking array but this galaxy has already had its arrays freed.\nGalaxy %d \t Halo %d \t Tree %d \t Time spanned %.4eMyr\n", p, Gal[p].HaloNr, tree, time_spanned);

  for(i = StellarTracking_Len - 1; i > num_shuffled - 1; --i)
  {
      Gal[p].Stellar_Stars[i] = Gal[p].Stellar_Stars[i-num_shuffled]; // Shuffle the current elements of the array far enough along so we can store the new stars.
  }
  XPRINT(p < ngal, "We have the case where the galaxy p is greater than the number of galaxies.  p = %d \t ngal = %d\n", p, ngal);

  for(i = 0; i < num_shuffled; ++i)
  {
    Gal[p].Stellar_Stars[i] = stars_spread; // Update the vacated elements with the new stars.
  }

  Gal[p].Total_Stellar_SF_Time = 0.0; // We've updated so reset our variables.
  Gal[p].Total_Stellar_Stars = 0.0;
}


#ifndef GEMMFORGE_REFERENCE_GEMM_H
#define GEMMFORGE_REFERENCE_GEMM_H

#include "typedef.h"
#include <iostream>

#define GEMMFORGE 1
#define OPENBLAS 2

#define CPU_BACKEND CONCRETE_CPU_BACKEND

#if CPU_BACKEND == OPENBLAS
#include <cblas.h>
#endif

namespace gemmforge {
  namespace reference {
    enum class LayoutType {
      Trans, NoTrans
    };

    void singleGemm(LayoutType TypeA,
                    LayoutType TypeB,
                    int M, int N, int K,
                    real Alpha, real *A, int Lda,
                    real *B, int Ldb,
                    real Beta, real *C, int Ldc);

    real *findData(real *Data, unsigned Stride, unsigned BlockId);
    real *findData(real **Data, unsigned Stride, unsigned BlockId);

    template<typename AT, typename BT, typename CT>
    void gemm(LayoutType TypeA,
              LayoutType TypeB,
              int M, int N, int K,
              real Alpha, AT A, int Lda,
              BT B, int Ldb,
              real Beta, CT C, int Ldc,
              unsigned OffsetA,
              unsigned OffsetB,
              unsigned OffsetC,
              unsigned NumElements) {

      for (unsigned Index = 0; Index < NumElements; ++Index) {
        real *MatrixA = findData(A, OffsetA, Index);
        real *MatrixB = findData(B, OffsetB, Index);
        real *MatrixC = findData(C, OffsetC, Index);

#if CPU_BACKEND == GEMMFORGE
        singleGemm(TypeA, TypeB,
                   M, N, K,
                   Alpha, MatrixA, Lda,
                   MatrixB, Ldb,
                   Beta, MatrixC, Ldc);

#elif CPU_BACKEND == OPENBLAS
  CBLAS_LAYOUT Layout = CblasColMajor;
  CBLAS_TRANSPOSE TransA = TypeA == LayoutType::Trans ? CblasTrans : CblasNoTrans;
  CBLAS_TRANSPOSE TransB = TypeB == LayoutType::Trans ? CblasTrans : CblasNoTrans;

#if REAL_SIZE == 4
  cblas_sgemm(Layout, TransA, TransB,
              M, N, K,
              Alpha, MatrixA, Lda,
              MatrixB, Ldb,
              Beta, MatrixC, Ldc);
#elif REAL_SIZE == 8
  cblas_dgemm(Layout, TransA, TransB,
              M, N, K,
              Alpha, MatrixA, Lda,
              MatrixB, Ldb,
              Beta, MatrixC, Ldc);
#endif

#else
#error "Chosen reference CPU-GEMM impl. is not supported"
#endif
      }
    }
  }
}

#endif //GEMMFORGE_REFERENCE_GEMM_H


#ifndef BSPTN_H
#define BSPTN_H

#include <vector>
#include <gsl/gsl_sf_bessel.h>
#include "SpecialFunctions.h"

// NUMERICAL KERNELS /////

/// CREATION OF KERNEL ARRAYS ////

//Kernel array sizes

// 1st ORDER
extern double F1b_k1;
extern double G1b_k1;
extern double F1b_k2;
extern double G1b_k2;
extern double F1b_k3;
extern double G1b_k3;

extern double F1b_1;
extern double G1b_1;

extern double F1b_2;
extern double G1b_2;

extern double F1b_3;
extern double G1b_3;

// 2nd order
extern double F2b_k12;
extern double G2b_k12;
extern double F2b_k13;
extern double G2b_k13;
extern double F2b_k23;
extern double G2b_k23;

extern double F2b_p2mp;
extern double G2b_p2mp;

extern double F2b_p3mp;
extern double G2b_p3mp;

extern double F2b_12a;
extern double G2b_12a;

extern double F2b_13a;
extern double G2b_13a;

extern double F2b_23a;
extern double G2b_23a;

// 3rd order

extern double F3b_12mp;
extern double G3b_12mp;

extern double F3b_13mp;
extern double G3b_13mp;

extern double F3b_21mp;
extern double G3b_21mp;

extern double F3b_23mp;
extern double G3b_23mp;

extern double F3b_31mp;
extern double G3b_31mp;

extern double F3b_32mp;
extern double G3b_32mp;

extern double F3b_1pp;
extern double G3b_1pp;

extern double F3b_2pp;
extern double G3b_2pp;

extern double F3b_3pp;
extern double G3b_3pp;

// 4th order
extern double F4b_12pp;
extern double G4b_12pp;

extern double F4b_13pp;
extern double G4b_13pp;
extern double F4b_23pp;
extern double G4b_23pp;

const double myh0sqr = pow2(1./2997.92);

class BSPTN {
public:
  // tree level  equation solvers : dgp, f(R)
  void initnb0_dgp(double A, double k[], double x[], double kargs[], double omega0, double par1, double par2, double par3);
  void initnb0_fofr(double A, double k[], double x[], double kargs[], double omega0, double par1, double par2, double par3);

  // 1-loop level  equation solvers : dgp, f(R)
  void initnb1_dgp(double A, double k[], double x[], double kargs[], double omega0, double par1, double par2, double par3, double epars[]);
  void initnb1_fr(double A, double k[], double x[], double kargs[], double omega0, double par1, double par2, double par3, double epars[]);

};

#endif // BSPTN_H