/*
 * BRAINS
 * (B)LR (R)everberation-mapping (A)nalysis (I)n AGNs with (N)ested (S)ampling
 * Yan-Rong Li, liyanrong@ihep.ac.cn
 * Thu, Aug 4, 2016
 */

/*! \file dnest_sa.c
 *  \brief run dnest sampling for sa analysis.
 */

#ifdef SA

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
 #include <stddef.h>
#include <math.h>
#include <float.h>
#include <string.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
#include <gsl/gsl_interp.h>
#include <mpi.h>

#include "brains.h"

DNestFptrSet *fptrset_sa;

int dnest_sa(int argc, char **argv)
{
  int i;
  double logz_sa;

  set_sa_blr_model();

  num_params_blr = 0;  /* RM parameter numbers */
  num_params_sa_blr = num_params_sa_blr_model + num_params_sa_extpar;
  num_params_blr_tot = num_params_sa_blr;
  num_params_sa = num_params_sa_blr;
  num_params = num_params_sa;

  par_fix = (int *) malloc(num_params * sizeof(int));
  par_fix_val = (double *) malloc(num_params * sizeof(double));
  par_range_model = malloc( num_params * sizeof(double *));
  par_prior_gaussian = malloc( num_params * sizeof(double *));
  for(i=0; i<num_params; i++)
  {
    par_range_model[i] = malloc(2*sizeof(double));
    par_prior_gaussian[i] = malloc(2*sizeof(double));
  }
  par_prior_model = malloc( num_params * sizeof(int));

  fptrset_sa = dnest_malloc_fptrset();
  /* setup functions used for dnest*/
  fptrset_sa->from_prior = from_prior_sa;
  fptrset_sa->print_particle = print_particle_sa;
  fptrset_sa->restart_action = restart_action_sa;
  fptrset_sa->accept_action = accept_action_sa;
  fptrset_sa->kill_action = kill_action_sa;
  fptrset_sa->perturb = perturb_sa;
  fptrset_sa->read_particle = read_particle_sa;

  fptrset_sa->log_likelihoods_cal_initial = log_likelihoods_cal_initial_sa;
  fptrset_sa->log_likelihoods_cal_restart = log_likelihoods_cal_restart_sa;
  fptrset_sa->log_likelihoods_cal = log_likelihoods_cal_sa;

  set_par_range_sa();
  set_par_fix_sa_blrmodel();

  for(i=num_params_sa_blr_model; i<num_params; i++)
  {
    par_fix[i] = 0;
    par_fix_val[i] = -DBL_MAX;
  }

  /* fix DA */
  par_fix[num_params_sa_blr_model] = 1;
  par_fix_val[num_params_sa_blr_model] = log(550.0);

  /* fix FA */
  par_fix[num_params_sa_blr_model+2] = 1;
  par_fix_val[num_params_sa_blr_model+2] = log(1.0);

  print_par_names_sa();

  force_update = parset.flag_force_update;
  if(parset.flag_para_name != 1)
    logz_sa = dnest(argc, argv, fptrset_sa, num_params, dnest_options_file);

  dnest_free_fptrset(fptrset_sa);

  return 0;
}

void set_par_range_sa()
{
  int i;

  /* setup parameter range, BLR parameters first */
  for(i=0; i<num_params_sa_blr_model; i++)
  {
    par_range_model[i][0] = sa_blr_range_model[i][0];
    par_range_model[i][1] = sa_blr_range_model[i][1];

    par_prior_model[i] = UNIFORM;
    par_prior_gaussian[i][0] = 0.0;
    par_prior_gaussian[i][1] = 0.0;
  }

  /* sa extra parameters */
  for(i=num_params_sa_blr_model; i<num_params_sa; i++)
  {
    par_range_model[i][0] = sa_extpar_range[i-num_params_sa_blr_model][0];
    par_range_model[i][1] = sa_extpar_range[i-num_params_sa_blr_model][1];

    par_prior_model[i] = UNIFORM;
    par_prior_gaussian[i][0] = 0.0;
    par_prior_gaussian[i][1] = 0.0;
  }
  return;
}

/*!
 *  print names and prior ranges for parameters
 *
 */
void print_par_names_sa()
{
  if(thistask != roottask)
    return;

  int i, j;
  FILE *fp;
  char fname[BRAINS_MAX_STR_LENGTH], str_fmt[BRAINS_MAX_STR_LENGTH];

  sprintf(fname, "%s/%s", parset.file_dir, "data/para_names_sa.txt");
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s.\n", fname);
    exit(0);
  }

  strcpy(str_fmt, "%4d %-15s %10.6f %10.6f %4d %4d %15.6e\n");

  printf("# Print parameter name in %s\n", fname);

  fprintf(fp, "#*************************************************\n");
  fprint_version(fp);
  fprintf(fp, "#*************************************************\n");

  fprintf(fp, "%4s %-15s %10s %10s %4s %4s %15s\n", "#", "Par", "Min", "Max", "Prior", "Fix", "Val");

  i=-1;
  for(j=0; j<num_params_sa_blr_model; j++)
  {
    i++;
    fprintf(fp, str_fmt, i, "SA BLR model", par_range_model[i][0], par_range_model[i][1], par_prior_model[i],
                            par_fix[i], par_fix_val[i]);
  }

  for(j=0; j<num_params_sa_extpar; j++)
  {
    i++;
    fprintf(fp, str_fmt, i, "SA ExtPar", par_range_model[i][0], par_range_model[i][1], par_prior_model[i],
                            par_fix[i], par_fix_val[i]);
  }
  fclose(fp);
  return;
}

/*!
 * this function generates a sample from prior.
 */
void from_prior_sa(void *model)
{
  int i;
  double *pm = (double *)model;

  for(i=0; i<num_params; i++)
  {
    if(par_prior_model[i] == GAUSSIAN)
    {
      pm[i] = dnest_randn()*par_prior_gaussian[i][1] + par_prior_gaussian[i][0];
      dnest_wrap(&pm[i], par_range_model[i][0], par_range_model[i][1]);
    }
    else
    {
      pm[i] = par_range_model[i][0] + dnest_rand() * (par_range_model[i][1] - par_range_model[i][0]);
    }
  }

  /* cope with fixed parameters */
  for(i=0; i<num_params; i++)
  {
    if(par_fix[i] == 1)
      pm[i] = par_fix_val[i];
  }

  which_parameter_update = -1;

  return;
}

/*!
 * this function calculates likelihood at initial step.
 */
double log_likelihoods_cal_initial_sa(const void *model)
{
  double logL;
  logL = prob_sa(model);
  return logL;
}

/*!
 * this function calculates likelihood at initial step.
 */
double log_likelihoods_cal_restart_sa(const void *model)
{
  double logL;
  logL = prob_sa(model);
  return logL;
}

/*!
 * this function calculates likelihood.
 */
double log_likelihoods_cal_sa(const void *model)
{
  double logL;
  logL = prob_sa(model);
  return logL;
}

/*!
 * this function prints out parameters.
 */
void print_particle_sa(FILE *fp, const void *model)
{
  int i;
  double *pm = (double *)model;

  for(i=0; i<num_params; i++)
  {
    fprintf(fp, "%e ", pm[i] );
  }
  fprintf(fp, "\n");
  return;
}

/*!
 * This function read the particle from the file.
 */
void read_particle_sa(FILE *fp, void *model)
{
  int j;
  double *psample = (double *)model;

  for(j=0; j < dnest_num_params; j++)
  {
    if(fscanf(fp, "%lf", psample+j) < 1)
    {
      printf("%f\n", *psample);
      fprintf(stderr, "#Error: Cannot read file %s.\n", options.sample_file);
      exit(0);
    }
  }
  return;
}

/*!
 * this function perturbs parameters.
 */
double perturb_sa(void *model)
{
  double *pm = (double *)model;
  double logH = 0.0, limit1, limit2, width;
  int which, which_level, count_saves;

  /*
   * fixed parameters need not to update
   * perturb important parameters more frequently
   */
  do
  {
    which = dnest_rand_int(num_params);
  }while(par_fix[which] == 1);

  which_parameter_update = which;

  /* level-dependent width */
  count_saves = dnest_get_count_saves();
  which_level_update = dnest_get_which_level_update();
  which_level = which_level_update > (size_levels-10)?(size_levels-10):which_level_update;

  if( which_level > 0 && count_saves > 1000)
  {
    limit1 = limits[(which_level-1) * num_params *2 + which *2];
    limit2 = limits[(which_level-1) * num_params *2 + which *2 + 1];
    width = (limit2 - limit1);
    width /= (3*2.35);
  }
  else
  {
    limit1 = par_range_model[which][0];
    limit2 = par_range_model[which][1];
    width = (par_range_model[which][1] - par_range_model[which][0]);
    width /= (2.35);
  }

  if(par_prior_model[which] == GAUSSIAN)
  {
    logH -= (-0.5*pow((pm[which] - par_prior_gaussian[which][0])/par_prior_gaussian[which][1], 2.0) );
    pm[which] += dnest_randh() * width;
    dnest_wrap(&pm[which], par_range_model[which][0], par_range_model[which][1]);
    //dnest_wrap(&pm[which], limit1, limit2);
    logH += (-0.5*pow((pm[which] - par_prior_gaussian[which][0])/par_prior_gaussian[which][1], 2.0) );
  }
  else
  {
    pm[which] += dnest_randh() * width;
    dnest_wrap(&(pm[which]), par_range_model[which][0], par_range_model[which][1]);
    //dnest_wrap(&pm[which], limit1, limit2);
  }

  return logH;
}

/*!
 * this function calculates likelihood.
 */
double log_likelihoods_cal_sa_exam(const void *model)
{
  return 0.0;
}

/*
 * action when a particle's move is accepted.
 * usually store some values with no need to recompute.
 */
void accept_action_sa()
{
  return;
}

/*
 * action when particle i is killed in cdnest sampling.
 * particle i_copy's properties is copyed to particle i.
 */
void kill_action_sa(int i, int i_copy)
{
  return;
}

#endif


/*
 * Copyright 2020 Makani Technologies LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef SIM_PHYSICS_ROTOR_DATABASE_3D_H_
#define SIM_PHYSICS_ROTOR_DATABASE_3D_H_

#include <gsl/gsl_matrix.h>
#include <gsl/gsl_vector.h>

#include <string>

#include "common/macros.h"

class RotorDatabase3d {
 public:
  explicit RotorDatabase3d(const std::string &filename);
  virtual ~RotorDatabase3d();

  // The rotor force prime frame is attached to an isolated rotor, and is
  // defined as follows:
  // X_prime: The rotor shaft axis, positive in the thrusting direction.
  // Y_prime: The in-plane axis in the Z_prime cross X_prime direction.
  // Z_prime: The in-plane axis aligned with the apparent edgewise wind
  //          direction. Positive is into the wind.

  // Z_prime force [N] is a rotor in-plane force [N].
  double CalcForceZPrime(double angular_rate, double axial_vel,
                         double edgewise_vel, double air_density) const;

  // Y_prime force [N] is the rotor in-plane side force [N].
  // The sign is flipped if rotor direction does not match direction the
  // the database was created assuming.
  double CalcForceYPrime(double angular_rate, double axial_vel,
                         double edgewise_vel, double air_density,
                         int dir) const;

  // Roll moment [N-m] is the right-handed moment about the Z_prime force axis.
  double CalcMomZPrime(double angular_rate, double axial_vel,
                       double edgewise_vel, double air_density, int dir) const;

  // Pitch moment [N-m] is the right-handed moment about the Y_prime axis:
  // positive is pitch up/back.
  double CalcMomYPrime(double angular_rate, double axial_vel,
                       double edgewise_vel, double air_density) const;

  // Thrust [N] is positive for propulsive/motoring, and is equivalent
  // to the rotor X or X_prime axis force.
  double CalcThrust(double angular_rate, double axial_vel, double edgewise_vel,
                    double air_density) const;

  // Torque [N-m] is aerodynamic moment about thrust axis (opposes rotation
  // direction for positive thrusts).
  double CalcTorque(double angular_rate, double axial_vel, double edgewise_vel,
                    double air_density, int dir) const;

  double LookupZForceCoeff(double angular_rate, double axial_vel,
                           double edgewise_vel) const;
  double LookupYForceCoeff(double angular_rate, double axial_vel,
                           double edgewise_vel) const;
  double LookupZMomCoeff(double angular_rate, double axial_vel,
                         double edgewise_vel) const;
  double LookupYMomCoeff(double angular_rate, double axial_vel,
                         double edgewise_vel) const;
  double LookupThrustCoeff(double angular_rate, double axial_vel,
                           double edgewise_vel) const;
  double LookupTorqueCoeff(double angular_rate, double axial_vel,
                           double edgewise_vel) const;
  double CalcThrustScaleFactor(double angular_rate, double axial_vel,
                               double edgewise_vel) const;
  double CalcTorqueScaleFactor(double angular_rate, double axial_vel,
                               double edgewise_vel) const;

 private:
  double radius_;
  double radius_pow4_;
  double radius_pow5_;
  int32_t spin_;

  gsl_vector *angular_rates_;
  gsl_vector *axial_vels_;
  gsl_vector *edgewise_vels_;

  gsl_vector *z_prime_force_coeffs_;
  gsl_vector *y_prime_force_coeffs_;
  gsl_vector *z_prime_mom_coeffs_;
  gsl_vector *y_prime_mom_coeffs_;
  gsl_vector *thrust_coeffs_;
  gsl_vector *torque_coeffs_;

  DISALLOW_COPY_AND_ASSIGN(RotorDatabase3d);
};

#endif  // SIM_PHYSICS_ROTOR_DATABASE_3D_H_


/*
 * C version of Diffusive Nested Sampling (DNest4) by Brendon J. Brewer
 *
 * Yan-Rong Li, liyanrong@mail.ihep.ac.cn
 * Jun 30, 2016
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdbool.h>
#include <gsl/gsl_rng.h>

#include "dnestvars.h"

/* output files */
FILE *fsample, *fsample_info;

/* random number generator */
const gsl_rng_type * dnest_gsl_T;
gsl_rng * dnest_gsl_r;

Options options;
char options_file[STR_MAX_LENGTH];

// sampler
bool save_to_disk;
double compression;
unsigned int regularisation;

void *particles;
int dnest_size_of_modeltype;
int particle_offset_size, particle_offset_double;

LikelihoodType *log_likelihoods;
unsigned int *level_assignments;

// number account of unaccepted times
unsigned int *account_unaccepts;

int size_levels;
Level *levels;
unsigned int count_saves, num_saves, num_saves_restart;
int dnest_which_particle_update; // which particle to be updated
int dnest_which_level_update;    // which level to be updated;
unsigned long long int count_mcmc_steps;
LikelihoodType *above;
unsigned int size_above;

double post_logz;
int dnest_num_params;
char dnest_sample_postfix[STR_MAX_LENGTH], dnest_sample_tag[STR_MAX_LENGTH], dnest_sample_dir[STR_MAX_LENGTH];

double *limits, *copies_of_limits;

int *dnest_perturb_accept;
int dnest_root;

int dnest_flag_restart=0, dnest_flag_postprc=0, dnest_flag_sample_info=0, dnest_flag_limits=0;
double dnest_post_temp=1.0;
char file_restart[STR_MAX_LENGTH], file_save_restart[STR_MAX_LENGTH];

void *dnest_arg;

//***********************************************
/*                  functions                  */
double mod(double y, double x);
void dnest_wrap(double *x, double min, double max);
void wrap_limit(double *x, double min, double max);
int mod_int(int y, int x);
int dnest_cmp(const void *pa, const void *pb);

void options_load(int max_num_saves, double ptol);
void setup(int argc, char** argv, DNestFptrSet *fptrset, int num_params, char *sample_dir, int max_num_saves, double ptol);
void finalise();

double dnest(int argc, char **argv, DNestFptrSet *fptrset,  int num_params, char *sample_dir,
             int max_num_saves, double pdff, const void *arg);
void dnest_run();
void dnest_mcmc_run();
void update_particle(unsigned int which);
void update_level_assignment(unsigned int which);
double log_push(unsigned int which_level);
bool enough_levels(Level *l, int size_l);
void do_bookkeeping();
void save_levels();
void save_particle();
void save_limits();
void kill_lagging_particles();
void renormalise_visits();
void recalculate_log_X();
double dnest_randh();
double dnest_rand();
double dnest_randn();
int dnest_rand_int(int size);
void dnest_postprocess(double temperature, int max_num_saves, double ptol);
void postprocess(double temperature);
void initialize_output_file();
void close_output_file();
void dnest_save_restart();
void dnest_restart();
void dnest_restart_action(int iflag);
void dnest_accept_action();
void dnest_kill_action(int i, int i_copy);
void dnest_print_particle(FILE *fp, const void *model, const void *arg);
void dnest_read_particle(FILE *fp, void *model);
int dnest_get_size_levels();
int dnest_get_which_level_update();
int dnest_get_which_particle_update();
void dnest_get_posterior_sample_file(char *fname);
int dnest_check_version(char *verion_str);
unsigned int dnest_get_which_num_saves();
unsigned int dnest_get_count_saves();
unsigned long long int dnest_get_count_mcmc_steps();
void dnest_check_fptrset(DNestFptrSet *fptrset);
DNestFptrSet * dnest_malloc_fptrset();
void dnest_free_fptrset(DNestFptrSet * fptrset);
/*=====================================================*/
// users responsible for following functions
void (*print_particle)(FILE *fp, const void *model, const void *arg);
void (*read_particle)(FILE *fp, void *model);
void (*from_prior)(void *model, const void *arg);
double (*log_likelihoods_cal)(const void *model, const void *arg);
double (*log_likelihoods_cal_initial)(const void *model, const void *arg);
double (*log_likelihoods_cal_restart)(const void *model, const void *arg);
double (*perturb)(void *model, const void *arg);
void (*restart_action)(int iflag);
void (*accept_action)();
void (*kill_action)(int i, int i_copy);
/*=====================================================*/


#ifndef _UTIL_H
#define _UTIL_H

#include <omp.h>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <time.h>
#include <chrono>
#include <string>
#include <random>
#include <algorithm>
#include <mutex>
#include <atomic>
#include <tbb/pipeline.h>
#include "blocks.pb.h"
#ifdef __APPLE__
extern "C"
{
#include <cblas.h>
}
extern "C"
{
  void cblas_saxpy(const int N, const float alpha, const float *X,
		   const int incX, float *Y, const int incY);
  void cblas_scopy(const int N, const float *X, const int incX,
		   float *Y, const int incY);
  float cblas_sdot(const int N, const float  *X, const int incX,
		    const float  *Y, const int incY);
}
#else
#include "mkl.h"
#endif
typedef unsigned long long uint64;
typedef unsigned int       uint32;
typedef std::chrono::high_resolution_clock Time;
#ifdef LEVEL1_DCACHE_LINESIZE
#define CACHE_LINE_SIZE LEVEL1_DCACHE_LINESIZE
#else
#define CACHE_LINE_SIZE 64
#endif

typedef struct {
    int u_, v_;
    float r_;
} Record;

extern std::chrono::time_point<Time> s,e;
extern std::default_random_engine generator;
extern std::normal_distribution<float> gaussian;

#ifdef FETCH
inline void prefetch_range(char *addr, size_t len) {
  char *cp;
  char *end = addr + len;

  for (cp = addr; cp < end; cp += CACHE_LINE_SIZE)
    __builtin_prefetch(cp,1,0);
}
#endif

inline void align_alloc(float** u, int nu, int dim) {
    int piece = nu/1050000+1;
    int nn = nu/piece;
    int k;
    for(k=0; k<piece-1; k++) {
        u[k*nn] = (float*)mkl_malloc(nn*dim*sizeof(float), CACHE_LINE_SIZE);
        for(int i=1; i<nn; i++)
            u[k*nn+i] = u[k*nn+i-1] + dim;
    }
    u[k*nn] = (float*)mkl_malloc((nn+nu%piece)*dim*sizeof(float), CACHE_LINE_SIZE);
    for(int i=1; i<nn+nu%piece; i++)
        u[k*nn+i] = u[k*nn+i-1] + dim;
}

inline void plain_read(const char* data, mf::Blocks& blocks) {
  FILE* fr = fopen(data, "rb");
  std::vector<char> buf;
  uint32 isize;
  mf::Block* bk;
  while(fread(&isize, 1, sizeof(isize), fr)) {
      buf.resize(isize);
      fread((char*)buf.data(), 1, isize, fr);
      bk = blocks.add_block();
      bk->ParseFromArray(buf.data(), isize);
  }
  fclose(fr);
}

inline float active(float val, int type) {
  switch(type) {
    case 0: return val;                     //least square
    case 1: return 1.0f/(1.0f+expf(-val));  //sigmoid
  }
}
inline float cal_grad(float r, float pred, int type) {
  switch(type) {
    case 0: return r - pred;          //least square
    case 1: return r - pred;          //0-1 logistic regression
  }
}

inline float next_float(){
    return static_cast<float>( rand() ) / (static_cast<float>( RAND_MAX )+1.0);
}

inline float next_float2(){
    return (static_cast<float>( rand() ) + 1.0 ) / (static_cast<float>(RAND_MAX) + 2.0);
}

inline float normsqr(float* x, int num) {
   return cblas_sdot(num, x, 1, x, 1);
}

inline float sample_normal(){
    float x,y,s;
    do{
        x = 2 * next_float2() - 1.0;
        y = 2 * next_float2() - 1.0;
        s = x*x + y*y;
    }while( s >= 1.0 || s == 0.0 );

    return x * sqrt( -2.0 * log(s) / s ) ;
}

inline float sample_gamma( float alpha, float beta ) {
    if ( alpha < 1.0 ) {
        float u;
        do {
            u = next_float();
        } while (u == 0.0);
        return sample_gamma(alpha + 1.0, beta) * pow(u, 1.0 / alpha);
    } else {
        float d,c,x,v,u;
        d = alpha - 1.0/3.0;
        c = 1.0 / sqrt( 9.0 * d );
        do {
            do {
                x = sample_normal();
                v = 1.0 + c*x;
            } while ( v <= 0.0 );
            v = v * v * v;
            u = next_float();
        } while ( (u >= (1.0 - 0.0331 * (x*x) * (x*x)))
            && (log(u) >= (0.5 * x * x + d * (1.0 - v + log(v)))) );
        return d * v / beta;
    }
}

inline void gamma_posterior( float &lambda, float prior_alpha, float prior_beta, float psum_sqr, float psum_cnt ){
    float alpha = prior_alpha + 0.5*psum_cnt;
    float beta  = prior_beta + 0.5*psum_sqr;
    lambda = sample_gamma( alpha, beta );
}

inline void normsqr_col(float** m, int d, int size, float* norm) {
#pragma omp parallel for
    for(int i=0; i<d; i++) {
        for(int j=0; j<size; j++) norm[i] += m[j][i]*m[j][i];
    }
}

inline int padding(int dim) {
    return ((dim*sizeof(float)-1)/CACHE_LINE_SIZE*CACHE_LINE_SIZE+CACHE_LINE_SIZE)/sizeof(float);
}

#endif


/* wavelet/gsl_wavelet.h
 *
 * Copyright (C) 2004 Ivo Alxneit
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#ifndef __GSL_WAVELET2D_H__
#define __GSL_WAVELET2D_H__
#include <stdlib.h>
#include <gsl/gsl_errno.h>
#include <gsl/gsl_vector_double.h>
#include <gsl/gsl_matrix_double.h>
#include <gsl/gsl_wavelet.h>
#include <gsl/gsl_types.h>

#undef __BEGIN_DECLS
#undef __END_DECLS
#ifdef __cplusplus
# define __BEGIN_DECLS extern "C" {
# define __END_DECLS }
#else
# define __BEGIN_DECLS          /* empty */
# define __END_DECLS            /* empty */
#endif

__BEGIN_DECLS

GSL_EXPORT int gsl_wavelet2d_transform (const gsl_wavelet * w,
                                        double *data,
                                        size_t tda, size_t size1, size_t size2,
                                        gsl_wavelet_direction dir,
                                        gsl_wavelet_workspace * work);

GSL_EXPORT int gsl_wavelet2d_transform_forward (const gsl_wavelet * w,
                                                double *data,
                                                size_t tda, size_t size1, size_t size2,
                                                gsl_wavelet_workspace * work);

GSL_EXPORT int gsl_wavelet2d_transform_inverse (const gsl_wavelet * w,
                                                double *data,
                                                size_t tda, size_t size1, size_t size2,
                                                gsl_wavelet_workspace * work);

GSL_EXPORT int gsl_wavelet2d_nstransform (const gsl_wavelet * w,
                                          double *data,
                                          size_t tda, size_t size1, size_t size2,
                                          gsl_wavelet_direction dir,
                                          gsl_wavelet_workspace * work);

GSL_EXPORT int gsl_wavelet2d_nstransform_forward (const gsl_wavelet * w,
                                                  double *data,
                                                  size_t tda, size_t size1, size_t size2,
                                                  gsl_wavelet_workspace * work);

GSL_EXPORT int gsl_wavelet2d_nstransform_inverse (const gsl_wavelet * w,
                                                  double *data,
                                                  size_t tda, size_t size1, size_t size2,
                                                  gsl_wavelet_workspace * work);

GSL_EXPORT
int
gsl_wavelet2d_transform_matrix (const gsl_wavelet * w,
                                gsl_matrix * a,
                                gsl_wavelet_direction dir,
                                gsl_wavelet_workspace * work);

GSL_EXPORT
int
gsl_wavelet2d_transform_matrix_forward (const gsl_wavelet * w,
                                        gsl_matrix * a,
                                        gsl_wavelet_workspace * work);

GSL_EXPORT
int
gsl_wavelet2d_transform_matrix_inverse (const gsl_wavelet * w,
                                        gsl_matrix * a,
                                        gsl_wavelet_workspace * work);

GSL_EXPORT
int
gsl_wavelet2d_nstransform_matrix (const gsl_wavelet * w,
                                  gsl_matrix * a,
                                  gsl_wavelet_direction dir,
                                  gsl_wavelet_workspace * work);

GSL_EXPORT
int
gsl_wavelet2d_nstransform_matrix_forward (const gsl_wavelet * w,
                                          gsl_matrix * a,
                                          gsl_wavelet_workspace * work);

GSL_EXPORT
int
gsl_wavelet2d_nstransform_matrix_inverse (const gsl_wavelet * w,
                                          gsl_matrix * a,
                                          gsl_wavelet_workspace * work);

__END_DECLS

#endif /* __GSL_WAVELET2D_H__ */


/* randist/shuffle.c
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 2007 James Theiler, Brian Gough
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#include <config.h>
#include <stdlib.h>
#include <math.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>

/* Inline swap and copy functions for moving objects around */

static inline
void swap (void * base, size_t size, size_t i, size_t j)
{
  register char * a = size * i + (char *) base ;
  register char * b = size * j + (char *) base ;
  register size_t s = size ;

  if (i == j)
    return ;

  do
    {
      char tmp = *a;
      *a++ = *b;
      *b++ = tmp;
    }
  while (--s > 0);
}

static inline void
copy (void * dest, size_t i, void * src, size_t j, size_t size)
{
  register char * a = size * i + (char *) dest ;
  register char * b = size * j + (char *) src ;
  register size_t s = size ;

  do
    {
      *a++ = *b++;
    }
  while (--s > 0);
}

/* Randomly permute (shuffle) N indices

   Supply an array x[N] with nmemb members, each of size size and on
   return it will be shuffled into a random order.  The algorithm is
   from Knuth, SemiNumerical Algorithms, v2, p139, who cites Moses and
   Oakford, and Durstenfeld */

void
gsl_ran_shuffle (const gsl_rng * r, void * base, size_t n, size_t size)
{
  size_t i ;

  for (i = n - 1; i > 0; i--)
    {
      size_t j = gsl_rng_uniform_int(r, i+1); /* originally (i + 1) * gsl_rng_uniform (r) */

      swap (base, size, i, j) ;
    }
}

int
gsl_ran_choose (const gsl_rng * r, void * dest, size_t k, void * src,
                 size_t n, size_t size)
{
  size_t i, j = 0;

  /* Choose k out of n items, return an array x[] of the k items.
     These items will prevserve the relative order of the original
     input -- you can use shuffle() to randomize the output if you
     wish */

  if (k > n)
    {
      GSL_ERROR ("k is greater than n, cannot sample more than n items",
                 GSL_EINVAL) ;
    }

  for (i = 0; i < n && j < k; i++)
    {
      if ((n - i) * gsl_rng_uniform (r) < k - j)
        {
          copy (dest, j, src, i, size) ;
          j++ ;
        }
    }

  return GSL_SUCCESS;
}

void
gsl_ran_sample (const gsl_rng * r, void * dest, size_t k, void * src,
                size_t n, size_t size)
{
  size_t i, j = 0;

  /* Choose k out of n items, with replacement */

  for (i = 0; i < k; i++)
    {
      j = gsl_rng_uniform_int (r, n);  /* originally n * gsl_rng_uniform (r) */

      copy (dest, i, src, j, size) ;
    }
}


// THE VIRTUAL BRAIN -- C
// A fast implementation of TVB-style brain network models based on
// DYNAMIC MEAN FIELD MODEL Deco et al. 2014 Journal of Neuroscience
//
//  m.schirner@fu-berlin.de
//  michael.schirner@charite.de
//
// MIT LICENSE
// Copyright 2020 Michael Schirner
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#include <stdio.h>
#include <xmmintrin.h>
#include <emmintrin.h>
#include <stdlib.h>
#include <time.h>
#include <string.h>
#include <math.h>
#include <pthread.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
//#include <semaphore.h>

/*

 Definition of pthread barrier -- since OSX doesn't seem to have it

 */

 typedef int pthread_barrierattr_t;
 typedef struct
 {
 pthread_mutex_t mutex;
 pthread_cond_t cond;
 int count;
 int tripCount;
 } pthread_barrier_t;

 int pthread_barrier_init(pthread_barrier_t *barrier, const pthread_barrierattr_t *attr, unsigned int count)
 {
 if(count == 0)
 {
 //errno = EINVAL;
 return -1;
 }
 if(pthread_mutex_init(&barrier->mutex, 0) < 0)
 {
 return -1;
 }
 if(pthread_cond_init(&barrier->cond, 0) < 0)
 {
 pthread_mutex_destroy(&barrier->mutex);
 return -1;
 }
 barrier->tripCount = count;
 barrier->count = 0;

 return 0;
 }

 int pthread_barrier_destroy(pthread_barrier_t *barrier)
 {
 pthread_cond_destroy(&barrier->cond);
 pthread_mutex_destroy(&barrier->mutex);
 return 0;
 }

 int pthread_barrier_wait(pthread_barrier_t *barrier)
 {
 pthread_mutex_lock(&barrier->mutex);
 ++(barrier->count);
 if(barrier->count >= barrier->tripCount)
 {
 barrier->count = 0;
 pthread_cond_broadcast(&barrier->cond);
 pthread_mutex_unlock(&barrier->mutex);
 return 1;
 }
 else
 {
 pthread_cond_wait(&barrier->cond, &(barrier->mutex));
 pthread_mutex_unlock(&barrier->mutex);
 return 0;
 }
 }

/*

 End: Definition of pthread barrier

 */

pthread_mutex_t mutex_thrcount;

struct Xi_p{
    float **Xi_elems;
};

struct SC_capS{
    float *cap;
};

struct SC_inpregS{
    int *inpreg;
};

pthread_barrier_t mybarrier_base, mybarrier1, mybarrier2, mybarrier3;

#define REAL float
//#define REAL double

int divRoundClosest(const int n, const int d)
{
    return ((n < 0) ^ (d < 0)) ? ((n - d/2)/d) : ((n + d/2)/d);
}

int divRoundUp(const int x, const int y)
{
    return (x + y - 1) / y;
}

/* Compute Pearson's correlation coefficient */
float corr(float *x, float *y, int n){
    int i;
    float mx=0, my=0;

    /* Calculate the mean of the two series x[], y[] */
    for (i=0; i<n; i++) {
        mx += x[i];
        my += y[i];
    }
    mx /= n;
    my /= n;

    /* Calculate the correlation */
    float sxy = 0, sxsq = 0, sysq = 0, tmpx, tmpy;
    for (i=0; i<n; i++) {
        tmpx = x[i] - mx;
        tmpy = y[i] - my;
        sxy += tmpx*tmpy;
        sxsq += tmpx*tmpx;
        sysq += tmpy*tmpy;
    }

    return (sxy / (sqrt(sxsq)*sqrt(sysq)));
}

//float gaussrand_ret()
//{
//    static double V1=0.0, V2=0.0, S=0.0, U1=0.0, U2=0.0;
//    S=0.0;
//    do {
//        U1 = (double)rand() / RAND_MAX;
//        U2 = (double)rand() / RAND_MAX;
//        V1 = 2 * U1 - 1;
//        V2 = 2 * U2 - 1;
//        S = V1 * V1 + V2 * V2;
//    } while(S >= 1 || S == 0);
//
//    return (float)(V1 * sqrt(-2 * log(S) / S));
//}
//
//static inline void gaussrand(float *randnum)
//{
//    static double V1=0.0, V2=0.0, S=0.0, U1=0.0, U2=0.0;
//
//    S=0.0;
//    do {
//        U1 = (double)rand() / RAND_MAX;
//        U2 = (double)rand() / RAND_MAX;
//        V1 = 2 * U1 - 1;
//        V2 = 2 * U2 - 1;
//        S = V1 * V1 + V2 * V2;
//    } while(S >= 1 || S == 0);
//
//    randnum[0] = (float)(V1 * sqrt(-2 * log(S) / S));
//    randnum[1] = (float)(V2 * sqrt(-2 * log(S) / S));
//
//    S=0.0;
//    do {
//        U1 = (double)rand() / RAND_MAX;
//        U2 = (double)rand() / RAND_MAX;
//        V1 = 2 * U1 - 1;
//        V2 = 2 * U2 - 1;
//        S = V1 * V1 + V2 * V2;
//    } while(S >= 1 || S == 0);
//
//    randnum[2] = (float)(V1 * sqrt(-2 * log(S) / S));
//    randnum[3] = (float)(V2 * sqrt(-2 * log(S) / S));
//}

int importGlobalConnectivity(char *SC_cap_filename, char *SC_dist_filename, char *SC_inputreg_filename, int regions, float **region_activity, struct Xi_p **reg_globinp_p, float global_trans_v, int **n_conn_table, float G_J_NMDA, struct SC_capS **SC_cap, float **SC_rowsums, struct SC_inpregS **SC_inpreg)
{

    int i,j,k, maxdelay=0, tmpint;
    float *region_activity_p;
    double tmp, tmp2;
    struct Xi_p *reg_globinp_pp;
    struct SC_capS      *SC_capp;
    struct SC_inpregS   *SC_inpregp;

    // Open SC files
    FILE *file_cap, *file_dist;
    file_cap=fopen(SC_cap_filename, "r");
    file_dist=fopen(SC_dist_filename, "r");
    if (file_cap==NULL || file_dist==NULL)
    {
        printf( "\nERROR: Could not open SC files. Terminating... \n\n");
        exit(0);
    }

    // Allocate a counter that stores number of region input connections for each region and the SCcap array
    *SC_rowsums = (float *)_mm_malloc(regions*sizeof(float),16);
    *n_conn_table = (int *)_mm_malloc(regions*sizeof(int),16);
    *SC_cap = (struct SC_capS *)_mm_malloc(regions*sizeof(struct SC_capS),16);
    SC_capp = *SC_cap;
    *SC_inpreg = (struct SC_inpregS *)_mm_malloc(regions*sizeof(struct SC_inpregS),16);
    SC_inpregp = *SC_inpreg;
    if(*n_conn_table==NULL || SC_capp==NULL || SC_rowsums==NULL || SC_inpregp==NULL){
        printf("Running out of memory. Terminating.\n");fclose(file_dist);fclose(file_cap);exit(2);
    }

    // Read out the maximal fiber length and the degree of each node and rewind SC files
    int n_entries = 0, curr_col = 0, curr_row = 0, curr_row_nonzero = 0;
    double tmp_max = -9999;
    while(fscanf(file_dist,"%lf",&tmp) != EOF && fscanf(file_cap,"%lf",&tmp2) != EOF){
        if (tmp_max < tmp) tmp_max = tmp;
        n_entries++;

        if (tmp2 > 0.0) curr_row_nonzero++;

        curr_col++;
        if (curr_col == regions) {
            curr_col = 0;
            (*n_conn_table)[curr_row] = curr_row_nonzero;
            curr_row_nonzero = 0;
            curr_row++;
        }
    }

    if (n_entries < regions * regions) {
        printf( "ERROR: Unexpected end-of-file in file. File contains less input than expected. Terminating... \n");
        exit(0);
    }

    rewind(file_dist);
    rewind(file_cap);

    maxdelay = (int)(((tmp_max/global_trans_v)*10)+0.5); // *10 for getting from m/s to 10kHz sampling, +0.5 for rounding by casting
    if (maxdelay < 1) maxdelay = 1; // Case: no time delays

    // Allocate ringbuffer that contains region activity for each past time-step until maxdelay and another ringbuffer that contains pointers to the first ringbuffer
    *region_activity = (float *)_mm_malloc(maxdelay*regions*sizeof(float),16);
    region_activity_p = *region_activity;
    *reg_globinp_p = (struct Xi_p *)_mm_malloc(maxdelay*regions*sizeof(struct Xi_p),16);
    reg_globinp_pp = *reg_globinp_p;
    if(region_activity_p==NULL || reg_globinp_p==NULL){
        printf("Running out of memory. Terminating.\n");fclose(file_dist);exit(2);
    }
    for (j=0; j<maxdelay*regions; j++) {
        region_activity_p[j]=0.001;
    }

    // Read SC files and set pointers for each input region and correspoding delay for each ringbuffer time-step
    int ring_buff_position;
    for (i=0; i<regions; i++) {

        if ((*n_conn_table)[i] > 0) {
            // SC strength and inp region numbers
            SC_capp[i].cap          = (float *)_mm_malloc(((*n_conn_table)[i])*sizeof(float),16);
            SC_inpregp[i].inpreg    = (int *)_mm_malloc(((*n_conn_table)[i])*sizeof(int),16);
            if(SC_capp[i].cap==NULL || SC_inpregp[i].inpreg==NULL){
                printf("Running out of memory. Terminating.\n");exit(2);
            }

            // Allocate memory for input-region-pointer arrays for each time-step in ringbuffer
            for (j=0; j<maxdelay; j++){
                reg_globinp_pp[i+j*regions].Xi_elems=(float **)_mm_malloc(((*n_conn_table)[i])*sizeof(float *),16);
                if(reg_globinp_pp[i+j*regions].Xi_elems==NULL){
                    printf("Running out of memory. Terminating.\n");exit(2);
                }
            }

            float sum_caps=0.0;
            // Read incoming connections and set pointers
            curr_row_nonzero = 0;
            for (j=0; j<regions; j++) {
                if(fscanf(file_cap,"%lf",&tmp) != EOF && fscanf(file_dist,"%lf",&tmp2) != EOF){
                    if (tmp > 0.0) {
                        tmpint = (int)(((tmp2/global_trans_v)*10)+0.5); //  *10 for getting from m/s or mm/ms to 10kHz sampling, +0.5 for rounding by casting
                        if (tmpint < 0 || tmpint > maxdelay){
                            printf( "\nERROR: Negative or too high (larger than maximum specified number) connection length/delay %d -> %d. Terminating... \n\n",i,j);exit(0);
                        }
                        if (tmpint <= 0) tmpint = 1; // If time delay is smaller than integration step size, than set time delay to one integration step

                        SC_capp[i].cap[curr_row_nonzero] = (float)tmp * G_J_NMDA;
                        //sum_caps                += (float)tmp;
                        sum_caps                += SC_capp[i].cap[curr_row_nonzero];
                        SC_inpregp[i].inpreg[curr_row_nonzero]  =  j;

                        ring_buff_position=maxdelay*regions - tmpint*regions + j;
                        for (k=0; k<maxdelay; k++) {
                            reg_globinp_pp[i+k*regions].Xi_elems[curr_row_nonzero]=&region_activity_p[ring_buff_position];
                            ring_buff_position += regions;
                            if (ring_buff_position > (maxdelay*regions-1)) ring_buff_position -= maxdelay*regions;
                        }

                        curr_row_nonzero++;
                    }
                } else{
                    printf( "\nERROR: Unexpected end-of-file in file %s or %s. File contains less input than expected. Terminating... \n\n", SC_cap_filename, SC_dist_filename);
                    exit(0);
                }

            }
            if (sum_caps <= 0) {
                printf( "\nERROR: Sum of connection strenghts is negative or zero. sum-caps node %d = %f. Terminating... \n\n",i,sum_caps);exit(0);
            }
            (*SC_rowsums)[i] = sum_caps;
        }
    }

    fclose(file_dist);fclose(file_cap);
    return maxdelay;
}

/*
 Initialize thread barriers
 */
void initialize_thread_barriers(int n_threads) {
    pthread_barrier_init(&mybarrier1, NULL, n_threads);
    pthread_barrier_init(&mybarrier2, NULL, n_threads);
    pthread_barrier_init(&mybarrier3, NULL, n_threads);
    return;
}

/* create thread argument struct for thr_func() */
typedef struct _thread_data_t {
    int                 tid, rand_num_seed;
    int                 nodes, nodes_vec, fake_nodes, n_threads, vectorization_grade, time_steps, BOLD_TR, BOLD_ts_len;
    float               *J_i;
    int                 reg_act_size;
    float               *region_activity;
    float               model_dt;
    __m128              _gamma;
    __m128              _one;
    __m128              _imintau_E;
    __m128              _dt;
    __m128              _sigma_sqrt_dt;
    __m128              _sigma;
    __m128              _gamma_I;
    __m128              _imintau_I;
    __m128              _min_d_I;
    __m128              _b_I;
    __m128              _J_NMDA;
    __m128              _w_I__I_0;
    __m128              _a_I;
    __m128              _min_d_E;
    __m128              _b_E;
    __m128              _a_E;
    __m128              _w_plus_J_NMDA;
    __m128              _w_E__I_0;
    struct SC_capS      *SC_cap;
    struct SC_inpregS   *SC_inpreg;
    struct Xi_p         *reg_globinp_p;
    int                 *n_conn_table;
    float               *BOLD_ex;
    char                *output_file;
} thread_data_t;

/*

 Function for multi-threaded FIC tuning

 */
void *run_simulation(void *arg)
{

    /* Local function parameters (no concurrency) */
    int j, i_node_vec, i_node_vec_local, k, int_i, ts;
    float tmpglobinput, tmpglobinput_FFI;
    __m128 _tmp_H_E, _tmp_H_I, _tmp_I_I, _tmp_I_E;
    float tmp_exp_E[4]          __attribute__((aligned(16)));
    float tmp_exp_I[4]          __attribute__((aligned(16)));
    __m128          *_tmp_exp_E         = (__m128*)tmp_exp_E;
    __m128          *_tmp_exp_I         = (__m128*)tmp_exp_I;
    float rand_number[4]        __attribute__((aligned(16)));
    __m128          *_rand_number       = (__m128*)rand_number;
    int     ring_buf_pos                = 0;

    /* Global function parameters (some with concurrency) */
    thread_data_t *thr_data = (thread_data_t *)arg;
    //printf("Hello from thread %d, process %d\n", thr_data->tid, thr_data->rank);
    int     t_id                    = thr_data->tid;
    int     rand_num_seed           = thr_data->rand_num_seed;
    int     nodes                   = thr_data->nodes;
    int     fake_nodes              = thr_data->fake_nodes;
    const   int nodes_vec           = thr_data->nodes_vec;
    int     n_threads               = thr_data->n_threads;
    int     vectorization_grade     = thr_data->vectorization_grade;
    int     reg_act_size            = thr_data->reg_act_size;
    int     *n_conn_table           = thr_data->n_conn_table;
    float   *J_i                    = thr_data->J_i;
    float   *region_activity        = thr_data->region_activity;
    float   *BOLD_ex                = thr_data->BOLD_ex;
    const   __m128 _gamma           = thr_data->_gamma;
    const   __m128 _one             = thr_data->_one;
    const   __m128 _imintau_E       = thr_data->_imintau_E;
    const   __m128 _dt              = thr_data->_dt;
    const   __m128 _sigma_sqrt_dt   = thr_data->_sigma_sqrt_dt;
    const   __m128 _gamma_I         = thr_data->_gamma_I;
    const   __m128 _imintau_I       = thr_data->_imintau_I;
    const   __m128 _min_d_I         = thr_data->_min_d_I;
    const   __m128 _b_I             = thr_data->_b_I;
    const   __m128 _J_NMDA          = thr_data->_J_NMDA;
    const   __m128 _w_I__I_0        = thr_data->_w_I__I_0;
    const   __m128 _a_I             = thr_data->_a_I;
    const   __m128 _min_d_E         = thr_data->_min_d_E;
    const   __m128 _b_E             = thr_data->_b_E;
    const   __m128 _a_E             = thr_data->_a_E;
    const   __m128 _w_plus_J_NMDA   = thr_data->_w_plus_J_NMDA;
    const   __m128 _w_E__I_0        = thr_data->_w_E__I_0;
    struct  SC_capS *SC_cap         = thr_data->SC_cap;
    struct  Xi_p *reg_globinp_p     = thr_data->reg_globinp_p;
    int     time_steps              = thr_data->time_steps;
    int     BOLD_TR                 = thr_data->BOLD_TR;
    float   model_dt                = thr_data->model_dt;
    int     BOLD_ts_len             = thr_data->BOLD_ts_len;

    /* Parallelism: divide work among threads */
    int nodes_vec_mt        = divRoundUp(nodes_vec, n_threads);
    int nodes_mt            = nodes_vec_mt  * vectorization_grade;

    int start_nodes_vec_mt  = t_id         * nodes_vec_mt;
    int start_nodes_mt      = t_id         * nodes_mt;
    int end_nodes_vec_mt    = (t_id + 1)   * nodes_vec_mt;
    int end_nodes_mt        = (t_id + 1)   * nodes_mt;
    int end_nodes_mt_glob   = end_nodes_mt; // end_nodes_mt_glob may differ from end_nodes_mt in the last thread (fake-nodes vs nodes)

    /* Correct settings for last thread */
    if (end_nodes_mt > nodes) {
        end_nodes_vec_mt    = nodes_vec;
        end_nodes_mt        = fake_nodes;
        end_nodes_mt_glob   = nodes;
        nodes_mt            = end_nodes_mt - start_nodes_mt;
        nodes_vec_mt        = end_nodes_vec_mt - start_nodes_vec_mt;
    }

    printf("thread %d: start: %d end: %d size: %d\n", thr_data->tid, start_nodes_mt, end_nodes_mt, nodes_mt);

    /* Check whether splitting makes sense (i.e. each thread has something to do). Terminate if not. */
    if (nodes_vec_mt <= 1) {
        printf("The splitting is ineffective (e.g. fewer nodes than threads or some threads have nothing to do). Use different numbers of threads. Terminating.\n");
        exit(0);
    }

    /* Set up barriers */
    if (t_id == 0) {
        initialize_thread_barriers(n_threads);
    }
    pthread_barrier_wait(&mybarrier_base);

    /* Initialize random number generator */
    //const gsl_rng_type * T;
    gsl_rng *r = gsl_rng_alloc (gsl_rng_mt19937);
    gsl_rng_set (r, (t_id+rand_num_seed)); // Random number seed -- otherwise every node gets the same random nums
    srand((unsigned)(t_id+rand_num_seed));

    // All threads
    float   *meanFR                 = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // summation array for mean firing rate over all threads
    float   *meanFR_INH             = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // summation array for mean firing rate over all threads
    float   *global_input           = (float *)_mm_malloc(nodes_mt * sizeof(float),16);
    float   *global_input_FFI       = (float *)_mm_malloc(nodes_mt * sizeof(float),16);
    float   *S_i_E                  = (float *)_mm_malloc(nodes_mt * sizeof(float),16);
    float   *S_i_I                  = (float *)_mm_malloc(nodes_mt * sizeof(float),16);
    //float   *r_i_E                    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);
    //float   *r_i_I                    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);
    float   *J_i_local              = (float *)_mm_malloc(nodes_mt * sizeof(float),16);

    if (meanFR == NULL || meanFR_INH == NULL || global_input == NULL || global_input_FFI == NULL || S_i_E==NULL || S_i_I==NULL || J_i_local==NULL) {
        printf( "ERROR: Running out of memory. Aborting... \n");
        exit(0);
    }

    // Cast to vector/SSE arrays
    __m128          *_meanFR         = (__m128*)meanFR;
    __m128          *_meanFR_INH     = (__m128*)meanFR_INH;
    __m128          *_global_input   = (__m128*)global_input;
    __m128          *_global_input_FFI = (__m128*)global_input_FFI;
    __m128          *_S_i_E          = (__m128*)S_i_E;
    __m128          *_S_i_I          = (__m128*)S_i_I;
    //__m128          *_r_i_E             = (__m128*)r_i_E;
    //__m128          *_r_i_I             = (__m128*)r_i_I;
    __m128          *_J_i_local      = (__m128*)J_i_local;

    //Balloon-Windkessel model parameters / arrays
    float rho = 0.34, alpha = 0.32, tau = 0.98, y = 1.0/0.41, kappa = 1.0/0.65;
    float V_0 = 0.02, k1 = 7 * rho, k2 = 2.0, k3 = 2 * rho - 0.2, ialpha = 1.0/alpha, itau = 1.0/tau, oneminrho = (1.0 - rho);
    float f_tmp;
    float *BOLD       = (float *)_mm_malloc(nodes_mt * BOLD_ts_len * sizeof(float),16);  // resulting BOLD data
    float *bw_x_ex    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 1 of BW-model (exc. pop.)
    float *bw_f_ex    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 2 of BW-model (exc. pop.)
    float *bw_nu_ex   = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 3 of BW-model (exc. pop.)
    float *bw_q_ex    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 4 of BW-model (exc. pop.)
    //float *bw_x_in    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 1 of BW-model (inh. pop.)
    //float *bw_f_in    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 2 of BW-model (inh. pop.)
    //float *bw_nu_in   = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 3 of BW-model (inh. pop.)
    //float *bw_q_in    = (float *)_mm_malloc(nodes_mt * sizeof(float),16);  // State-variable 4 of BW-model (inh. pop.)
    //if (bw_x_ex == NULL || bw_f_ex == NULL || bw_nu_ex==NULL || bw_q_ex==NULL || bw_x_in==NULL || bw_f_in==NULL || bw_nu_in==NULL || bw_q_in==NULL) {}
    if (bw_x_ex == NULL || bw_f_ex == NULL || bw_nu_ex==NULL || bw_q_ex==NULL) {
        printf( "ERROR: Running out of memory. Aborting... \n");
        exit(0);
    }

    // Reset arrays and variables
    ring_buf_pos     = 0;
    for (j = 0; j < nodes_mt; j++) {
        meanFR[j]           = 0.0;
        meanFR_INH[j]       = 0.0;
        global_input[j]     = 0.00;
        global_input_FFI[j] = 0.00;
        S_i_E[j]            = 0.00;
        S_i_I[j]            = 0.00;
        J_i_local[j]        = J_i[j+start_nodes_mt];
    }
    if (t_id == 0) {
        for (j=0; j<reg_act_size; j++) {
            region_activity[j]=0.00;
        }
    }

    // Reset Balloon-Windkessel model parameters and arrays
    for (j = 0; j < nodes_mt; j++) {
        bw_x_ex[j] = 0.0;
        bw_f_ex[j] = 1.0;
        bw_nu_ex[j] = 1.0;
        bw_q_ex[j] = 1.0;
        //bw_x_in[j] = 0.0;
        //bw_f_in[j] = 1.0;
        //bw_nu_in[j] = 1.0;
        //bw_q_in[j] = 1.0;
    }

    /* Barrier: only start simulating when arrays were cleared */
    pthread_barrier_wait(&mybarrier1);

    /*  Simulation loop */
    int ts_bold_i = 0;
    for (ts = 0; ts < time_steps; ts++) {

        if (t_id == 0){
            printf("%.1f %% \r", ((float)ts / (float)time_steps) * 100.0f );
        }

        // dt = 0.1 ms => 10 integration iterations per ms
        for (int_i = 0; int_i < 10; int_i++) {
            /* Barrier: only start integrating input when all other threads are finished copying their results to buffer */
            pthread_barrier_wait(&mybarrier2);

            /* Compute global coupling */
            i_node_vec_local = 0;
            for(j=start_nodes_mt; j<end_nodes_mt_glob; j++){
                tmpglobinput     = 0;
                tmpglobinput_FFI = 0;
                for (k=0; k<n_conn_table[j]; k++) {
                    tmpglobinput     += *reg_globinp_p[j+ring_buf_pos].Xi_elems[k] * SC_cap[j].cap[k];
                    //tmpglobinput_FFI += *reg_globinp_p[j+ring_buf_pos].Xi_elems[k] * SC_cap_FFI[j].cap[k];
                }
                global_input[i_node_vec_local]     = tmpglobinput;
                //global_input_FFI[i_node_vec_local] = tmpglobinput_FFI;

                i_node_vec_local++;
            }

            i_node_vec_local = 0;
            for (i_node_vec = start_nodes_vec_mt; i_node_vec < end_nodes_vec_mt; i_node_vec++) {

                // Excitatory population firing rate
                _tmp_I_E    = _mm_sub_ps(_mm_mul_ps(_a_E,_mm_add_ps(_mm_add_ps(_w_E__I_0,_mm_mul_ps(_w_plus_J_NMDA, _S_i_E[i_node_vec_local])),_mm_sub_ps(_global_input[i_node_vec_local],_mm_mul_ps(_J_i_local[i_node_vec_local], _S_i_I[i_node_vec_local])))),_b_E);

                *_tmp_exp_E     = _mm_mul_ps(_min_d_E, _tmp_I_E);
                tmp_exp_E[0]    = tmp_exp_E[0] != 0 ? expf(tmp_exp_E[0]) : 0.9;
                tmp_exp_E[1]    = tmp_exp_E[1] != 0 ? expf(tmp_exp_E[1]) : 0.9;
                tmp_exp_E[2]    = tmp_exp_E[2] != 0 ? expf(tmp_exp_E[2]) : 0.9;
                tmp_exp_E[3]    = tmp_exp_E[3] != 0 ? expf(tmp_exp_E[3]) : 0.9;
                _tmp_H_E        = _mm_div_ps(_tmp_I_E, _mm_sub_ps(_one, *_tmp_exp_E));
                _meanFR[i_node_vec_local] = _mm_add_ps(_meanFR[i_node_vec_local],_tmp_H_E);
                //_r_i_E[i_node_vec_local]  = _tmp_H_E;

                // Inhibitory population firing rate
                _tmp_I_I = _mm_sub_ps(_mm_mul_ps(_a_I,_mm_sub_ps(_mm_add_ps(_mm_add_ps(_w_I__I_0,_global_input_FFI[i_node_vec_local]),_mm_mul_ps(_J_NMDA, _S_i_E[i_node_vec_local])), _S_i_I[i_node_vec_local])),_b_I);
                *_tmp_exp_I   = _mm_mul_ps(_min_d_I, _tmp_I_I);
                tmp_exp_I[0]  = tmp_exp_I[0] != 0 ? expf(tmp_exp_I[0]) : 0.9;
                tmp_exp_I[1]  = tmp_exp_I[1] != 0 ? expf(tmp_exp_I[1]) : 0.9;
                tmp_exp_I[2]  = tmp_exp_I[2] != 0 ? expf(tmp_exp_I[2]) : 0.9;
                tmp_exp_I[3]  = tmp_exp_I[3] != 0 ? expf(tmp_exp_I[3]) : 0.9;
                _tmp_H_I  = _mm_div_ps(_tmp_I_I, _mm_sub_ps(_one, *_tmp_exp_I));
                _meanFR_INH[i_node_vec_local] = _mm_add_ps(_meanFR_INH[i_node_vec_local],_tmp_H_I);
                //_r_i_I[i_node_vec_local] = _tmp_H_I;

                //gaussrand(rand_number);
                rand_number[0] = (float)gsl_ran_gaussian(r, 1.0);
                rand_number[1] = (float)gsl_ran_gaussian(r, 1.0);
                rand_number[2] = (float)gsl_ran_gaussian(r, 1.0);
                rand_number[3] = (float)gsl_ran_gaussian(r, 1.0);

                _S_i_I[i_node_vec_local] = _mm_add_ps(_mm_add_ps(_mm_mul_ps(_sigma_sqrt_dt, *_rand_number),_S_i_I[i_node_vec_local]),_mm_mul_ps(_dt,_mm_add_ps(_mm_mul_ps(_imintau_I, _S_i_I[i_node_vec_local]),_mm_mul_ps(_tmp_H_I,_gamma_I))));

                //gaussrand(rand_number);
                rand_number[0] = (float)gsl_ran_gaussian(r, 1.0);
                rand_number[1] = (float)gsl_ran_gaussian(r, 1.0);
                rand_number[2] = (float)gsl_ran_gaussian(r, 1.0);
                rand_number[3] = (float)gsl_ran_gaussian(r, 1.0);

                _S_i_E[i_node_vec_local] = _mm_add_ps(_mm_add_ps(_mm_mul_ps(_sigma_sqrt_dt, *_rand_number),_S_i_E[i_node_vec_local]),_mm_mul_ps(_dt, _mm_add_ps(_mm_mul_ps(_imintau_E, _S_i_E[i_node_vec_local]),_mm_mul_ps(_mm_mul_ps(_mm_sub_ps(_one, _S_i_E[i_node_vec_local]),_gamma),_tmp_H_E))));

                i_node_vec_local++;
            }

            /* ensure that synaptic activity is within its boundaries */
            for(j=0; j<nodes_mt; j++){
                S_i_E[j] = S_i_E[j] >= 0.0 ? S_i_E[j] : 0.0;
                S_i_E[j] = S_i_E[j] <= 1.0 ? S_i_E[j] : 1.0;
                S_i_I[j] = S_i_I[j] >= 0.0 ? S_i_I[j] : 0.0;
                S_i_I[j] = S_i_I[j] <= 1.0 ? S_i_I[j] : 1.0;
            }

            /* Copy result of this thread into global region_activity buffer */
            memcpy(&region_activity[ring_buf_pos+start_nodes_mt], S_i_E, nodes_mt*sizeof( float ));

            /* Shift region_activity ring-buffer start */
            ring_buf_pos = ring_buf_pos<(reg_act_size-nodes) ? (ring_buf_pos+nodes) : 0;

        }

        /*
         Compute BOLD for that time-step (subsampled to 1 ms)
         */
        for (j = 0; j < nodes_mt; j++) {
            bw_x_ex[j]  = bw_x_ex[j]  +  model_dt * (S_i_E[j] - kappa * bw_x_ex[j] - y * (bw_f_ex[j] - 1.0));
            f_tmp       = bw_f_ex[j]  +  model_dt * bw_x_ex[j];
            bw_nu_ex[j] = bw_nu_ex[j] +  model_dt * itau * (bw_f_ex[j] - powf(bw_nu_ex[j], ialpha));
            bw_q_ex[j]  = bw_q_ex[j]  +  model_dt * itau * (bw_f_ex[j] * (1.0 - powf(oneminrho,(1.0/bw_f_ex[j]))) / rho  - powf(bw_nu_ex[j],ialpha) * bw_q_ex[j] / bw_nu_ex[j]);
            bw_f_ex[j]  = f_tmp;

            /*
             bw_x_in[j]  = bw_x_in[j]  +  model_dt * (S_i_I[j] - kappa * bw_x_in[j] - y * (bw_f_in[j] - 1.0));
             f_tmp       = bw_f_in[j]  +  model_dt * bw_x_in[j];
             bw_nu_in[j] = bw_nu_in[j] +  model_dt * itau * (bw_f_in[j] - powf(bw_nu_in[j], ialpha));
             bw_q_in[j]  = bw_q_in[j]  +  model_dt * itau * (bw_f_in[j] * (1.0 - powf(oneminrho,(1.0/bw_f_in[j]))) / rho  - powf(bw_nu_in[j],ialpha) * bw_q_in[j] / bw_nu_in[j]);
             bw_f_in[j]  = f_tmp;
             */
        }

        if (ts % BOLD_TR == 0) {
            for (j = 0; j < nodes_mt; j++) {
                BOLD[ts_bold_i + j * BOLD_ts_len] = 100 / rho * V_0 * (k1 * (1 - bw_q_ex[j]) + k2 * (1 - bw_q_ex[j]/bw_nu_ex[j]) + k3 * (1 - bw_nu_ex[j]));
            }
            ts_bold_i++;
        }
    }

    /* Copy results of this thread into shared memory buffer */
    memcpy(&BOLD_ex[start_nodes_mt*BOLD_ts_len], BOLD, nodes_mt * BOLD_ts_len * sizeof( float ));

    /* Wait until other threads finished before writing out result */
    pthread_barrier_wait(&mybarrier3);

    /* Write out fMRI time series */
    if (t_id == 0){
        FILE *FCout = fopen(thr_data->output_file, "w");
        for (j = 0; j < nodes; j++) {
            for (k = 0; k < ts_bold_i; k++) {
                fprintf(FCout, "%.5f ",BOLD_ex[j*BOLD_ts_len + k]);
            }
            fprintf(FCout, "\n");
        }
        fclose(FCout);
    }

    gsl_rng_free (r);
    pthread_exit(NULL);
}

/*
 Usage: tvbii <paramfile> <subject_id>
 */

int main(int argc, char *argv[])
{
    /*
     Check whether right number of arguments was passed to program call
     */
    if (argc != 4 || atoi(argv[3]) <= 0) {
        printf("\nERROR: Invalid arguments.\n\nUsage: tvbii <paramset_file> <sub_id> <#threads>\n\nTerminating... \n\n");
        printf("\nProvided arguments:\n");
        int i;
        for (i=0; i<argc; i++) {
            printf("%s\n", argv[i]);
        }
        exit(0);
    }

    /*
     Get current time and do some initializations
     */
    time_t  start = time(NULL);
    int     i, j;
    int     n_threads = atoi(argv[3]);

    /*
     Global model and integration parameters
     */
    const float dt                  = 0.1;      // Integration step length dt = 0.1 ms
    const float sqrt_dt             = sqrtf(dt);// Noise in the diffusion part scaled by sqrt of dt
    const float model_dt            = 0.001;    // Time-step of model (sampling-rate=1000 Hz)
    const int   vectorization_grade = 4;        // How many operations can be done simultaneously. Depends on CPU Architecture and available intrinsics.
    int         time_steps          = 667*1.94*1000;    // Simulation length
    int         nodes               = 84;    // Number of surface vertices; must be a multiple of vectorization grade
    int         fake_nodes          = 84;
    float       global_trans_v      = 1.0;     // Global transmission velocity (m/s); Local time-delays can be ommited since smaller than integration time-step
    float       G                   = 0.5;        // Global coupling strength
    int         BOLD_TR             = 1940;     // TR of BOLD data
    int         rand_num_seed       = 1403;

    /*
     Local model: DMF-Parameters from Deco et al. JNeuro 2014
     */
    float w_plus  = 1.4;          // local excitatory recurrence
    //float I_ext   = 0;            // External stimulation
    float J_NMDA  = 0.15;         // (nA) excitatory synaptic coupling
    //float J_i     = 1.0;          // 1 for no-FIC, !=1 for Feedback Inhibition Control
    const float a_E     = 310;          // (n/C)
    const float b_E     = 125;          // (Hz)
    const float d_E     = 0.16;         // (s)
    const float a_I     = 615;          // (n/C)
    const float b_I     = 177;          // (Hz)
    const float d_I     = 0.087;        // (s)
    const float gamma   = 0.641/1000.0; // factor 1000 for expressing everything in ms
    const float tau_E   = 100;          // (ms) Time constant of NMDA (excitatory)
    const float tau_I   = 10;           // (ms) Time constant of GABA (inhibitory)
    float       sigma   = 0.00316228;   // (nA) Noise amplitude
    const float I_0     = 0.382;        // (nA) overall effective external input
    const float w_E     = 1.0;          // scaling of external input for excitatory pool
    const float w_I     = 0.7;          // scaling of external input for inhibitory pool
    const float gamma_I = 1.0/1000.0;   // for expressing inhib. pop. in ms
    float       tmpJi   = 1.0;          // Feedback inhibition J_i

    /* Read parameters from input file. Input file is a simple text file that contains one line with parameters and white spaces in between. */
    FILE *file;
    char param_file[300];memset(param_file, 0, 300*sizeof(char));
    snprintf(param_file, sizeof(param_file), "/input/%s",argv[1]);
    file=fopen(param_file, "r");
    if (file==NULL){
        printf( "\nERROR: Could not open file %s. Terminating... \n\n", param_file);
        exit(0);
    }
    if(fscanf(file,"%d",&nodes) != EOF && fscanf(file,"%f",&G) != EOF && fscanf(file,"%f",&J_NMDA) != EOF && fscanf(file,"%f",&w_plus) != EOF && fscanf(file,"%f",&tmpJi) != EOF && fscanf(file,"%f",&sigma) != EOF && fscanf(file,"%d",&time_steps) != EOF && fscanf(file,"%d",&BOLD_TR) != EOF && fscanf(file,"%f",&global_trans_v) != EOF && fscanf(file,"%d",&rand_num_seed) != EOF){
    } else{
        printf( "\nERROR: Unexpected end-of-file in file %s. File contains less input than expected. Terminating... \n\n", param_file);
        exit(0);
    }
    fclose(file);

    char output_file[300];memset(output_file, 0, 300*sizeof(char));
    snprintf(output_file, sizeof(output_file), "/output/%s_%s_fMRI.txt",argv[2],argv[1]);

    /* Add fake regions to make region count a multiple of vectorization grade */
    if (nodes % vectorization_grade != 0){
        printf( "\nWarning: Specified number of nodes (%d) is not a multiple of vectorization degree (%d). Will add some fake nodes... \n\n", nodes, vectorization_grade);

        int remainder   = nodes%vectorization_grade;
        if (remainder > 0) {
            fake_nodes = nodes + (vectorization_grade - remainder);
        }
    } else{
        fake_nodes = nodes;
    }

    /* Initialize random number generator */
    srand((unsigned)rand_num_seed);

    /* Derived parameters */
    const float sigma_sqrt_dt = sqrt_dt * sigma;
    const int   nodes_vec     = fake_nodes/vectorization_grade;
    const float min_d_E       = -1.0 * d_E;
    const float min_d_I       = -1.0 * d_I;
    const float imintau_E     = -1.0 / tau_E;
    const float imintau_I     = -1.0 / tau_I;
    const float w_E__I_0      = w_E * I_0;
    const float w_I__I_0      = w_I * I_0;
    const float one           = 1.0;
    const float w_plus__J_NMDA= w_plus * J_NMDA;
    const float G_J_NMDA      = G * J_NMDA;
          float TR            = (float)BOLD_TR / 1000;   // (s) TR of fMRI data
          int   BOLD_ts_len   = time_steps / (TR / model_dt) + 1;  // Length of BOLD time-series written to HDD

    /*
     Import and setup global and local connectivity
     */
    int         *n_conn_table;
    float       *region_activity, *SC_rowsums;
    struct Xi_p *reg_globinp_p;
    struct SC_capS      *SC_cap;
    struct SC_inpregS   *SC_inpreg;
    char cap_file[300];memset(cap_file, 0, 300*sizeof(char));
    snprintf(cap_file, sizeof(cap_file), "/input/%s_SC_weights.txt",argv[2]);
    char dist_file[300];memset(dist_file, 0, 300*sizeof(char));
    snprintf(dist_file, sizeof(dist_file), "/input/%s_SC_distances.txt",argv[2]);
    char reg_file[300];memset(reg_file, 0, 300*sizeof(char));
    snprintf(reg_file, sizeof(reg_file), "/input/%s_SC_regionids.txt",argv[2]);

    int         maxdelay = importGlobalConnectivity(cap_file, dist_file, reg_file, nodes, &region_activity, &reg_globinp_p, global_trans_v, &n_conn_table, G_J_NMDA, &SC_cap, &SC_rowsums, &SC_inpreg);

    int         reg_act_size = nodes * maxdelay;

    /*
     Initialize and/or cast to vector-intrinsics types for variables & parameters
     */
    float *J_i      = (float *)_mm_malloc(fake_nodes * sizeof(float),16); // (nA) inhibitory synaptic coupling
    float *BOLD_ex  = (float *)_mm_malloc(nodes * BOLD_ts_len * sizeof(float),16);

    if (J_i == NULL || BOLD_ex == NULL) {
        printf( "ERROR: Running out of memory. Aborting... \n");
        _mm_free(J_i);_mm_free(BOLD_ex);
        return 1;
    }
    // Initialize state variables / parameters
    for (j = 0; j < fake_nodes; j++) {
        J_i[j]              = tmpJi;
    }
    const __m128    _dt                 = _mm_load1_ps(&dt);
    const __m128    _sigma_sqrt_dt      = _mm_load1_ps(&sigma_sqrt_dt);
    const __m128    _w_plus_J_NMDA      = _mm_load1_ps(&w_plus__J_NMDA);
    const __m128    _a_E                = _mm_load1_ps(&a_E);
    const __m128    _b_E                = _mm_load1_ps(&b_E);
    const __m128    _min_d_E            = _mm_load1_ps(&min_d_E);
    const __m128    _a_I                = _mm_load1_ps(&a_I);
    const __m128    _b_I                = _mm_load1_ps(&b_I);
    const __m128    _min_d_I            = _mm_load1_ps(&min_d_I);
    const __m128    _gamma              = _mm_load1_ps(&gamma);
    const __m128    _gamma_I            = _mm_load1_ps(&gamma_I);
    const __m128    _imintau_E          = _mm_load1_ps(&imintau_E);
    const __m128    _imintau_I          = _mm_load1_ps(&imintau_I);
    const __m128    _w_E__I_0           = _mm_load1_ps(&w_E__I_0);
    const __m128    _w_I__I_0           = _mm_load1_ps(&w_I__I_0);
    float           tmp_sigma           = sigma*dt;// pre-compute dt*sigma for the integration of sigma*randnumber in equations (9) and (10) of Deco2014
    const __m128    _sigma              = _mm_load1_ps(&tmp_sigma);
    const __m128    _one                = _mm_load1_ps(&one);
    const __m128    _J_NMDA             = _mm_load1_ps(&J_NMDA);

    /*
     Start multithread simulation
     */

    /* Create threads */
    pthread_barrier_init(&mybarrier_base, NULL, n_threads);
    pthread_t       thr[n_threads];
    thread_data_t   thr_data[n_threads];
    int rc;
    for (i = 0; i < n_threads; ++i) {
        thr_data[i].tid             =	i               ;
        thr_data[i].n_threads       =	n_threads       ;
        thr_data[i].vectorization_grade = vectorization_grade;
        thr_data[i].nodes			=	nodes			;
        thr_data[i].fake_nodes      =   fake_nodes      ;
        thr_data[i].J_i             =	J_i            	;
        thr_data[i].reg_act_size	=	reg_act_size	;
        thr_data[i].region_activity	=	region_activity	;
        thr_data[i]._gamma			=	_gamma			;
        thr_data[i]._one			=	_one			;
        thr_data[i]._imintau_E		=	_imintau_E		;
        thr_data[i]._dt             =	_dt             ;
        thr_data[i]._sigma_sqrt_dt  =   _sigma_sqrt_dt  ;
        thr_data[i]._sigma			=	_sigma			;
        thr_data[i]._gamma_I		=	_gamma_I		;
        thr_data[i]._imintau_I		=	_imintau_I		;
        thr_data[i]._min_d_I		=	_min_d_I		;
        thr_data[i]._b_I			=	_b_I			;
        thr_data[i]._J_NMDA         =	_J_NMDA         ;
        thr_data[i]._w_I__I_0		=	_w_I__I_0		;
        thr_data[i]._a_I			=	_a_I			;
        thr_data[i]._min_d_E		=	_min_d_E		;
        thr_data[i]._b_E			=	_b_E			;
        thr_data[i]._a_E			=	_a_E			;
        thr_data[i]._w_plus_J_NMDA	=	_w_plus_J_NMDA	;
        thr_data[i]._w_E__I_0		=	_w_E__I_0		;
        thr_data[i].nodes_vec		=	nodes_vec		;
        thr_data[i].SC_cap			=	SC_cap			;
        thr_data[i].SC_inpreg       =	SC_inpreg       ;
        thr_data[i].reg_globinp_p	=	reg_globinp_p	;
        thr_data[i].n_conn_table	=	n_conn_table	;
        thr_data[i].time_steps      =   time_steps      ;
        thr_data[i].BOLD_TR         =   BOLD_TR         ;
        thr_data[i].model_dt        =   model_dt        ;
        thr_data[i].BOLD_ex         =   BOLD_ex         ;
        thr_data[i].rand_num_seed   =   rand_num_seed   ;
        thr_data[i].BOLD_ts_len     =   BOLD_ts_len     ;
        thr_data[i].output_file     =   output_file     ;

        if ((rc = pthread_create(&thr[i], NULL, run_simulation, &thr_data[i]))) {
            fprintf(stderr, "error: pthread_create, rc: %d\n", rc);
            return EXIT_FAILURE;
        }
    }

    /* block until all threads complete */
    for (i = 0; i < n_threads; ++i) {
        pthread_join(thr[i], NULL);
    }
    printf("Threads finished. Back to main thread.\n");

    printf("MT-TVBii finished. Execution took %.2f s for %d nodes. Goodbye!\n", (float)(time(NULL) - start), nodes);

    return 0;
}