/* BRAINS
 * (B)LR (R)everberation-mapping (A)nalysis (I)n AGNs with (N)ested (S)ampling
 * Yan-Rong Li, liyanrong@ihep.ac.cn
 * Thu, Aug 4, 2016
 */

/*!
 * \file sim.c
 * \brief generate mocked 2d data.
 */
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stddef.h>
#include <math.h>
#include <float.h>
#include <string.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
#include <gsl/gsl_interp.h>
#include <mpi.h>

#include "brains.h"

/////////////////////////////////////////////////////////////////////////////////
#ifdef SpecAstro
/* baselines for 3C 273 dataset observed by the GRAVITY (Nature, 2020, 563, 657),
 * unit is meter */
int n_base_sa_3c273 = 24;
double base_sa_3c273[]={
  -39.847287,   18.757261,
  -72.901618,  -13.864206,
  -105.767452, -59.897942,
  -32.453674,  -32.669331,
  -64.714063,  -78.769286,
  -32.064961,  -46.108491,
  -53.774627,   21.045077,
  -71.062525,  -10.421538,
  -79.670158,  -55.520144,
  -16.867270,  -31.447448,
  -25.151512,  -76.555774,
   -8.846202,  -45.116789,
  -54.064283,   19.430318,
  -86.349699,  -12.827258,
  -115.711019, -58.525511,
  -32.484928,  -32.180585,
  -61.579109,  -77.881063,
  -29.231494,  -45.704607,
  -52.779207,   21.321411,
  -83.468788,  -12.802497,
  -107.827509, -59.908816,
  -28.601448,  -33.449429,
  -50.961674,  -81.626333,
  -23.699820,  -46.979455
};
#endif
//////////////////////////////////////////////////////////////////////////////////////

void *model;

void sim()
{
  if(thistask != roottask)
    return;

  FILE *fp;
  char fname[200];
  int i, j, incr;

  sim_init();

  double *pm = (double *)model, error, fcon;

  smooth_init(parset.n_vel_recon, TransV);

  if(parset.flag_dim == -1)
  {
    /* note that here use sigma_hat = sigma/sqrt(tau) */
    printf("sim with ln(sigma) = %f and  ln(taud) = %f.\n", var_param[1], var_param[2]);
    reconstruct_con_from_varmodel(exp(var_param[1]), exp(var_param[2]), 1.0, 0.0);
  }
  else
  {
    con_scale = 1.0;
    line_scale = 1.0;
    line_error_mean = con_error_mean = 0.01;
    /* arguments: sigma_hat, tau, alapha, and syserr */
    create_con_from_random(0.03, 45.0, 1.0, 0.0);
  }
  calculate_con_rm(model);

  sprintf(fname, "%s/%s", parset.file_dir, "/data/sim_con_full.txt");
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s\n", fname);
    exit(-1);
  }

  for(i=0; i<parset.n_con_recon; i++)
  {
    fprintf(fp, "%e %e %e\n", Tcon[i]*(1.0+parset.redshift), Fcon[i]/con_scale, Fcerrs[i]/con_scale);
  }
  fclose(fp);

  sprintf(fname, "%s/%s", parset.file_dir, "/data/sim_con.txt");
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s\n", fname);
    exit(-1);
  }

  if(parset.flag_dim != -2)
  {
    gsl_interp_init(gsl_linear, Tcon, Fcon, parset.n_con_recon);
    for(i=0; i<n_con_data; i++)
    {
      //fprintf(fp, "%f %f %f\n", Tcon[i], Fcon[i]/con_scale, Fcerrs[i]/con_scale);
      fcon = gsl_interp_eval(gsl_linear, Tcon, Fcon, Tcon_data[i], gsl_acc);
      fprintf(fp, "%e %e %e\n", Tcon_data[i]*(1.0+parset.redshift),
            (fcon+gsl_ran_ugaussian(gsl_r)*con_error_mean)/con_scale, con_error_mean/con_scale);
    }
  }
  else
  {
    incr = fmax(0.5/(Tcon[1]-Tcon[0]), 1.0); //cadence to be 0.5day or increasement to be 1
    for(i=0; i<parset.n_con_recon; i+=incr)
    {
      if(Tcon[i] >= 0.0)
      {
        fprintf(fp, "%e %e %e\n", Tcon[i]*(1.0+parset.redshift), Fcon[i]/con_scale, Fcerrs[i]/con_scale);
      }
    }
  }
  fclose(fp);

  gsl_interp_init(gsl_linear, Tcon, Fcon_rm, parset.n_con_recon);
  transfun_1d_cal(model, 0);
  calculate_line_from_blrmodel(model, Tline, Fline, parset.n_line_recon);

  sprintf(fname, "%s/%s", parset.file_dir, "/data/sim_line.txt");
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s\n", fname);
    exit(-1);
  }

  if(parset.flag_dim == -1)
  {
    error = line_error_mean * sqrt(n_line_data) * (Vline_data[1] - Vline_data[0]);
  }
  else
  {
    error = line_error_mean;
  }
  for(i=0; i<parset.n_line_recon; i++)
  {
    fprintf(fp, "%e %e %e\n", Tline[i]*(1.0+parset.redshift), Fline[i]/line_scale + gsl_ran_ugaussian(gsl_r)*error/line_scale,
      error/line_scale);
  }

  // output transfer function.
  sprintf(fname, "%s/%s", parset.file_dir, parset.tran_out_file);
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s\n", fname);
    exit(-1);
  }

  for(i=0; i<parset.n_tau; i++)
  {
    fprintf(fp, "%e %e\n", TransTau[i], Trans1D[i]);
  }
  fclose(fp);

  transfun_2d_cal(model, TransV, Trans2D, parset.n_vel_recon, 1);
  calculate_line2d_from_blrmodel(model, Tline, TransV,
          Trans2D, Fline2d, parset.n_line_recon, parset.n_vel_recon);

  sprintf(fname, "%s/%s", parset.file_dir, "/data/sim_line2d.txt");
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s\n", fname);
    exit(-1);
  }

  fprintf(fp, "# %d %d\n", parset.n_line_recon, parset.n_vel_recon);
  for(i=0; i<parset.n_line_recon; i++)
  {
    fprintf(fp, "# %f\n", Tline[i]*(1.0+parset.redshift));
    for(j=0; j<parset.n_vel_recon; j++)
    {
      fprintf(fp, "%e %e %e\n", TransW[j],
        (Fline2d[i*parset.n_vel_recon + j] + gsl_ran_ugaussian(gsl_r)*line_error_mean*0.3)/line_scale, line_error_mean/line_scale);
    }

    fprintf(fp, "\n");
  }
  fclose(fp);

  sprintf(fname, "%s/%s", parset.file_dir, "/data/sim_broadening.txt");
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s\n", fname);
    exit(-1);
  }

  for(i=0; i<parset.n_line_recon; i++)
  {
    fprintf(fp, "%f %f\n", parset.InstRes * VelUnit, parset.InstRes_err * VelUnit);
  }
  fclose(fp);

  // output 2d transfer function
  sprintf(fname, "%s/%s", parset.file_dir, parset.tran2d_out_file);
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s\n", fname);
    exit(-1);
  }
  fprintf(fp, "# %d %d\n", parset.n_tau, parset.n_vel_recon);
  for(i=0; i<parset.n_tau; i++)
  {
    for(j=0; j<parset.n_vel_recon; j++)
    {
      fprintf(fp, "%e %e %e\n", TransV[j]*VelUnit, TransTau[i], Trans2D[i*parset.n_vel_recon + j]);
    }

    fprintf(fp, "\n");
  }
  fclose(fp);

#ifdef SpecAstro
  double *sa_pm;
  sa_pm = (double *)pm + num_params_blr;

  sa_smooth_init(parset.n_sa_vel_recon, vel_sa, parset.sa_InstRes);

  gen_sa_cloud_sample((void *)sa_pm, 3, 0);
  calculate_sa_sim_with_sample(pm, vel_sa, parset.n_sa_vel_recon, base_sa, parset.n_sa_base_recon,
                                   phase_sa, Fline_sa);

  sprintf(fname, "%s/%s", parset.file_dir, "data/sim_sa.txt");
  fp = fopen(fname, "w");
  if(fp == NULL)
  {
    fprintf(stderr, "# Error: Cannot open file %s.\n", fname);
    exit(0);
  }
  // output sa line
  fprintf(fp, "# %d %d %d\n", 1, parset.n_sa_vel_recon, parset.n_sa_base_recon);
  for(j=0; j<parset.n_sa_vel_recon; j++)
  {
    fprintf(fp, "%e %e %e\n", wave_sa[j], Fline_sa[j] + gsl_ran_ugaussian(gsl_r)*sa_line_error_mean, sa_line_error_mean);
  }
  fprintf(fp, "\n");
  for(i=0; i<parset.n_sa_base_recon; i++)
  {
    fprintf(fp, "# %f %f\n", base_sa[i*2], base_sa[i*2+1]);
    for(j=0; j<parset.n_sa_vel_recon; j++)
    {
      fprintf(fp, "%e %e %e\n", wave_sa[j],
       phase_sa[i*parset.n_sa_vel_recon + j]/(PhaseFactor * wave_sa[j]) + gsl_ran_ugaussian(gsl_r)*sa_phase_error_mean,
       sa_phase_error_mean);
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
  sa_smooth_end();

#endif

  smooth_end();
  sim_end();
}

void sim_init()
{
  int i, j, idx;
  double dT, Tspan;
  double *pm, Rblr, mbh;

  switch(parset.flag_blrmodel)
  {
    case -1:
      num_params_blr_model = num_params_MyTransfun2d;
      transfun_1d_cal = transfun_1d_cal_mytransfun;
      transfun_2d_cal = transfun_2d_cal_mytransfun;
      break;

    case 0:
      num_params_blr_model = num_params_MyBLRmodel2d;
      gen_cloud_sample = gen_cloud_sample_mymodel;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;

    case 1:
      num_params_blr_model = sizeof(BLRmodel1)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model1;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;
    case 2:
      num_params_blr_model = sizeof(BLRmodel2)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model2;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;
    case 3:
      num_params_blr_model = sizeof(BLRmodel3)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model3;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;
    case 4:
      num_params_blr_model = sizeof(BLRmodel4)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model4;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;
    case 5:
      num_params_blr_model = sizeof(BLRmodel5)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model5;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;

    case 6:
      num_params_blr_model = sizeof(BLRmodel6)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model6;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;

    case 7:
      num_params_blr_model = sizeof(BLRmodel7)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model7;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;

    case 8:
      num_params_blr_model = sizeof(BLRmodel8)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model8;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;

    case 9:
      num_params_blr_model = sizeof(BLRmodel9)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model9;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;

    default:
      num_params_blr_model = sizeof(BLRmodel1)/sizeof(double);
      gen_cloud_sample = gen_cloud_sample_model1;
      transfun_1d_cal = transfun_1d_cal_cloud;
      transfun_2d_cal = transfun_2d_cal_cloud;
      break;
  }

#ifdef SpecAstro
  set_sa_blr_model();
  /* SA */
  num_params_sa_blr = num_params_sa_blr_model + num_params_sa_extpar;
  num_params_sa = num_params_sa_blr;

#endif

  /* use epoch-independent broadening */
  if(parset.flag_InstRes > 1)
  {
    num_params_res = 1;
    parset.InstRes = 220.0/VelUnit;
    parset.InstRes_err = 0.0;
  }
  else
  {
    parset.InstRes /= VelUnit;
    parset.InstRes_err = 0.0;
  }

  if(parset.flag_narrowline > 1)
  {
    printf("# set flag_narrowline to 1.\n");
    parset.flag_narrowline = 1;
  }
  else if(parset.flag_narrowline == 0)
  {
    parset.width_narrowline = 0.0;
  }

  parset.flag_linecenter = 0;
  num_params_linecenter = 0;

  parset.num_particles = 1;
  which_particle_update = 0;
  force_update = 1;
  which_parameter_update = -1;

  num_params_blr = num_params_blr_model + num_params_nlr
                 + num_params_res + num_params_linecenter + 1; /* include line sys err */
  num_params_var = num_params_drw + num_params_trend + num_params_difftrend + num_params_resp;

  num_params_blr_tot = num_params_blr;
#ifdef SpecAstro
  num_params_blr_tot  += num_params_sa_blr;
#endif
  num_params = num_params_blr_tot + num_params_var + parset.n_con_recon;

  /* index of A and Ag */
  idx_resp = num_params_blr_tot + num_params_drw + num_params_trend;
  /* index of different trend */
  idx_difftrend = idx_resp + num_params_resp;

  model = malloc(num_params * sizeof(double));
  par_fix = (int *) malloc(num_params * sizeof(int));
  par_fix_val = (double *) malloc(num_params * sizeof(double));

  /* setup parameters */
  pm = (double *)model;
  if(parset.flag_blrmodel == -1)
  {
    set_par_value_mytransfun_sim(pm);
  }
  else
  {
    set_par_value_sim(pm, parset.flag_blrmodel);
    set_par_fix_blrmodel();
    for(i=0; i<num_params_blr_model; i++)
    {
      if(par_fix[i] == 1)
      {
        pm[i] = par_fix_val[i];
      }
    }
  }

  /* spectral broadening, note this is a deviation from the input value */
  pm[num_params_blr_model + num_params_nlr ] = 0.0;

  pm[idx_resp + 0] = log(1.0); //A
  pm[idx_resp + 1] = 0.0;      //Ag

#ifdef SpecAstro
  double *sa_model = pm + num_params_blr;
  set_idx_par_mutual();
  set_par_value_sim(sa_model, parset.flag_sa_blrmodel);
  set_par_fix_sa_blrmodel();
  for(i=0; i<num_params_sa_blr_model; i++)
  {
    if(par_fix[num_params_blr + i] == 1)
    {
      pm[num_params_blr + i] = par_fix_val[num_params_blr + i];
    }
  }

  /* set the same mbh and inc */
  sa_model[idx_sa_par_mutual[0]] = pm[idx_rm_par_mutual[0]]; //mbh
  sa_model[idx_sa_par_mutual[1]] = pm[idx_rm_par_mutual[1]]; //inc

  sa_model[num_params_sa_blr_model]     = log(550.0); //DA
  sa_model[num_params_sa_blr_model + 1] = 0.0;        //PA
  sa_model[num_params_sa_blr_model + 2] = 0.0;        //FA
  sa_model[num_params_sa_blr_model + 3] = 0.0;        //CO
#endif

  Fcon = malloc(parset.n_con_recon * sizeof(double));
  Fcon_rm = malloc(parset.n_con_recon * sizeof(double));

  idx = get_idx_mbh_from_blrmodel();
  mbh = exp(pm[idx]);

  if(parset.flag_blrmodel != 8) /* model 8 is particular */
  {
    Rblr = exp(pm[0]);
  }
  else
  {
    Rblr = exp(pm[9]);
  }

  if(parset.flag_dim == -1)
  {
    Tspan = Tcon_data[n_con_data -1] - Tcon_data[0];

  /* set time array for continuum */
    Tcon_min = Tcon_data[0] - time_back_set - 10.0;

    Tcon_max = fmax(Tcon_data[n_con_data-1], Tline_data[n_line_data-1]) + fmax(0.05*Tspan, 10.0);
    dT = (Tcon_max - Tcon_min)/(parset.n_con_recon -1);

    for(i=0; i<parset.n_con_recon; i++)
    {
      Tcon[i] = Tcon_min + i*dT;
    }
  }
  else
  {
    Tspan = Rblr*10.0;

    rcloud_min_set = 0.0;
    rcloud_max_set = Tspan/2.0;
    if(parset.rcloud_max > 0.0)
      rcloud_max_set = fmin(rcloud_max_set, parset.rcloud_max);
    printf("RM rcloud_min_max_set: %f %f\n", rcloud_min_set, rcloud_max_set);

    time_back_set = 2.0*rcloud_max_set;
    Tcon_min = 0.0 - time_back_set - 10.0;
    Tcon_max = Tspan + 10.0;

    dT = (Tcon_max - Tcon_min)/(parset.n_con_recon -1);

    for(i=0; i<parset.n_con_recon; i++)
    {
      Tcon[i] = Tcon_min + i*dT;
    }
  }

  /* set Larr_rec */
  for(i=0;i<parset.n_con_recon;i++)
  {
    Larr_rec[i*nq + 0]=1.0;
    for(j=1; j<nq; j++)
      Larr_rec[i*nq + j] = pow(Tcon[i], j);
  }

  if(parset.flag_dim == -1)
  {
    parset.n_line_recon = n_line_data;
    parset.n_vel_recon = n_vel_data;
  }

  TransTau = malloc(parset.n_tau * sizeof(double));
  TransV = malloc(parset.n_vel_recon * sizeof(double));
  TransW = malloc(parset.n_vel_recon * sizeof(double));
  Trans1D = malloc(parset.n_tau * sizeof(double));
  Trans2D = malloc(parset.n_tau * parset.n_vel_recon * sizeof(double));
  Tline = malloc(parset.n_line_recon * sizeof(double));
  Fline = malloc(parset.n_line_recon * sizeof(double));
  Fline2d = malloc(parset.n_line_recon * parset.n_vel_recon * sizeof(double));

  if(parset.flag_dim == -1)
  {
    memcpy(Tline, Tline_data, n_line_data * sizeof(double));
    memcpy(TransV, Vline_data, n_vel_data * sizeof(double));
    memcpy(TransW, Wline_data, n_vel_data * sizeof(double));
  }
  else
  {
    Tline_min = 0.0;
    Tline_max = Tcon_max - 1.0;

    dT = (Tline_max - Tline_min)/(parset.n_line_recon - 1);

    for(i=0; i<parset.n_line_recon; i++)
    {
      Tline[i] = Tline_min + i*dT;
    }

    double vel_max_set, vel_min_set;
    vel_max_set = sqrt(pow(2.0*sqrt(mbh/Rblr), 2.0) + pow(2.0*parset.InstRes, 2.0));
    vel_min_set = - vel_max_set;
    double dVel = (vel_max_set- vel_min_set)/(parset.n_vel_recon -1.0);

    for(i=0; i<parset.n_vel_recon; i++)
    {
      TransV[i] = vel_min_set + dVel*i;
      TransW[i] = (1.0 + TransV[i]/C_Unit) * parset.linecenter * (1.0+parset.redshift);
    }
  }

  clouds_tau = malloc(parset.n_cloud_per_task * sizeof(double));
  clouds_weight = malloc(parset.n_cloud_per_task * sizeof(double));
  clouds_vel = malloc(parset.n_cloud_per_task * parset.n_vel_per_cloud * sizeof(double));

  if(parset.flag_save_clouds && thistask == roottask)
  {
    if(parset.n_cloud_per_task <= 1000)
      icr_cloud_save = 1;
    else
      icr_cloud_save = parset.n_cloud_per_task/1000;

    char fname[200];
    sprintf(fname, "%s/%s", parset.file_dir, parset.cloud_out_file);
    fcloud_out = fopen(fname, "w");
    if(fcloud_out == NULL)
    {
      fprintf(stderr, "# Error: Cannot open file %s\n", fname);
      exit(-1);
    }
  }

#ifdef SpecAstro
  double saRblr;

  if(parset.flag_dim == -1)
  {
    parset.n_sa_vel_recon = n_vel_sa_data;
    parset.n_sa_base_recon = n_base_sa_data;
  }
  else
  {
    sa_flux_norm = 1.0;
    parset.n_sa_vel_recon = 40;
    if(parset.flag_gravity == 1)
      parset.n_sa_base_recon = n_base_sa_3c273;
    else
      parset.n_sa_base_recon = 20;

    sa_phase_error_mean = 0.01;
    sa_line_error_mean = 0.01;
  }

  vel_sa = malloc(parset.n_sa_vel_recon * sizeof(double));
  wave_sa = malloc(parset.n_sa_vel_recon * sizeof(double));
  Fline_sa = malloc(parset.n_sa_vel_recon * sizeof(double));
  base_sa = malloc(parset.n_sa_base_recon * 2 * sizeof(double));
  phase_sa = malloc(parset.n_sa_vel_recon * parset.n_sa_base_recon * sizeof(double));

  clouds_alpha = malloc(parset.n_cloud_per_task * sizeof(double));
  clouds_beta = malloc(parset.n_cloud_per_task * sizeof(double));

  workspace_phase = malloc(parset.n_sa_vel_recon * 3 * sizeof(double));

  if(parset.flag_sa_blrmodel != 8)
  {
    saRblr = exp(pm[num_params_blr]);
  }
  else
  {
    saRblr = exp(pm[num_params_blr + 9]);
  }
  rcloud_max_set = fmax(rcloud_max_set, saRblr * 5.0);
  printf("SA rcloud_min_max_set: %f %f\n", rcloud_min_set, rcloud_max_set);

  if(parset.flag_dim == -1)
  {
    memcpy(vel_sa, vel_sa_data, parset.n_sa_vel_recon * sizeof(double));
    memcpy(wave_sa, wave_sa_data, parset.n_sa_vel_recon * sizeof(double));

    memcpy(base_sa, base_sa_data, parset.n_sa_base_recon * 2 * sizeof(double));
  }
  else
  {
    double vel_max_set, vel_min_set;
    vel_max_set = sqrt(pow(2.0*sqrt(mbh/saRblr), 2.0) + pow(2.0*parset.sa_InstRes, 2.0));
    vel_min_set = - vel_max_set;
    double dVel = (vel_max_set- vel_min_set)/(parset.n_sa_vel_recon -1.0);
    for(i=0; i<parset.n_sa_vel_recon; i++)
    {
      vel_sa[i] = vel_min_set + dVel*i;
      wave_sa[i] = (1.0 + vel_sa[i]/C_Unit) * parset.sa_linecenter * (1.0+parset.redshift);
    }

    if(parset.flag_gravity == 1)
      memcpy(base_sa, base_sa_3c273, n_base_sa_3c273*2*sizeof(double));
    else
    {
      double phi;
      for(i=0; i<parset.n_sa_base_recon; i++)
      {
        phi = -PI/2.0 + PI/parset.n_sa_base_recon * i;
        base_sa[i*2+0] = 100.0*cos(phi);
        base_sa[i*2+1] = 100.0*sin(phi);
      }
    }

  }
#endif

  return;
}

void sim_end()
{
  free(model);
  free(par_fix);
  free(par_fix_val);
  free(Fcon);
  free(Fcon_rm);

  free(TransTau);
  free(TransV);
  free(TransW);
  free(Tline);
  free(Fline);
  free(Fline2d);
  free(Trans2D);
  free(Trans1D);

  free(clouds_tau);
  free(clouds_weight);
  free(clouds_vel);

  if(parset.flag_save_clouds && thistask == roottask)
  {
    fclose(fcloud_out);
  }

#ifdef SpecAstro

  free(vel_sa);
  free(wave_sa);
  free(Fline_sa);
  free(base_sa);
  free(phase_sa);

  free(clouds_alpha);
  free(clouds_beta);

  free(workspace_phase);
#endif
}

/*
 * set parameter values for either RM or SA
 */
void set_par_value_sim(double *pm, int flag_model)
{
  int i;
  switch(flag_model)
  {
    case 0:
      set_par_value_mymodel_sim(pm);
      break;

    case 1:
      i=0;
      pm[i++] = log(4.0);
      pm[i++] = 0.9;
      pm[i++] = 0.2;
      pm[i++] = cos(20.0/180.0*PI);
      pm[i++] = 40.0;
      pm[i++] = 0.0;
      pm[i++] = log(3.0);
      pm[i++] = 0.1;
      pm[i++] = 0.5;
      break;

    case 2:
      i=0;
      pm[i++] = log(4.0);
      pm[i++] = 0.9;
      pm[i++] = 0.2;
      pm[i++] = cos(20.0/180.0*PI);
      pm[i++] = 40.0;
      pm[i++] = 0.0;
      pm[i++] = log(3.0);
      pm[i++] = log(0.01);
      pm[i++] = log(0.1);
      break;

    case 3:
      i=0;
      pm[i++] = log(3.0);
      pm[i++] = log(5.0);
      pm[i++] = -1.0;
      pm[i++] = cos(20.0/180.0*PI);
      pm[i++] = 40.0;
      pm[i++] = 0.0;
      pm[i++] = log(3.0);
      pm[i++] = 0.5;
      pm[i++] = 0.5;
      break;

    case 4:
      i=0;
      pm[i++] = log(3.0);
      pm[i++] = log(5.0);
      pm[i++] = -1.0;
      pm[i++] = cos(20.0/180.0*PI);
      pm[i++] = 40.0;
      pm[i++] = 0.0;
      pm[i++] = log(3.0);
      pm[i++] = 0.5;
      pm[i++] = 0.5;
      break;

    case 5:
      i=0;
      pm[i++] = log(4.0);  //mu
      pm[i++] = 0.5;       //Fin
      pm[i++] = log(2.0);  //Fout
      pm[i++] = 1.5;       //alpha
      pm[i++] = cos(20.0/180.0*PI);      //inc
      pm[i++] = 40.0;      //opn
      pm[i++] = 0.5;       //k
      pm[i++] = 2.0;       //gam
      pm[i++] = 0.5;      //xi
      pm[i++] = log(2.0); //mbh
      pm[i++] = 0.5;      //fellip
      pm[i++] = 0.5;      //fflow
      pm[i++] = log(0.01);
      pm[i++] = log(0.1);
      pm[i++] = log(0.01);
      pm[i++] = log(0.1);
      pm[i++] = 0.0;      //theta_rot
      pm[i++] = log(0.001); //sig_turb
      break;

    case 6:
      i=0;
      pm[i++] = log(4.0);   // mu
      pm[i++] = 1.0;        // beta
      pm[i++] = 0.25;        // F
      pm[i++] = cos(20.0/180.0*PI); // inc
      pm[i++] = 40.0;       // opn
      pm[i++] = 0.0;        // kappa
      pm[i++] = 1.0;        // gamma
      pm[i++] = 1.0;        // obscuration
      pm[i++] = log(2.0);  //mbh
      pm[i++] = 0.5;       //fellip
      pm[i++] = 0.4;       //fflow
      pm[i++] = log(0.01); //
      pm[i++] = log(0.1);  //
      pm[i++] = log(0.01); //
      pm[i++] = log(0.1);  //
      pm[i++] = 0.0;       // theta_rot
      pm[i++] = log(0.001);  // sig_turb
      break;

    case 7:
      i=0;
      pm[i++] = log(4.0);  //mu
      pm[i++] = 0.8;       //beta
      pm[i++] = 0.1;       //F
      pm[i++] = cos(20.0/180.0*PI);//inc
      pm[i++] = 40.0;      //opn
      pm[i++] = 0.0;       //kappa
      pm[i++] = 1.0;       //gamma
      pm[i++] = 0.0;       //xi

      pm[i++] = 0.5;      //fsh
      pm[i++] = log(8.0); //mu_un
      pm[i++] = 1.2;      //beta_un
      pm[i++] = 0.1;      //F_un
      pm[i++] = 20.0;     //opn_un

      pm[i++] = log(2.0); //mbh
      pm[i++] = 0.5;      //fellip
      pm[i++] = 0.4;      //fflow
      pm[i++] = log(0.01);
      pm[i++] = log(0.1);
      pm[i++] = log(0.01);
      pm[i++] = log(0.1);
      pm[i++] = 0.0;      //theta_rot
      pm[i++] = 0.5;      //fellip_un
      pm[i++] = 0.4;      //fflow_un
      pm[i++] = log(0.001); // sig_turb
      break;

    case 8:
      i=0;
      pm[i++] = 50.0;   //theta_min
      pm[i++] = 20.0;   //dtheta_max
      pm[i++] = log(1.0); // r_min
      pm[i++] = 4.0;      // fr_max
      pm[i++] = 1.0;      // gamma
      pm[i++] = 1.0;      // alpha
      pm[i++] = -2.0;      // lambda
      pm[i++] = -0.5;      // k
      pm[i++] = 0.0;      // xi
      pm[i++] = log(30.0);     // Rv
      pm[i++] = log(20.0);     // Rblr, should larger than r_max=r_max * fr_max
      pm[i++] = cos(30.0/180.0*PI);     // inc
      pm[i++] = log(4.0); // mbh
      break;

    case 9:
      i=0;
      pm[i++] = log(4.0);   //mu
      pm[i++] = 1.0;        //beta
      pm[i++] = 0.2;        //F
      pm[i++] = cos(30.0/180.0*PI);     // inc
      pm[i++] = 30.0;       //opn
      pm[i++] = log(4.0);   // mbh
      break;
  }
  return;
}

/*
 * get index of mbh from a BLR model.
 *
 */
int get_idx_mbh_from_blrmodel()
{
  int idx = -1;
  switch(parset.flag_blrmodel)
  {
    case 0:
      idx = offsetof(MyBLRmodel, mbh);
      break;

    case 1:
      idx = offsetof(BLRmodel1, mbh);
      break;

    case 2:
      idx = offsetof(BLRmodel2, mbh);
      break;

    case 3:
      idx = offsetof(BLRmodel3, mbh);
      break;

    case 4:
      idx = offsetof(BLRmodel4, mbh);
      break;

    case 5:
      idx = offsetof(BLRmodel5, mbh);
      break;

    case 6:
      idx = offsetof(BLRmodel6, mbh);
      break;

    case 7:
      idx = offsetof(BLRmodel7, mbh);
      break;

    case 8:
      idx = offsetof(BLRmodel8, mbh);
      break;

    case 9:
      idx = offsetof(BLRmodel9, mbh);
      break;
  }
  return idx / sizeof(double);
}


#ifndef FILE_PROCESSINGUNIT
#define FILE_PROCESSINGUNIT

#include "../devicedata/memorymanager.h"
#include "timer.h"

#ifndef CPU_ONLY
    #include "gpu_handle.h"

    #ifdef MAGMA
        #include "GpuMagma_queue.h"
        #include "magma_v2.h"
    #endif
#else
    #pragma message("Cpu only mode is activated. Gpu suppport is disabled. If you wish to activate Gpu support, remove compiler flag CPU_ONLY")
#endif

#ifdef __INTEL_CLANG_COMPILER
  #pragma message("Intel Math Kernel Library is used for acceleration of linear algebra operations.")
  #include <mkl.h>
#else
  #pragma message("CBLAS/LAPACK/LAPACKE are used for acceleration of linear algebra operations.")
  #include <cblas.h>
  #include <lapacke.h>
#endif

#include <map>
#include <type_traits>
#include <iostream>
#include <memory>
#include <string>

template<class floating>
class ProcessingUnitDevice;

template<class floating>
using ProcessingUnit = std::shared_ptr<ProcessingUnitDevice<floating>>;

/**
 * Class OperationType is used for choosing the matrix operation
 * type in several operations of interface ProcessingUnitDevice.
 */
enum class OperationType {
    Identical, //!< taking the matrix without any changes
    Transposed, //!< taking the transposition of the matrix
    Hermitian //!< taking the complex conjugate of the matrix
};

/**
 * Class ProcessingUnitDevice is an abstract interface for easy calls to
 * BLAS/LAPACK/LAPACKE-like function calls on a certain device,
 * e.g. for CUDA/cuSOLVER on a Gpu, or for MAGMA on a Gpu.
 *
 * The floating point type is passed as a template parameter.
 *
 * @tparam floating floating point type
 */
template<class floating>
class ProcessingUnitDevice {
public:
    /**
     * Constructor.
     */
    ProcessingUnitDevice();

    /**
     * Destructor.
     */
    virtual ~ProcessingUnitDevice() = default;

    /**
     * Creates a new instance of a Timer stopwatch on the current device.
     * @return new Timer instance
     */
    virtual Timer create_timer() const = 0;

    /**
     * Returns the MemoryManager associated with the current device.
     * @return MemoryManager
     */
    virtual MemoryManager get_memory_manager() const = 0;

    /**
     * Returns the subclass's name in a human-readable format as a string
     * @return subclass's human-readable name
     */
    virtual std::string display() const = 0;

    /**
     * Function ixamax is an abstraction of calls isamax/idamax/etc.
     * It returns the index of the first element having maximum absolute value.
     *
     * See also
     * https://www.netlib.org/lapack/explore-html/d0/d73/group__aux__blas_ga285793254ff0adaf58c605682efb880c.html
     *
     * @param[in] n number of elements of input vector
     * @param[in] x a C-style array containing (1 + (@param n-1) * abs(@param incx) elements of type @tparam floating
     * @param[in] incx storage spacing between elements of @param x
     * @return index of first element having maximum absolute value
     */
    virtual int ixamax(const int n, const floating * const x, const int incx) const = 0;

    /**
     * Function xaxpy is an abstraction of calls saxpy/daxpy/etc.
     * It performs the calculation y := alpha * x + y
     *
     * See also
     * https://www.netlib.org/lapack/explore-html/de/da4/group__double__blas__level1_ga8f99d6a644d3396aa32db472e0cfc91c.html
     *
     * @param[in] n number of elements in input vectors
     * @param[in] alpha scalar alpha
     * @param[in] x a C-style array containing (1 + (@param n-1) * abs(@param incx) elements of type @tparam floating
     * @param[in] incx storage spacing between elements of @param x
     * @param[in,out] y a C-style array containing (1 + (@param n-1) * abs(@param incy) elements of type @tparam floating
     * @param[in] incy storage spacing between elements of @param y
     */
    virtual void xaxpy(const int n, const floating alpha, const floating * const x, const int incx, floating * const y, const int incy) const = 0;

    /**
     * Function xcopy is an abstraction of calls scopy/dcopy/etc.
     * It performs the copy operation dest := source
     *
     * See also
     * https://www.netlib.org/lapack/explore-html/de/da4/group__double__blas__level1_ga21cdaae1732dea58194c279fca30126d.html
     * @param[in] n number of elements in input vectors
     * @param[in] source a C-style array containing (1 + (@param n-1) * abs(@param inc_source) elements of type @tparam floating
     * @param[in] inc_source storage spacing between elements of @param source
     * @param[out] dest a C-style array containing (1 + (@param n-1) * abs(@param inc_dest) elements of type @tparam floating
     * @param[in] inc_dest storage spacing between elements of @param dest
     */
    virtual void xcopy(const int n, const floating * const source, const int inc_source, floating * const dest, const int inc_dest) const = 0;

    /**
     * Function xdot is an abstraction of calls sdot/ddot/etc.
     * It performs the dot product <x, y>
     *
     * See also
     * http://www.netlib.org/lapack/explore-html/de/da4/group__double__blas__level1_ga75066c4825cb6ff1c8ec4403ef8c843a.html
     *
     * @param[in] n number of elements in input vectors
     * @param[in] x a C-style array containing (1 + (@param n-1) * abs(@param incx) elements of type @tparam floating
     * @param[in] incx spacing between elements of @param x
     * @param[in,out] y a C-style array containing (1 + (@param n-1) * abs(@param incy) elements of type @tparam floating
     * @param[in] incy spacing between elements of @param y
     * @return the value of the dot product <x, y>
     */
    virtual floating xdot(const int n, const floating * const x, const int incx, const floating * const y, const int incy) const = 0;

    /**
     * Function xgemm is an abstraction of calls sgemm/dgemm/etc.
     * It performs the matrix product operation C := alpha * trans_A(A) * trans_B(B) + beta * C
     *
     * See also
     * http://www.netlib.org/lapack/explore-html/d1/d54/group__double__blas__level3_gaeda3cbd99c8fb834a60a6412878226e1.html
     *
     * @param[in] trans_A defines the transformation applied to matrix A (see enum class OperationType)
     * @param[in] trans_B defines the transformation applied to matrix B (see enum class OperationType)
     * @param[in] m the number of rows of matrix trans_A(A) and C
     * @param[in] n the number of columns of matrix trans_B(B) and C
     * @param[in] k the number of columns of matrix C
     * @param[in] alpha scalar applied to A
     * @param[in] A matrix stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] lda the first dimension of A
     * @param[in] B matrix stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] ldb the first dimension of B
     * @param[in] beta scalar applied to C
     * @param[in,out] C matrix stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] ldc the first dimension of C
     */
    virtual void xgemm(const OperationType trans_A, const OperationType trans_B,
                       const int m, const int n, const int k, const floating alpha, const floating * const A,
                       const int lda, const floating * const B, const int ldb, const floating beta, floating * const C, const int ldc) const = 0;

    /**
     * Function xgemv is an abstraction of calls sgemv/dgemv/etc.
     * It performs the matrix times vector operation y := alpha * trans(A) * x + beta * y
     *
     * See also
     * http://www.netlib.org/lapack/explore-html/d7/d15/group__double__blas__level2_gadd421a107a488d524859b4a64c1901a9.html
     *
     * @param[in] trans defines the transformation applied to matrix A (see enum class OperationType)
     * @param[in] m the number of rows of matrix A
     * @param[in] n the number of columns of matrix A
     * @param[in] alpha scalar applied to A
     * @param[in] A matrix stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] lda the first dimension of matrix A
     * @param[in] x A C-style array containing (1 + (@param DIM-1) * abs(@param incx) elements of type @tparam floating. DIM is @param n if trans is OperationType::Identical, and @param m else.
     * @param[in] incx storage spacing between elements of @param x
     * @param[in] beta scalar applied to @param y
     * @param[in,out] y A C-style array containing (1 + (@param DIM-1) * abs(@param incy) elements of type @tparam floating. DIM is @param n if trans is OperationType::Identical, and @param m else.
     * @param[in] incy storage spacing between elements of @param y
     */
    virtual void xgemv(const OperationType trans, const int m, const int n,
                       const floating alpha, floating const * const A, const int lda, floating const * const x, const int incx,
                       const floating beta, floating * const y, const int incy) const = 0;

    /**
     * Function xgetrf is an abstraction of calls sgetrf/dgetrf/etc.
     * It computes the PLU factorization of A general M-by-N matrix A. A is overwritten by L and U,
     * and L has unit diagonal elements.
     *
     * See also
     * http://www.netlib.org/lapack/explore-html/dd/d9a/group__double_g_ecomputational_ga0019443faea08275ca60a734d0593e60.html
     *
     * @param[in] m the number of rows of matrix A
     * @param[in] n the number of columns of matrix A
     * @param[in,out] A matrix stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] lda the first dimension of matrix A
     * @param[out] ipiv permutation array, row i of matrix A is interchanged with row ipiv[i]
     * @param[out] info If == 0, successful exit. If < 0, the i-th argument had illegal value. If > 0, U(i, i) is exactly 0 and cannot be used to solve A system of equations.
     */
    virtual void xgetrf(int * const m, int * const n, floating * const A, int * const lda,
                        int * const ipiv, int * const info) const = 0;

    /**
     * Function xgetri is an abstraction of calls sgetri/dgetri/etc.
     * It computes the inverse of A square matrix using its PLU factorization as calculated by xgetrf.
     *
     * See also
     * http://www.netlib.org/lapack/explore-html/dd/d9a/group__double_g_ecomputational_ga56d9c860ce4ce42ded7f914fdb0683ff.html
     *
     * @param[in] n order of square matrix A
     * @param[in,out] A LU-factorization matrix (as calculated by xgetrf) stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] lda the first dimension of matrix A
     * @param[in] ipiv permutation array from xgetrf
     * @param[out] work is temporary working space in form of A C-style array containing elements of type floating. On exit, if info == 0, work[1] returns optimal @param lwork
     * @param[in] lwork dimension of array @param work. If == -1, the routine calculates the optimal size of array @param work.
     * @param[out] info If == 0, successful exit. If < 0, the i-th argument had illegal value. If > 0, U(i, i) is exactly 0 and cannot be used to solve A system of equations.
     */
    virtual void xgetri(const int * const n, floating * const A, const int * const lda, const int * const ipiv,
                        floating * const work, const int * const lwork, int * const info) const = 0;

    /**
     * Function xgetrs is an abstraction of calls sgetrs/dgetrs/etc.
     * It solves A system of linear equations trans(A) * X = B using its PLU factorization as calculated by xgetrf.
     * Matrix A has to be square, and the result is stored in B.
     *
     * See also
     * http://www.netlib.org/lapack/explore-html/dd/d9a/group__double_g_ecomputational_ga58e332cb1b8ab770270843221a48296d.html
     *
     * @param[in] trans defines the transformation applied to matrix A (see enum class OperationType)
     * @param[in] n order of square matrix A
     * @param[in] nrhs number of right hand sides, i.e. the number of columns of B
     * @param[in] A LU-factorization matrix (as calculated by xgetrf) stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] lda the first dimension of matrix A
     * @param[in] ipiv permutation array from xgetrf
     * @param[in,out] B matrix of right hand sides stored as A column-wise C-style array containing elements of type @tparam floating
     * @param[in] ldb the first dimension of matrix B
     * @param[out] info If == 0, successful exit. If < 0, the i-th argument had illegal value.
     */
    virtual void xgetrs(const OperationType trans, const int * const n, const int * const nrhs,
                        const floating * const A, const int * const lda, const int * const ipiv,
                        floating * const B, const int * const ldb, int * const info) const = 0;

    /**
     * Function xscal is an abstraction of calls sscal/dscal/etc.
     * It performs the scaling operation x := alpha * x.
     *
     * See also
     * http://www.netlib.org/lapack/explore-html/de/da4/group__double__blas__level1_ga793bdd0739bbd0e0ec8655a0df08981a.html
     *
     * @param n number of elements in input vectors
     * @param alpha scalar alpha
     * @param x a C-style array containing (1 + (@param n-1) * abs(@param incx) elements of type @tparam floating
     * @param incx storage spacing between elements of @param x
     */
    virtual void xscal(const int n, const floating alpha, floating * const x, const int incx) const = 0;

    /**
     * Returns true if @tparam floating is float.
     * @return true if @tparam floating is float
     */
    constexpr static bool isFloat();

    /**
     * Returns true if @tparam floating is double.
     * @return true if @tparam floating is double
     */
    constexpr static bool isDouble();
};

/**
 * Class Cpu provides linear algebra operations on the Cpu using
 * BLAS/LAPACK/LAPACKE. All memory allocations also take place in
 * ordinary RAM.
 *
 * @tparam floating floating point type, either float or double
 */
template<class floating>
class Cpu : public ProcessingUnitDevice<floating> {
public:
    using ProcessingUnitDevice<floating>::isFloat;
    using ProcessingUnitDevice<floating>::isDouble;

    /**
     * Constructor
     */
    Cpu() = default;

    /**
     * Destructor
     */
    ~Cpu() override = default;

    Timer create_timer() const override;
    MemoryManager get_memory_manager() const override;
    std::string display() const override;

    int ixamax(const int n, const floating * const x, const int incx) const override;
    void xaxpy(const int n, const floating alpha, const floating * const x, const int incx, floating * const y, const int incy) const override;
    void xcopy(const int n, const floating * const source, const int inc_source, floating * const dest, const int inc_dest) const override;
    floating xdot(const int n, const floating * const x, const int incx, const floating * const y, const int incy) const override;
    void xgemm(const OperationType trans_A, const OperationType trans_B,
               const int M, const int N, const int K, const floating alpha, const floating * const A,
               const int lda, const floating * const B, const int ldb, const floating beta, floating * const C, const int ldc) const override;
    void xgemv(const OperationType trans, const int m, const int n,
               const floating alpha, floating const * const A, const int lda, floating const * const x, const int incx,
               const floating beta, floating * const y, const int incy) const override;
    void xgetrf(int * const m, int * const n, floating * const A, int * const lda,
                int * const ipiv, int * const info) const override;
    void xgetri(const int * const n, floating * const A, const int * const lda, const int * const ipiv,
                floating * const work, const int * const lwork, int * const info) const override;
    void xgetrs(const OperationType trans, const int * const n, const int * const nrhs,
                const floating * const A, const int * const lda, const int * const ipiv,
                floating * const B, const int * const ldb, int * const info) const override;
    void xscal(const int N, const floating alpha, floating * const x, const int incx) const override;

private:
    static std::map<OperationType, CBLAS_TRANSPOSE> to_internal_operation_blas; //!< Converts the OperationType to the corresponding CBLAS_TRANSPOSE type
    static std::map<OperationType, const char * const> to_internal_operation_lapack; //!< Converts the OperationType to the corresponding LAPACK transposition type
    static MemoryManager _device_manager; //!< Memory manager associated with Cpu memory
};

#ifndef CPU_ONLY

/**
 * Class Gpu provides linear algebra operations on the Gpu using
 * cuBLAS/cuSOLVER. All memory allocations also take place in
 * Gpu RAM.
 *
 * @tparam floating floating point type, either float or double
 */
template<class floating>
class Gpu : public ProcessingUnitDevice<floating> {
public:
    using ProcessingUnitDevice<floating>::isFloat;
    using ProcessingUnitDevice<floating>::isDouble;

    /**
     * Constructor
     */
    Gpu() = default;

    /**
     * Destructor
     */
    ~Gpu() override = default;

    Timer create_timer() const override;
    MemoryManager get_memory_manager() const override;
    std::string display() const override;

    /**
     * Returns the instance's cuBLAS handle.
     * @return cuBLAS handle
     */
    auto get_cublas_handle() const;

    /**
     * Returns the instance's cuSOLVER handle.
     * @return cuSOLVER handle
     */
    auto get_cusolver_handle() const;

    int ixamax(const int n, const floating * const x, const int incx) const override;
    void xaxpy(const int n, const floating alpha, const floating * const x, const int incx, floating * const y, const int incy) const override;
    void xcopy(const int n, const floating * const source, const int inc_source, floating * const dest, const int inc_dest) const override;
    floating xdot(const int n, const floating * const x, const int incx, const floating * const y, const int incy) const override;
    void xgemm(const OperationType trans_A, const OperationType trans_B,
               const int M, const int N, const int K, const floating alpha, const floating * const A,
               const int lda, const floating * const B, const int ldb, const floating beta, floating * const C, const int ldc) const override;
    void xgemv(const OperationType trans, const int m, const int n,
               const floating alpha, floating const * const A, const int lda, floating const * const x, const int incx,
               const floating beta, floating * const y, const int incy) const override;
    void xgetrf(int * const m, int * const n, floating * const A, int * const lda,
                int * const ipiv, int * const info) const override;
    void xgetri(const int * const n, floating * const A, const int * const lda, const int * const ipiv,
                floating * const work, const int * const lwork, int * const info) const override;
    void xgetrs(const OperationType trans, const int * const n, const int * const nrhs,
                const floating * const A, const int * const lda, const int * const ipiv,
                floating * const B, const int * const ldb, int * const info) const override;
    void xscal(const int N, const floating alpha, floating * const x, const int incx) const override;

private:
    static std::map<OperationType, cublasOperation_t> to_internal_operation; //!< Converts the OperationType to the corresponding cuBLAS operation type
    static GpuHandle _handle; //!< Gpu handle for cuBLAS/cuSOLVER operations
    static MemoryManager _deviceManager; //!< Memory manager associated with CUDA memory

};

#ifdef MAGMA
/**
 * Class GpuMagma provides linear algebra operations on the Gpu using
 * MAGMA. All memory allocations also take place in Gpu RAM.
 *
 * @tparam floating floating point type, either float or double
 */
template<class floating>
class GpuMagma : public ProcessingUnitDevice<floating> {
public:
    using ProcessingUnitDevice<floating>::isFloat;
    using ProcessingUnitDevice<floating>::isDouble;

    /**
     * Constructor
     */
    GpuMagma() = default;

    /**
     * Destructor
     */
    ~GpuMagma() override = default;

    Timer create_timer() const override;
    MemoryManager get_memory_manager() const override;
    std::string display() const override;

    /**
     * Returns the instance's magma queue.
     */
    auto get_magma_queue() const;

    int ixamax(const int n, const floating * const x, const int incx) const override;
    void xaxpy(const int n, const floating alpha, const floating * const x, const int incx, floating * const y, const int incy) const override;
    void xcopy(const int n, const floating * const source, const int inc_source, floating * const dest, const int inc_dest) const override;
    floating xdot(const int n, const floating * const x, const int incx, const floating * const y, const int incy) const override;
    void xgemm(const OperationType trans_A, const OperationType trans_B,
               const int M, const int N, const int K, const floating alpha, const floating * const A,
               const int lda, const floating * const B, const int ldb, const floating beta, floating * const C, const int ldc) const override;
    void xgemv(const OperationType trans, const int m, const int n,
               const floating alpha, floating const * const A, const int lda, floating const * const x, const int incx,
               const floating beta, floating * const y, const int incy) const override;
    void xgetrf(int * const m, int * const n, floating * const A, int * const lda,
                int * const ipiv, int * const info) const override;
    void xgetri(const int * const n, floating * const A, const int * const lda, const int * const ipiv,
                floating * const work, const int * const lwork, int * const info) const override;
    void xgetrs(const OperationType trans, const int * const n, const int * const nrhs,
                const floating * const A, const int * const lda, const int * const ipiv,
                floating * const B, const int * const ldb, int * const info) const override;
    void xscal(const int N, const floating alpha, floating * const x, const int incx) const override;

private:
    static std::map<OperationType, magma_trans_t> to_internal_operation; //!< Converts the OperationType to the corresponding MAGMA transposition type
    static GpuMagmaQueue _queue; //!< MAGMA queue for MAGMA operations
    static MemoryManager _device_manager; //!< Memory manager associated with CUDA memory
};

/**
 * Class ReplacementNumber is used for passing the operation to GpuMixed, which is then calculated via
 * GpuMagma instead of ordinary Gpu processing unit.
 */
enum class ReplacementNumber {
    IXAMAX,
    XAXPY,
    XCOPY,
    XDOT,
    XGEMM,
    XGEMV,
    XGETRF,
    XGETRS,
    XSCAL
};

/**
 * Class GpuMixed provides linear algebra operations on the Gpu using
 * cuBLAS/cuSOLVER, except for the one operation specified by replacementNumber,
 * which is calculated using MAGMA.
 * All memory allocations also take place in Gpu RAM.
 *
 * @tparam floating floating point type, either float or double
 */
template<class floating>
class GpuMixed : public ProcessingUnitDevice<floating> {
public:
    /**
     * Constructor
     */
    GpuMixed(const ReplacementNumber replacementNumber = ReplacementNumber::XGETRF);

    /**
     * Destructor
     */
    ~GpuMixed() override = default;

    Timer create_timer() const override;

    MemoryManager get_memory_manager() const override;

    std::string display() const override;

    int ixamax(const int n, const floating * const x, const int incx) const override;
    void xaxpy(const int n, const floating alpha, const floating * const x, const int incx, floating * const y, const int incy) const override;
    void xcopy(const int n, const floating * const source, const int inc_source, floating * const dest, const int inc_dest) const override;
    floating xdot(const int n, const floating * const x, const int incx, const floating * const y, const int incy) const override;
    void xgemm(const OperationType trans_A, const OperationType trans_B,
               const int M, const int N, const int K, const floating alpha, const floating * const A,
               const int lda, const floating * const B, const int ldb, const floating beta, floating * const C, const int ldc) const override;
    void xgemv(const OperationType trans, const int m, const int n,
               const floating alpha, floating const * const A, const int lda, floating const * const x, const int incx,
               const floating beta, floating * const y, const int incy) const override;
    void xgetrf(int * const m, int * const n, floating * const A, int * const lda,
                int * const ipiv, int * const info) const override;
    void xgetri(const int * const n, floating * const A, const int * const lda, const int * const ipiv,
                floating * const work, const int * const lwork, int * const info) const override;
    void xgetrs(const OperationType trans, const int * const n, const int * const nrhs,
                const floating * const A, const int * const lda, const int * const ipiv,
                floating * const B, const int * const ldb, int * const info) const override;
    void xscal(const int N, const floating alpha, floating * const x, const int incx) const override;

    ReplacementNumber get_replacement_number() const;

private:
    static Gpu<floating> _gpu; //!< Gpu processing unit
    static GpuMagma<floating> _gpu_magma; //!< MAGMA processing unit
    ReplacementNumber _replacementNumber; //!< current instance's operation which should be exectued on MAGMA instead of cuBLAS/cuSOLVER
};
#endif
#endif

#include "processingunit.hpp"
#include "processingunit_cpu.hpp"

#ifndef CPU_ONLY
    #include "processingunit_gpu.hpp"
    #ifdef MAGMA
        #include "processingunit_gpu_magma.hpp"
        #include "processingunit_gpu_mixed.hpp"
    #endif
#endif

#endif


#ifndef TOPICH
#define TOPICH

#include <gsl/gsl_vector.h>
#include <math.h>
#include "utils.h"
#include "typedefs.h"
#include "hyperparameter.h"

#define MH_ETA_STDEV 0.005
#define MH_GAM_STDEV 0.005

/* === topic methods === */

/*
 * update the count of a word in a topic
 *
 */

void topic_update_word(topic* t, int w, double update);

/*
 * update the document count in a topic
 *
 */

void topic_update_doc_cnt(topic* t, double update);

/*
 * write log probability in a file
 *
 */

void topic_write_log_prob(topic* t, FILE* f);

/* === tree methods === */

/*
 * allocate a new tree with a certain depth
 *
 */

tree* tree_new(int depth,
               int nwords,
               gsl_vector* eta,
               gsl_vector* gam,
               double scaling_shape,
               double scaling_scale);

/*
 * add children to a node down to the depth of the tree;
 * return the leaf of that path
 *
 */

topic* tree_fill(topic* t);

/*
 * add a child to a topic
 *
 */

topic* topic_add_child(topic* t);

/*
 * make a new topic
 *
 */

topic* topic_new(int nwords, int level, topic* parent, tree* tr);

/*
 * given a leaf with 0 instances, delete the leaf and all ancestors
 * which also have 0 instances.  (!!! use asserts here)
 *
 */

void tree_prune(topic* t);

/*
 * delete a node from the tree
 *
 */

void delete_node(topic* t);

/*
 * write a tree to file
 *
 */

void tree_write_log_prob(tree* tree, FILE* f);

/*
 * sample a document path from a tree
 *
 */

void populate_prob_dfs(topic* node, doc* d, double* logsum, double* pprob, int root_level);
void tree_sample_doc_path(tree* tr, doc* d, short do_remove, int root_level);

/*
 * sample a new path in the tree for a document
 *
 */

void tree_sample_path_for_doc(tree* t, doc* d);

/*
 * update the tree from an entire document
 *
 */

void tree_update_from_doc(doc* d, double update, int root_level);

/*
 * sample a leaf from the tree with populated probabilties
 *
 */

topic* tree_sample_path(topic* node, double logsum);
topic* tree_sample_dfs(double r, topic* node, double* sum, double logsum);
void tree_add_doc_to_path(topic* node, doc* d, int root_level);
void tree_remove_doc_from_path(tree* tr, doc* d, int root_level);

/*
 * write a tree to a file
 *
 */

void write_tree(tree* tf, FILE* file);
void write_tree_levels(tree* tr, FILE* file);
void write_tree_level_dfs(topic* t, FILE* f);
void write_tree_topics_dfs(topic* t, FILE* f);

/*
 * scores
 *
 */

double gamma_score(topic* t);
double gamma_score_PY(topic* t, double gam_add);
double eta_score(topic* t);
double log_gamma_ratio(doc* d, topic* t, int level);
double log_gamma_ratio_new(doc* d, int level, double eta, int nterms);
void tree_mh_update_eta(tree* tr);
void dfs_sample_scaling(topic* t);

/*
 * copying a tree
 *
 */

void copy_topic(const topic* src, topic* dest);
void copy_tree_dfs(const topic* src, topic* dest);
tree * copy_tree(const tree* tr);

void free_tree(tree * tr);
void free_tree_dfs(topic * t);

int ntopics_in_tree(tree * tr);
int ntopics_in_tree_dfs(topic * t);

#endif


/*************************************************************************************
 * Copyright (C) 2017, Trung Pham <me@trungbpham.com>                                *
 * All rights reserved.                                                              *
 *                                                                                *
 * Redistribution and use in source and binary forms, with or without                *
 * modification, are permitted provided that the following conditions are met:       *
 *                                                                                *
 *       * Redistributions of source code must retain the above copyright notice,    *
 *       this list of conditions and the following disclaimer.                       *
 *                                                                                *
 *       * Redistributions in binary form must reproduce the above copyright notice, *
 *       this list of conditions and the following disclaimer in the documentation   *
 *       and/or other materials provided with the distribution.                      *
 *                                                                                *
 *       * Neither the name of cfeap nor the names of its contributors may be used   *
 *       to endorse or promote products derived from this software without specific  *
 *       prior written permission.                                                   *
 *                                                                                *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"       *
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE         *
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE    *
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE      *
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL        *
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR        *
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER        *
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,     *
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE     *
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              *
 *************************************************************************************/

#ifndef __CBLAS_WRAPPER_H__
#define __CBLAS_WRAPPER_H__

#include <stdio.h>
#include <cblas.h>

int cblw_scalar_vec_mul_x(size_t dim, double a, double *x);

int cblw_scalar_mat_mul_x(size_t col, size_t row, double a, double *X);

int cblw_scalar_vec_mul_y(size_t dim, double a, double *x, double *y);

int cblw_scalar_mat_mul_y(size_t col, size_t row, double a, double *X, double *Y);

int cblw_mat_vec_mul(size_t col, size_t row, double *A, double *x, double *y);

int cblw_mat_mat_mul(size_t rowA, size_t colB, size_t colA_rowB, double *A, double *B, double *C);

#endif //__CBLAS_WRAPPER_H__


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <time.h>
#include <gsl/gsl_rng.h>

#include "allvars.h"
#include "proto.h"

void init()
{
  int i;
  double dT, T1, T2;

  strcpy(fname_results, "data/results.txt");
  fp_results = fopen(fname_results, "w");

#ifdef JAVELIN
  fprintf(fp_results, "JAVELIN-single top-hat transfer function.\n");
  printf("JAVELIN.\n");
  nc_lim_low = nc_lim_up = 1;
#elif defined TOPHAT
  fprintf(fp_results, "multiple top-hat transfer function.\n");
  printf("TOPHATs.\n");
#else
  fprintf(fp_results, "multiple Gaussian transfer function.\n");
  printf("GAUSSIANs.\n");
#endif

  memory_alloc_data();
  read_data();

  nall = ncon + nline;
  nall_data = ncon_data + nline_data;
  memory_alloc();

  scale_con = scale_line = 1.0;
  scale_light_curves();

  memcpy(Fall_data, Fcon_data, ncon_data*sizeof(double));
  memcpy(Fall_data+ncon_data, Fline_data, nline_data*sizeof(double));

  len_con = Tcon_data[ncon_data-1] - Tcon_data[0];
  len_line = Tline_data[nline_data-1] - Tline_data[0];

  cad_con = 0.0;
  for(i=0; i<ncon_data-1; i++)
  {
    cad_con += Tcon_data[i+1] - Tcon_data[i];
  }
  cad_con /= (ncon_data-1);

  cad_line = 0.0;
  for(i=0; i<nline_data-1; i++)
  {
    cad_line += Tline_data[i+1] - Tline_data[i];
  }
  cad_line /= (nline_data-1);

  printf("len: %f %f\n", len_con, len_line);
  printf("cad: %f %f\n", cad_con, cad_line);

  T1 = Tcon_data[0];
  T2 = Tcon_data[ncon_data-1];

  dT = T2-T1;
  T1 -= 0.1*dT;
  T2 += 0.1*dT;
  dT = (T2-T1)/(ncon-1.0);
  for(i=0; i<ncon; i++)
    Tcon[i] = T1 + dT *i;

  T1 = Tline_data[0];
  T2 = Tline_data[nline_data-1];

  dT = T2-T1;
  T1 -= 0.1*dT;
  T2 += 0.1*dT;
  dT = (T2-T1)/(nline-1.0);
  for(i=0; i<nline; i++)
    Tline[i] = T1 + dT *i;
}

void scale_light_curves()
{
  int i;
  double mean, norm;

  mean = 0.0;
  norm = 0.0;
  for(i=0; i<ncon_data; i++)
  {
    mean += Fcon_data[i]/(Fcerrs_data[i] * Fcerrs_data[i]);
    norm += 1.0/(Fcerrs_data[i] * Fcerrs_data[i]);
  }
  mean /= norm;
  scale_con = mean;

  for(i=0; i<ncon_data;i++)
  {
    Fcon_data[i] /= mean;
    Fcerrs_data[i] /= mean;
  }

  mean = 0.0;
  norm = 0.0;
  for(i=0; i<nline_data; i++)
  {
    mean += Fline_data[i]/(Flerrs_data[i]*Flerrs_data[i]);
    norm += 1.0/(Flerrs_data[i]*Flerrs_data[i]);
  }
  mean /= norm;
  scale_line = mean;
  for(i=0; i<nline_data;i++)
  {
    Fline_data[i] /= mean;
    Flerrs_data[i] /= mean;
  }

  printf("scale: %e %e\n", scale_con, scale_line);
  fprintf(fp_results, "scale: %e %e\n", scale_con, scale_line);
}

/*
 *
 */
void memory_alloc_data()
{
  Tcon_data = malloc(ndata_max*sizeof(double));
  if(Tcon_data==NULL)
  {
    strcpy(str_error_exit, "Tcon_data");
    error_exit(7);
  }

  Fcon_data = malloc(ndata_max*sizeof(double));
  if(Fcon_data==NULL)
  {
    strcpy(str_error_exit, "Fcon_data");
    error_exit(7);
  }

  Fcerrs_data = malloc(ndata_max*sizeof(double));
  if(Fcerrs_data==NULL)
  {
    strcpy(str_error_exit, "Fcerrs_data");
    error_exit(7);
  }

  Tline_data = malloc(ndata_max*sizeof(double));
  if(Tline_data==NULL)
  {
    strcpy(str_error_exit, "Tline_data");
    error_exit(7);
  }

  Fline_data = malloc(ndata_max*sizeof(double));
  if(Fline_data==NULL)
  {
    strcpy(str_error_exit, "Fline_data");
    error_exit(7);
  }

  Flerrs_data = malloc(ndata_max*sizeof(double));
  if(Flerrs_data==NULL)
  {
    strcpy(str_error_exit, "Flerrs_data");
    error_exit(7);
  }
}
void memory_alloc()
{
  workspace_ipiv = malloc(ndata_max*sizeof(double));
  workspace = malloc(10*ndata_max*sizeof(double) + 10);

  Fall_data = array_malloc(nall_data);

  Smat = array_malloc(nall_data*nall_data);
  Nmat = array_malloc(nall_data*nall_data);
  Cmat = array_malloc(nall_data*nall_data);
  ICmat = array_malloc(nall_data*nall_data);
  ICvmat = array_malloc(nall_data*nall_data);

  USmat = array_malloc(nall*nall_data);
  ASmat = array_malloc(nall*nall);

  Tmat1 = array_malloc(nall_data*nall_data);
  Tmat2 = array_malloc(nall_data*nall_data);
  Tmat3 = array_malloc(nall_data*nall_data);
  Tmat4 = array_malloc(nall_data*nall_data);

  Tcon = array_malloc(ncon);
  Fcon = array_malloc(ncon);
  Fcerrs = array_malloc(ncon);
  Tline = array_malloc(nline);
  Fline = array_malloc(nline);
  Flerrs = array_malloc(nline);

  theta_range = matrix_malloc(ntheta_max, 2);
  theta_best = array_malloc(ntheta_max);
  theta_best_var = array_malloc(ntheta_max*2);
  theta_input = array_malloc(ntheta_max);
  sigma_input = array_malloc(ntheta_max);
  theta_fixed = malloc(ntheta_max*sizeof(int));

  theta_best_con = array_malloc(ntheta_max);
  theta_best_var_con = array_malloc(ntheta_max*2);

  cov_matrix = array_malloc(ntheta_max * ntheta_max);

  grid_tau = array_malloc(nc_lim_up);
  TF_tau = array_malloc(ntau);
  TF = array_malloc(ntau);

  gsl_T = gsl_rng_default;
  gsl_r = gsl_rng_alloc (gsl_T);
  gsl_rng_set(gsl_r, time(NULL));
}


#include "jobqueue.h"
#include "degnome.h"
#include "fitfunc.h"
#include "flagparse.c"
#include <stdio.h>
#include <string.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
#include <time.h>
#include <pthread.h>
#include <unistd.h>
#include <limits.h>

typedef struct JobData JobData;
struct JobData {
	Degnome* child;
	Degnome* p1;
	Degnome* p2;
};

void usage(void);
void help_menu(void);
int jobfunc(void* p, void* tdat);

const char* usageMsg =
	"Usage: polygensim [-h] [-c chromosome_length] [-e mutation_effect]\n"
	"\t\t  [-g num_generations] [-m mutation_rate]\n"
	"\t\t  [-o crossover_rate] [-p population_size]\n"
	"\t\t  [-t num_threads] [--seed rngseed]\n"
	"\t\t  [--target hat_height target]\n"
	"\t\t  [--sqrt | --linear | --close | --ceiling | --log]\n";

const char* helpMsg =
	"OPTIONS\n"
	"\t -c chromosome_length\n"
	"\t\t Set chromosome length for the current simulation.\n"
	"\t\t Default chromosome length is 10.\n\n"
	"\t -e mutation_effect\n"
	"\t\t Set how much a mutation will effect a gene on average.\n"
	"\t\t Default mutation effect is 2.\n\n"
	"\t -g num_generations\n"
	"\t\t Set how many generations this simulation will run for.\n"
	"\t\t Default number of generations is 1000.\n\n"
	"\t -h\t Display this help menu.\n\n"
	"\t -m mutation_rate\n"
	"\t\t Set the mutation rate for the current simulation.\n"
	"\t\t Default mutation rate is 1.\n\n"
	"\t -o crossover_rate\n"
	"\t\t Set the crossover rate for the current simulation.\n"
	"\t\t Default crossover rate is 2.\n\n"
	"\t -p population_size\n"
	"\t\t Set the population size for the current simulation.\n"
	"\t\t Default population size is 10.\n"
	"\t -t num_threads\n"
	"\t\t Select the number of threads to be used in the current run.\n"
	"\t\t Default is 0 (which will result in 3/4 of cores being used).\n"
	"\t\t Must be 1 if a seed is used in order to prevent race conditions.\n\n"
	"\t --seed rngseed\n"
	"\t\t Select the seed used by the RNG in the current run.\n"
	"\t\t Default seed is 0 (which will result in a random seed).\n\n"
	"\t --target hat_height target\n"
	"\t\t Sets the ideal hat height for the current simulation\n"
	"\t\t Used for fitness functions that have an \"ideal\" value.\n\n"
	"\t --sqrt\t\t fitness will be sqrt(hat_height)\n\n"
	"\t --linear\t fitness will be hat_height\n\n"
	"\t --close\t fitness will be (target - abs(target - hat_height))\n\n"
	"\t --ceiling\t fitness will quickly level off after passing target\n\n";

pthread_mutex_t seedLock = PTHREAD_MUTEX_INITIALIZER;
unsigned long rngseed = 0;

//	there is no need for the line 'int chrom_size' as it is declared as a global variable in degnome.h
int pop_size;
int num_gens;
int mutation_rate;
int mutation_effect;
int crossover_rate;

int num_threads = 0;
JobQueue* jq;

void *ThreadState_new(void *notused);
void ThreadState_free(void *rng);

void *ThreadState_new(void *notused) {
	// Lock seed, initialize random number generator, increment seed,
	// and unlock.
	gsl_rng *rng = gsl_rng_alloc(gsl_rng_taus);

	pthread_mutex_lock(&seedLock);
	gsl_rng_set(rng, rngseed);
	rngseed = (rngseed == ULONG_MAX ? 0 : rngseed + 1);
	pthread_mutex_unlock(&seedLock);

	return rng;
}

void ThreadState_free(void *rng) {
	gsl_rng_free((gsl_rng *) rng);
}

int jobfunc(void* p, void* tdat) {
	gsl_rng* rng = (gsl_rng*) tdat;
	JobData* data = (JobData*) p;																				//get data out
	Degnome_mate(data->child, data->p1, data->p2, rng, mutation_rate, mutation_effect, crossover_rate);			//mate

	return 0;		//exited without error
}

void usage(void) {
	fputs(usageMsg, stderr);
	exit(EXIT_FAILURE);
}

void help_menu(void) {
	fputs(helpMsg, stderr);
	exit(EXIT_FAILURE);
}

int main(int argc, char **argv) {

	int * flags = NULL;

	if (parse_flags(argc, argv, 1, &flags) == -1) {
		free(flags);
		usage();
	}

	if (flags[2] == 1) {
        free(flags);
		help_menu();
	}

	chrom_size = flags[6];
	mutation_effect = flags[7];
	num_gens = flags[8];
	mutation_rate = flags[9];
	crossover_rate = flags[10];
	pop_size = flags[11];

	num_threads = flags[12];

	if(flags[13] == 0){
		set_function("linear");
	}
	else if(flags[13] == 1){
		set_function("sqrt");
	}
	else if(flags[13] == 2){
		set_function("close");
	}
	else if(flags[13] == 3){
		set_function("ceiling");
	}
	else if(flags[13] == 4){
		set_function("log");
	}
	target_num = flags[14];

	if(flags[15] <= 0) {
		time_t currtime = time(NULL);                  // time
		unsigned long pid = (unsigned long) getpid();  // process id
		rngseed = currtime ^ pid;                      // random seed
	}
	else{
		rngseed = flags[15];
	}
	gsl_rng* rng = gsl_rng_alloc(gsl_rng_taus);    // rand generator
	gsl_rng_set(rng, rngseed);

	free(flags);

	if (num_threads <= 0) {
		if (num_threads < 0) {
			#ifdef DEBUG_MODE
				fprintf(stderr, "Error invalid number of threads: %u\n", num_threads);
			#endif
		}
		num_threads = (3*getNumCores()/4);
	}
	#ifdef DEBUG_MODE
		fprintf(stderr, "Final number of threads: %u\n", num_threads);
	#endif

	Degnome* parents;
	Degnome* children;
	Degnome* temp;

	printf("%u, %u, %u\n", chrom_size, pop_size, num_gens);

	parents = malloc(pop_size*sizeof(Degnome));
	children = malloc(pop_size*sizeof(Degnome));

	for (int i = 0; i < pop_size; i++) {
		parents[i].dna_array = malloc(chrom_size*sizeof(double));
		children[i].dna_array = malloc(chrom_size*sizeof(double));
		parents[i].hat_size = 0;

		for (int j = 0; j < chrom_size; j++) {
			parents[i].dna_array[j] = (i+j);	//children isn't initiilized
			parents[i].hat_size += (i+j);
		}
	}

	printf("Generation 0:\n");
	for (int i = 0; i < pop_size; i++) {
		printf("Degnome %u\n", i);
		for (int j = 0; j < chrom_size; j++) {
			printf("%lf\t", parents[i].dna_array[j]);
		}
		printf("\nTOTAL HAT SIZE: %lg\n\n", parents[i].hat_size);
	}

	jq = JobQueue_new(num_threads, NULL, ThreadState_new, ThreadState_free);

	JobData* dat = malloc(pop_size*sizeof(JobData));

	for (int i = 0; i < num_gens; i++) {

		double fit = get_fitness(parents[0].hat_size);

		double total_hat_size = fit;
		double cum_hat_size[pop_size];
		cum_hat_size[0] = fit;

		for (int j = 1; j < pop_size; j++) {
			fit = get_fitness(parents[j].hat_size);

			total_hat_size += fit;
			cum_hat_size[j] = (cum_hat_size[j-1] + fit);
		}

		for (int j = 0; j < pop_size; j++) {

			int m, d;

			double win_m = gsl_rng_uniform(rng);
			win_m *= total_hat_size;
			double win_d = gsl_rng_uniform(rng);
			win_d *= total_hat_size;

			// printf("win_m:%lf, wind:%lf, max: %lf\n", win_m,win_d,total_hat_size);

			for (m = 0; cum_hat_size[m] < win_m; m++) {
				continue;
			}

			for (d = 0; cum_hat_size[d] < win_d; d++) {
				continue;
			}

			// printf("m:%u, d:%u\n", m,d);
			dat[j].child = (children + j);
			dat[j].p1 = (parents + m);
			dat[j].p2 = (parents + d);

			JobQueue_addJob(jq, jobfunc, dat + j);
		}

		JobQueue_waitOnJobs(jq);
		temp = children;
		children = parents;
		parents = temp;
	}

	JobQueue_noMoreJobs(jq);

	printf("Generation %u:\n", num_gens);
	for (int i = 0; i < pop_size; i++) {
		printf("Degnome %u\n", i);
		for (int j = 0; j < chrom_size; j++) {
			printf("%lf\t", parents[i].dna_array[j]);
		}
		printf("\nTOTAL HAT SIZE: %lg\n\n", parents[i].hat_size);
	}

	//free everything

	JobQueue_free(jq);
	free(dat);

	for (int i = 0; i < pop_size; i++) {
		free(parents[i].dna_array);
		free(children[i].dna_array);
		parents[i].hat_size = 0;
	}

	free(parents);
	free(children);

	gsl_rng_free (rng);
}


#ifndef __DIRECT_KNN_H__
#define __DIRECT_KNN_H__

#include <mpi.h>
#include <utility>
#include <cmath>
#include <vector>
#include <string>
#include <blas.h>
#include <omp.h>
#include <utility>

#define KNN_MAX_BLOCK_SIZE 512
#define KNN_MAX_MATRIX_SIZE 2e7L

// MPI message tags
enum {
  TAG_R,
  TAG_Q,
  TAG_KMIN,
  TAG_ID,
  TAG_SIZE
};

enum {
  VERTICAL,
  HORIZONTAL
};

/**
 * Options struct used to specify how a rectangular-partitioned query should be performed.
 */
struct directQueryParams
{
	char queryType; ///< Perform a k-nearest ('K') or r-near ('R') search.
	int k; ///< Number of neighbors to find (if queryType is 'K').
	double r; ///< Radius for r-near search (if queryType is 'R').
	int refParts; ///< Partition reference points into this many approximately equal parts.
	int queryParts; ///< Partition query points into this many approximately equal parts.
};

struct directQueryResults
{
	std::pair<double, long> *neighbors;
	long *neighborCounts;
};

class maxheap_comp {
    public:
        bool operator() (const std::pair<double, long> &a, const std::pair<double, long> &b) {
            double diff = fabs(a.first-b.first)/a.first;
            if( std::isinf(diff) || std::isnan(diff) ) {      // 0/0 or x/0
                return a.first < b.first;
            }
            if( diff < 1.0e-8 ) {
                return a.second < b.second;
            }
            return a.first < b.first;
        }
};

namespace knn {

   /**
    * Compute the k nearest neighbors of each query point (OBSOLETE: For debugging purposes only; use directKQueryLowMem instead)..
    * \param ref an n*dim-length array of data points.
    * \param query an m*dim-length array of query points.
    * \param n number of local reference point
    * \param m number of local query points
    * \param k number of neighbors to find for each query point.
    * \param dim dimensionality of reference/query points
    * \return An array of pairs containing the *squared* distance to each point and its
    *	index in the local array.  Note that this is the index of the /point/ rather
    *	than the index of its location in the actual array of doubles.  To access the
    *	point, use (point index)*dim to index ref.
    */
   std::pair<double, long> *directKQuery
              ( double *ref, double *query, long n, long m, long k, int dim);

   /**
    * Compute the k nearest neighbors of each query point (OBSOLETE: For debugging purposes only; use directKQueryLowMem instead)..
    * \param ref an n*dim-length array of data points.
    * \param query an m*dim-length array of query points.
    * \param n number of local reference point
    * \param m number of local query points
    * \param k number of neighbors to find for each query point.
    * \param dim dimensionality of reference/query points
    * \param result [out] An array of m*k pairs (allocated by caller) containing the
    * *squared* distance to each point and its index in the local array.
    * Note that this is the index of the /point/ rather than the index of its location
    * in the actual array of doubles.  To access the point, use (point index)*dim to index ref.
    * Results for query point i begin at result[i*k].
    * \param dist An array used for calculating inter-point distances. If NULL or unspecified,
    * the function allocates and deallocates this array internally.  Otherwise, it must be
    * allocated size at least n*B, where B is the value returned by getBlockSize, and
    * deallocated at some point after the function returns.
    * \param sqnormr An array used for calculating inter-point distances. If NULL or unspecified,
    * the function allocates and deallocates this array internally.  Otherwise, it must be
    * allocated size at least n and deallocated at some point after the function returns.
    * \param sqnormq An array used for calculating inter-point distances. If NULL or unspecified,
    * the function allocates and deallocates this array internally.  Otherwise, it must be
    * allocated size at least B, where B is the value returned by getBlockSize, and
    * deallocated at some point after the function returns.
    */
 void directKQuery_small_a2a( double *ref, long n, int dim, int k,
                              std::pair<double, long> *result,
                              double *dist = NULL, double *sqnormr = NULL, double *sqnormq = NULL);

   /**
    * Compute the k nearest neighbors of each query point with a smaller memory footprint and usually better performance.
    * \param ref an n*dim-length array of data points.
    * \param query an m*dim-length array of query points.
    * \param n number of local reference point
    * \param m number of local query points
    * \param k number of neighbors to find for each query point.
    * \param dim dimensionality of reference/query points
    * \param result [out] An array of m*k pairs (allocated by caller) containing the
    * *squared* distance to each point and its
    * index in the local array.  Note that this is the index of the /point/ rather
    * than the index of its location in the actual array of doubles.  To access the
    * point, use (point index)*dim to index ref. Results for query point i begin at result[i*k].
    * \param dist An array used for calculating inter-point distances. If NULL or unspecified,
    * the function allocates and deallocates this array internally.  Otherwise, it must be
    * allocated size at least n*B, where B is the value returned by getBlockSize, and
    * deallocated at some point after the function returns.
    * \param sqnormr An array used for calculating inter-point distances. If NULL or unspecified,
    * the function allocates and deallocates this array internally.  Otherwise, it must be
    * allocated size at least n and deallocated at some point after the function returns.
    * \param sqnormq An array used for calculating inter-point distances. If NULL or unspecified,
    * the function allocates and deallocates this array internally.  Otherwise, it must be
    * allocated size at least B, where B is the value returned by getBlockSize, and
    * deallocated at some point after the function returns.
    */
   void directKQueryLowMem
              ( double *ref, double *query, long n, long m, long k, int dim, std::pair<double, long> *result,
                double *dist = NULL, double* sqnormr = NULL, double* sqnormq = NULL  );

   /**
    * Find all points in ref that lie within distance sqrt(R) of each
    * point in query.
    * \param ref an n*dim-length array of data points.
    * \param query an m*dim-length array of query points.
    * \param n number of local reference point
    * \param m number of local query points
    * \param R The *square* of the search radius.
    * \param dim dimensionality of reference/query points
    * \param glob_ids An array of length n containing the global id of each ref point.
    * \param neighbor_count [out] An array of length m containing the number of
    * neighbors found for the corresponding query point (allocated internally).
    * \param neighbors [out] An array of length sum(neighbor_count) containing
    * pairs of ids and squared distances for each neighbor found (allocated
    * internally).
    */
   void directRQuery
           ( double *ref, double *query, long n, long m, double R, int dim, long* glob_ids,
             int **neighbor_count, std::pair<double, long> **neighbors );

  /**
   * Find all points in ref that lie within distance sqrt(R[i]) of query point i.
    * \param ref an n*dim-length array of data points.
    * \param query an m*dim-length array of query points.
   * \param n number of local reference point
   * \param m number of local query points
   * \param R Array of length m, the *square* of the search radius for each point.
   * \param dim dimensionality of reference/query points
   * \param glob_ids An array of length n containing the global id of each ref point.
   * \param neighbor_count [out] An array of length m containing the number of
   * neighbors found for the corresponding query point (allocated internally).
   * \param neighbors [out] An array of length sum(neighbor_count) containing
   * pairs of ids and squared distances for each neighbor found (allocated
   * internally).
   */
  void directRQueryIndividual
  ( double *ref, double *query, long n, long m, double *R, int dim, long* glob_ids,
   int **neighbor_count, std::pair<double, long> **neighbors );

  /**
   * Find all points in ref that lie within distance sqrt(R[i]) of query point i.
    * \param ref an n*dim-length array of data points.
    * \param query an m*dim-length array of query points.
   * \param n number of local reference point
   * \param m number of local query points
   * \param R Array of length m, the *square* of the search radius for each point.
   * \param dim dimensionality of reference/query points
   * \param glob_ids An array of length n containing the global id of each ref point.
   * \param neighbor_count [out] An array of length m containing the number of
   * neighbors found for the corresponding query point (allocated internally).
   * \param neighbors [out] An array of length sum(neighbor_count) containing
   * pairs of ids and squared distances for each neighbor found (allocated
   * internally).
   */
void directRQueryIndividualK
( double *ref, double *query, long n, long m, int k, double *R, int dim, long *glob_ids,
 int **neighbor_count, std::pair<double, long> **neighbors  );

   /**
    * Computes the distances between all query points and all reference points.
    * \param ref an n*dim-length array of data points.
    * \param query an m*dim-length array of query points.
    * \param n number of reference point
    * \param m number of query points
    * \param dim dimensionality of reference/query points
    * \param dist [out] m*n matrix of the *squared* distances from query points to reference points.
    * \param sqnormr An array of length n used for storing the squared norm of each reference point.
    * If NULL or omitted, allocated and deallocated internally.
    * \param sqnromq An array of length m used for storing the squared norm of each reference point.
    * If NULL or omitted, allocated and deallocated internally.
    * \param useSqnormrInput If true, use precomputed squared norm values contained in sqnormr when
    * this function is called; if false, compute squared norms internally.
    */
   void compute_distances
              ( double *ref, double *query, long n, long m, int dim, double *dist,
                  double* sqnormr = NULL, double* sqnormq = NULL, bool useSqnormrInput = false );

   /**
    * Computes the square of the norm of each row of a matrix.
    * \param a The input matrix (row-major).
    * \param n The number of rows of a.
    * \param dim The number of columns of a.
    * \param b [out] The output array (length n).
    */
   void sqnorm ( double *a, long n, int dim, double *b);

   /**
    * Find the k-nearest neighbors of each query point using distributed cyclic-shift algorithm.
    * \param ref an nlocal*dim-length array of data points.
    * \param query an mlocal*dim-length array of query points.
    * \param glob_ids An array of length nlocal containing the unique ID of each reference point stored locally.
    * \param nlocal Number of local reference points.
    * \param mlocal Number of local query points.
    * \param k Number of neighbors to find for each query point.
    * \param dim Dimensionality of reference/query points.
    * \param comm The MPI communicator to use.
    * \return An array of pairs containing the distance to each neighbor and its index;
    * neighbors for point i begin at location i*k.
    * \note All input arrays MUST be allocated with malloc (or equivalent) and deallocated with free;
    * new and delete[] may result in a segmentation fault.
    */
   std::pair<double, long> * dist_directKQuery( double* &ref, double *query,
   						 long* &glob_ids,
   						 int nlocal, int mlocal,
   						 int k,
   						 int dim,
   						 MPI_Comm comm );

   /**
    * Find all reference points lying within distance sqrt(R) of each query point.
    * \param ref an nlocal*dim-length array of data points.
    * \param query an mlocal*dim-length array of query points.
    * \param n Number of global reference points
    * \param m Number of global query points
    * \param nlocal Number of local reference points
    * \param mlocal Number of local query points
    * \param R The *square* of the search radius.
    * \param dim Dimensionality of reference/query points
    * \param glob_ids An array of length nlocal containing the unique ID of each reference point stored locally.
    * \param rneighbors [out] An externally allocated array of mlocal vectors, used to store the neighbors of each
    * query point.
    * \param comm The MPI communicator to use.
    */
    void dist_directRQuery
              ( double* &ref, double *query, long n, long m, int nlocal, int mlocal, double R,
                int dim, long* &glob_ids, std::vector< std::pair<double, long> >* rneighbors,
                MPI_Comm comm  );

   /**
    * Find all reference points lying within distance sqrt(R) of each query point,
    * up to a maximum of max_neighbors (not necessarily the nearest).
    * \param ref an nlocal*dim-length array of data points.
    * \param query an mlocal*dim-length array of query points.
    * \param n Number of global reference points
    * \param m Number of global query points
    * \param nlocal Number of local reference points
    * \param mlocal Number of local query points
    * \param R The *square* of the search radius.
    * \param dim Dimensionality of reference/query points
    * \param glob_ids An array of length nlocal containing the unique ID of each reference point stored locally.
    * \param rneighbors [out] An externally allocated array of mlocal vectors, used to store the neighbors of each
    * query point.
    * \param max_neighbors The maximum number of neighbors returned for any given query point.
    * \param comm The MPI communicator to use.
    */
    void dist_directRQuery
    ( double* &ref, double *query, long n, long m, int nlocal, int mlocal, double R,
     int dim, long* &glob_ids, std::vector< std::pair<double, long> >* rneighbors, int max_neighbors,
     MPI_Comm comm  );

  /**
   * Find all reference points lying within distance sqrt(R[i]) of query point i.
   * \param ref an nlocal*dim-length array of data points.
   * \param query an mlocal*dim-length array of query points.
   * \param n Number of global reference points
   * \param m Number of global query points
   * \param nlocal Number of local reference points
   * \param mlocal Number of local query points
   * \param R Array of length m, the *square* of the search radius for each query point.
   * \param dim Dimensionality of reference/query points
   * \param glob_ids An array of length nlocal containing the unique ID of each reference point stored locally.
   * \param rneighbors [out] An externally allocated array of mlocal vectors, used to store the neighbors of each
   * query point.
   * \param comm The MPI communicator to use.
   */
  void dist_directRQueryIndividual
  ( double* &ref, double *query, long n, long m, int nlocal, int mlocal, double *R,
   int dim, long* &glob_ids, std::vector< std::pair<double, long> >* rneighbors,
   MPI_Comm comm  );

  /**
   * Find all reference points lying within distance sqrt(R[i]) of query point i.
   * \param ref an nlocal*dim-length array of data points.
   * \param query an mlocal*dim-length array of query points.
   * \param n Number of global reference points
   * \param m Number of global query points
   * \param nlocal Number of local reference points
   * \param mlocal Number of local query points
   * \param R Array of length m, the *square* of the search radius for each query point.
   * \param dim Dimensionality of reference/query points
   * \param glob_ids An array of length nlocal containing the unique ID of each reference point stored locally.
   * \param rneighbors [out] An externally allocated array of mlocal vectors, used to store the neighbors of each
   * query point.
   * \param comm The MPI communicator to use.
   */
   void dist_directRQueryIndividualK
( double* &ref, double *query, long n, long m, int nlocal, int mlocal, double *R, int k,
 int dim, long* &glob_ids, std::vector< std::pair<double, long> >* rneighbors,
 MPI_Comm comm  );

  /**
   * Find all reference points lying within distance sqrt(R[i]) of query point i,
   * up to a maximum of max_neighbors points (not necessarily the nearest).
   * \param ref an nlocal*dim-length array of data points.
   * \param query an mlocal*dim-length array of query points.
   * \param n Number of global reference points
   * \param m Number of global query points
   * \param nlocal Number of local reference points
   * \param mlocal Number of local query points
   * \param R Array of length m, the *square* of the search radius for each query point.
   * \param dim Dimensionality of reference/query points
   * \param glob_ids An array of length nlocal containing the unique ID of each reference point stored locally.
   * \param rneighbors [out] An externally allocated array of mlocal vectors, used to store the neighbors of each
   * query point.
   * \param max_neighbors The maximum number of neighbors to return for any single query point.
   * \param comm The MPI communicator to use.
   */
   void dist_directRQueryIndividual
   ( double* &ref, double *query, long n, long m, int nlocal, int mlocal, double *R,
    int dim, long* &glob_ids, std::vector< std::pair<double, long> >* rneighbors, int max_neighbors,
    MPI_Comm comm  );

   /**
    * Merges a and b and keeps only the k smallest from each of the m sets of distance/index pairs.
    * \param a one array of m*k distance+index pairs
    * \param b the other array of m*k distance+index pairs
    * \param m the number sets of size k
    * \param k k
    * \return A newly allocated array of size m*k
    */
   std::pair<double, long> *kmin_merge( std::pair<double, long> *a, std::pair<double, long> *b, long m, long k );

   /**
    * Reads reference and query points from files in parallel and partitions them for a query with the rectangular algorithm..
    * \param refFile The name of the file which contains the reference points.
    * \param queryFile The name of the file which contains the query points.
    * \param ref [out] an n*dim array of data points.
    * \param query [out] an m*dim array of query points.
    * \param n number of reference points.
    * \param m number of query points.
    * \param dim dimensionality of reference/query points.
    * \param refParts number of parts to split reference points into (refParts * queryParts = processes).
    * \param queryParts number of parts to split query points into (refParts * queryParts = processes).
    * \param comm The MPI communicator to use.
    * \return A pair object containing the number of reference points (first) and query points (second) stored at this process.
    */
   std::pair<int, int> partitionPoints(std::string refFile, std::string queryFile, double *& ref, double *& query, long n, long m, int dim, int refParts, int queryParts, MPI_Comm comm);

   /**
    * Returns the number of local elements in the partitioning used by dist_directKQuery.
    * \param rank This process's MPI rank.
    * \param size The size of the MPI communicator used.
    * \param num The global number of elements.
    * \return The number of elements this process stores locally.
    */
   inline int getNumLocal( int rank, int size, long num ) {
      if( rank < num%size)
         return std::ceil( (double)num / (double)size );
      else
         return num / size;
   }

   /**
    * Returns the offset of the locally stored elements in the global array.
    * \param rank This process's MPI rank.
    * \param size The size of the MPI communicator used.
    * \param num The global number of elements.
    * \return The global array offset.
    */
   inline long getGlobalArrayOffset( int rank, int size, long num ) {
      if( rank < num%size )
         return std::ceil( (double)num / (double)size ) * rank;
      else
         return std::ceil( (double)num / (double)size ) * (num%size) + (rank-(num%size)) * (num/size);
   }

   /**
    * Repartitions points with replication.
    * \param points The points to repartition (modified).
    * \param localPointCount The number of points the process has.
    * \param dim The dimensionality of the points.
    * \param globalIDs The global ID associated with each point (modified).
    * \param partitionCount The number of pieces to partition the points into (the number of replications is implicitly given by size / partitionCount)
    * \param comm The communicator to use.
    * \return The new number of points the process has.
   */
   int directRectRepartition(double *&points, int localPointCount, int dim, int *&globalIDs, int partitionCount, int direction, MPI_Comm comm);

   /**
    * Performs a direct kNN or rNN query using the rectangular-partitioned algorithm, automatically repartitioning points..
    * \param refPts The reference points (modified).
    * \param queryPts The query points (modified).
    * \param localRefCount The number of reference points the process has (modified).
    * \param localQueryCount The number of query points the process has (modified).
    * \param dim The dimensionality of the points.
    * \param refIDs The global IDS of the reference points the process has (modified).
    * \param queryIDs The global IDs of the query points the process has (modified).
    * \param params The parameters of the direct kNN or rNN query.
    * \param comm The communicator to use.
    * \return The result of the query.
   */
   directQueryResults directRectRepartitionAndQuery(double *&refPts, double *&queryPts, long& localRefCount, long& localQueryCount, int dim, int *&refIDs, int *&queryIDs, directQueryParams params, MPI_Comm comm);

   /**
    * Performs a rNN query on a rectangularly partitioned set of points.
    * \param refPoints The reference points.
    * \param queryPoints The query points.
    * \param localRefCount The number of reference points the process has.
    * \param localQueryCount The number of query points the process has.
    * \param dim The dimensionality of the points.
    * \param r The search radius of the query.
    * \param queryComm The communicator to use (all processes within this comm should have the same query points).
    * \return The result of the query.
   */
   directQueryResults directQueryRectR(double *refPoints, double *queryPoints, int localRefCount, int localQueryCount, int dim, double r, MPI_Comm queryComm);

   /**
    * Performs a kNN query on a rectangularly partitioned set of points.
    * \param refPoints The reference points.
    * \param queryPoints The query points.
    * \param localRefCount The number of reference points the process has.
    * \param localQueryCount The number of query points the process has.
    * \param dim The dimensionality of the points.
    * \param k The number of neighbors to find for each query point.
    * \param queryComm The communicator to use (all processes within this comm should have the same query points).
    * \return The result of the query.
   */
   directQueryResults directQueryRectK(double *refPoints, double *queryPoints, int localRefCount, int localQueryCount, int dim, int k, MPI_Comm queryComm);

   /**
    * Determine the number of query points that will be handled by each iteration of the LowMem queries.
    * \param n The number of reference points.
    * \param m The number of query points.
    * \return The block size the query will use.  Can be multiplied by n to determine size of distance matrix.
    */
   inline int getBlockSize(long n, long m) {
     int blocksize;
     if( m > KNN_MAX_BLOCK_SIZE || n > 10000L) {
       blocksize = std::min((long)KNN_MAX_BLOCK_SIZE, m); //number of query points handled in a given iteration
       if(n * (long)blocksize > KNN_MAX_MATRIX_SIZE) blocksize = std::min((long)(KNN_MAX_MATRIX_SIZE/n), (long)blocksize); //Shrink block size if n is huge.
       blocksize = std::max(blocksize, omp_get_max_threads()); //Make sure each thread has some work.
     } else {
        blocksize = m;
     }
     return blocksize;
   }

}
#endif


#ifndef _density_constructions_h
#define _density_constructions_h

#include "montecarlo.h"
#include "two_body_analytical.h"
#include <float.h>
#include <limits.h>
#include <gsl/gsl_sort.h>

int r_final_uniform(double x, double y, double vz, double mu, double t_peri ,int N, double a, char * directory);
int Vector_distribution(double x, double y, double vz, double mu, double t_peri,  int N, double a, char * directory);

typedef struct{
  double v[6];

} state_Vector;

int Save_Vector_Index(state_Vector *state_vector, int index, int N, char* directory);
int Chi_Square_Euler(double x, double y, double vz, double mu,  int N, char * directory, size_t Npoints);

#endif  // ! _density_constructions_h


#ifndef RADFIELD_H
#define RADFIELD_H

#include <cstdio>
#include <gsl/gsl_integration.h>
#include "types.h"
#include "sn3d.h"

namespace radfield
{
  void zero_estimators(int modelgridindex);
  void init(int my_rank, int ndo, int ndo_nonempty);
  void initialise_prev_titer_photoionestimators(void);
  void write_to_file(int modelgridindex, int timestep);
  void close_file(void);
  __host__ __device__ void update_estimators(int modelgridindex, double distance_e_cmf, double nu_cmf, const PKT *pkt_ptr, double t_current);
  void increment_lineestimator(int modelgridindex, int lineindex, double increment);
  __host__ __device__ double radfield(double nu, int modelgridindex);
  __host__ __device__ double dbb_mgi(double nu, int modelgridindex);
  void fit_parameters(int modelgridindex, int timestep);
  __host__ __device__ void set_J_normfactor(int modelgridindex, double normfactor);
  __host__ __device__ void normalise_J(int modelgridindex, double estimator_normfactor_over4pi);
  __host__ __device__ void normalise_nuJ(int modelgridindex, double estimator_normfactor_over4pi);
  __host__ __device__ double get_T_R_from_J(int modelgridindex);
  __host__ __device__ int get_Jblueindex(int lineindex);
  __host__ __device__ double get_Jb_lu(int modelgridindex, int jblueindex);
  __host__ __device__ int get_Jb_lu_contribcount(int modelgridindex, int jblueindex);
  void titer_J(int modelgridindex);
  void titer_nuJ(int modelgridindex);
  void reduce_estimators(void);
  void do_MPI_Bcast(int modelgridindex, int root, int root_node_id);
  void write_restart_data(FILE *gridsave_file);
  void read_restart_data(FILE *gridsave_file);
  __host__ __device__ void normalise_bf_estimators(int modelgridindex, double estimator_normfactor_over_H);
  double get_bfrate_estimator(int element, int lowerion, int lower, int phixstargetindex, int modelgridindex);
  void print_bfrate_contributions(int element, int lowerion, int lower, int phixstargetindex, int modelgridindex, double nnlowerlevel, double nnlowerion);
  void reset_bfrate_contributions(const int modelgridindex);
  int integrate(
    const gsl_function *f, double nu_a, double nu_b, double epsabs, double epsrel,
    size_t limit, int key, gsl_integration_workspace *workspace, double *result, double *abserr);

  __host__ __device__ inline double dbb(double nu, float T, float W)
  // returns J_nu for a dilute black body [ergs/s/sr/cm2/Hz]
  {
    return W * TWOHOVERCLIGHTSQUARED * pow(nu, 3) / expm1(HOVERKB * nu / T);
  }
}

#endif //RADFIELD_H