//cmc_topol.c --
// this uses stateSpace generated previously to
// create the topology matrix and the move type matrix
//defined by the markov chain

//this was only for testing purposes

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <gsl/gsl_matrix.h>
 #include <gsl/gsl_blas.h>

#include "AFS.h"
#include "adkGSL.h"
#include "cs.h"
#include "time.h"
#include "adkCSparse.h"
#include "AFS_pthreads.c"

#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y))
#define NTHREADS 4

int n1, n2;
const char *filename;

afsStateSpace *stateSpace;
void getParameters(int argc,const char **argv);
void usage();

int main(int argc, const char * argv[]){
	int i,N,tmpS,win,t;
	int rc;
	void *v;
	int nnz;
	matrixThreadObject_sparse *tObj[NTHREADS];
 	pthread_t threads[NTHREADS];

	getParameters(argc,argv);
	stateSpace = afsStateSpaceImportFromFile(filename);
	//afsStateSpaceRemoveAbsorbing(stateSpace);
	N = stateSpace->nstates;

	//Multithreaded construction of original transition matrix
//	printf("initializing original transition matrix using %d threads....\n",NTHREADS);
	//setup threads
	tmpS=0;
	win =(int) ceil( (((float)stateSpace->nstates) / NTHREADS) );
//	printf("nstates: %d winSize: %d\n",stateSpace->nstates,win);
	for(t=0;t<NTHREADS;t++){
		tObj[t] = malloc(sizeof( matrixThreadObject_sparse));
		tObj[t]->nnz = 0;
		tObj[t]->dim1 = malloc(sizeof(int) * stateSpace->nstates * 3);
		tObj[t]->dim2 = malloc(sizeof(int) * stateSpace->nstates * 3);
		tObj[t]->moveType = malloc(sizeof(int) * stateSpace->nstates *  3);
		tObj[t]->topol = malloc(sizeof(double) * stateSpace->nstates *  3);
		tObj[t]->start = tmpS;
		tObj[t]->stop = MIN(tmpS+win,stateSpace->nstates);
		tObj[t]->im_params = malloc(sizeof(im_lik_params));
		tObj[t]->im_params->stateSpace = stateSpace;
//		printf("start: %d stop: %d\n",tmpS,tmpS+win);
		rc = pthread_create(&threads[t], NULL, threadDoRows_sparse, (void *)tObj[t]);
		tmpS += win;
	}
//	printf("threads alloced....\n");

	//harvest jobs
	nnz=0;
	for(t=0;t<NTHREADS;t++){
		rc = pthread_join(threads[t], &v);
		tObj[t] = ((matrixThreadObject_sparse *)v);
		nnz += tObj[t]->nnz;
	}
	//now combine outputs for nnz, dim1, and dim2
	printf("nnz: %d\n",nnz);
	for(t=0;t<NTHREADS;t++){
		for(i=0;i<tObj[t]->nnz;i++){
			printf("%lf\t%d\t%d\t%d\n",tObj[t]->topol[i],tObj[t]->moveType[i],tObj[t]->dim1[i],tObj[t]->dim2[i]);
		}
	}
	//currentParams->nnz -= 1;
	////////// Multithreading done!

	// dim1 = malloc(sizeof(int) * N * 10);
	// dim2 = malloc(sizeof(int) * N * 10);
	// moveA = malloc(sizeof(int) * N * 10);
	// topA = malloc(sizeof(double) * N * 10);
	// for(i=0;i< (N*10);i++){
	// 	moveA[i]=0;
	// 	topA[i]=0;
	// }
	// nnz = coalMarkovChainTopologyMatrix_sparse(stateSpace,topA, moveA, dim1, dim2);
	// printf("nnz: %d\n",nnz);
	// for(i=0;i<nnz;i++){
	// 	printf("%f\t%d\t%d\t%d\n",topA[i],moveA[i],dim1[i],dim2[i]);
	// }
	return(0);
}

void getParameters(int argc, const char **argv){
//	int args;
//	int i;

	if( argc <= 1){
		usage();
	}
	filename = argv[1];

}

void usage(){
	fprintf(stderr,"usage: cmc_topol stateSpaceFile\n");
//	fprintf(stderr,"parameters: \n");
	exit(1);
}


/*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*
**                                                                                **
** This file forms part of the Underworld geophysics modelling application.         **
**                                                                                **
** For full license and copyright information, please refer to the LICENSE.md file  **
** located at the project root, or contact the authors.                             **
**                                                                                **
**~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*/
#include <petsc.h>
#include <petscmat.h>
#include <petscvec.h>
#include <petscksp.h>
#include <petscpc.h>
#include <petscversion.h>
#if ( (PETSC_VERSION_MAJOR >= 3) && (PETSC_VERSION_MINOR >=3) )
  #if (PETSC_VERSION_MINOR >=6)
     #include <petsc/private/kspimpl.h>
  #else
     #include <petsc-private/kspimpl.h>
  #endif
#else
  #include <private/kspimpl.h>
#endif
#include <StGermain/StGermain.h>
#include <StgDomain/StgDomain.h>
#include <StgFEM/StgFEM.h>
#include <PICellerator/PICellerator.h>
#include <Underworld/Underworld.h>
#include "Solvers/SLE/SLE.h" /* to give the AugLagStokes_SLE type */
#include "Solvers/KSPSolvers/KSPSolvers.h"
#include "BSSCR.h"
#include "stokes_block_scaling.h"
#include "stokes_mvblock_scaling.h"
#include "writeMatVec.h"
#include "createK2.h"
#include "mg.h"
#include "summary.h"
#include "ksp_pressure_nullspace.h"

#define BSSCR_GetPetscMatrix( matrix ) ( (Mat)(matrix) )
#define BSSCR_GetPetscVector( vector ) ( (Vec)(vector) )

typedef struct {
    PetscTruth initialrtol;    /* default relative residual decrease is computing from initial residual, not rhs */
    PetscTruth mininitialrtol; /* default relative residual decrease is computing from min of initial residual and rhs */
    Vec        work;
    PetscReal  pointwise_max;
} KSPPWConvergedCtx;

PetscErrorCode BSSCR_KSPNormInfMonitor( KSP ksp, PetscInt iteration, PetscReal residualNorm, void *dummy);
PetscErrorCode BSSCR_KSPNormInfToNorm2Monitor( KSP ksp, PetscInt iteration, PetscReal residualNorm, void *dummy);

PetscErrorCode BSSCR_KSPPWConvergedCreate(void **ctx);
PetscErrorCode BSSCR_KSPPWConvergedDestroy(void *ctx);
PetscErrorCode BSSCR_KSPNormInfConverged(KSP ksp,PetscInt n,PetscReal rnorm,KSPConvergedReason *reason,void *ctx);
PetscErrorCode BSSCR_KSPNormInfToNorm2Monitor(KSP ksp,PetscInt n,PetscReal rnorm, void *dummy);
PetscErrorCode BSSCR_KSPNormInfMonitor(KSP ksp,PetscInt n,PetscReal rnorm, void *dummy);
PetscErrorCode BSSCR_KSPNorm2RawMonitor(KSP ksp,PetscInt n,PetscReal rnorm, void *dummy);
PetscErrorCode BSSCR_KSPSetNormInfConvergenceTest(KSP ksp);

#undef __FUNCT__
#define __FUNCT__ "BSSCR_DRIVER_auglag"
PetscErrorCode BSSCR_DRIVER_auglag( KSP ksp, Mat stokes_A, Vec stokes_x, Vec stokes_b, Mat approxS,
                                          MatStokesBlockScaling BA, PetscTruth sym, KSP_BSSCR * bsscrp_self )
{
    PetscTruth uzawastyle, KisJustK=PETSC_TRUE, restorek, change_A11rhspresolve;
    PetscTruth usePreviousGuess, useNormInfStoppingConditions, useNormInfMonitor, found, forcecorrection;
    PetscTruth change_backsolve, mg_active, get_flops;
    PetscErrorCode ierr;
    KSPConvergedReason reason;

    PetscInt max_it,min_it;
    KSP ksp_inner, ksp_S, ksp_new_inner;
    PC pc_S, pcInner;
    Mat K,G,D,C, S, K2;
    Vec u,p,f,f2=0,f3=0,h, h_hat,t;
    Vec f_tmp;

    MGContext mgCtx;
    double mgSetupTime, problemBuildTime, scrSolveTime, RHSSolveTime, a11SingleSolveTime, penaltyNumber;// hFactor;
    double backsolveSetupTime, scrSetupTime, RHSSetupTime;
    int been_here = bsscrp_self->been_here;

    char name[PETSC_MAX_PATH_LEN];
    char suffix[PETSC_MAX_PATH_LEN];
    char str[PETSC_MAX_PATH_LEN];
    PetscTruth flg, extractMats;
    PetscLogDouble flopsA,flopsB;

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    //if( bsscrp_self->solver->st_sle->context->loadFromCheckPoint ){
    //    been_here=1;
    //}
    /* get sub matrix / vector objects */
    /* note that here, the matrix D should always exist. It is set up in  _StokesBlockKSPInterface_Solve in StokesBlockKSPInterface.c */
    /* now extract K,G etc from a MatNest object */
    MatNestGetSubMat( stokes_A, 0,0, &K );
    MatNestGetSubMat( stokes_A, 0,1, &G );
    MatNestGetSubMat( stokes_A, 1,0, &D );if(!D){ PetscPrintf( PETSC_COMM_WORLD, "D does not exist but should!!\n"); exit(1); }
    MatNestGetSubMat( stokes_A, 1,1, &C );
    VecNestGetSubVec( stokes_x, 0, &u );
    VecNestGetSubVec( stokes_x, 1, &p );
    VecNestGetSubVec( stokes_b, 0, &f );
    VecNestGetSubVec( stokes_b, 1, &h );

    // Try this ...
    // MatMPIAIJSetPreallocation(K, 375 ,PETSC_NULL, 375, PETSC_NULL);

    PetscPrintf( PETSC_COMM_WORLD, "AUGMENTED LAGRANGIAN K2 METHOD " );
    PetscPrintf( PETSC_COMM_WORLD, "- Penalty = %f\n\n", bsscrp_self->solver->penaltyNumber );
    sprintf(suffix,"%s","x");

    found = PETSC_FALSE;
    get_flops = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-get_flops", &get_flops, &found);

    PetscOptionsGetString( PETSC_NULL, "-matsuffix", suffix, PETSC_MAX_PATH_LEN-1, &extractMats );

    flg=0;
    PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
    if(flg){
        sprintf(str,"%s/K%s",name, suffix);
        bsscr_writeMat( K, str, "Writing K matrix in al Solver");
        sprintf(str,"%s/G%s",name, suffix);
        bsscr_writeMat( G, str, "Writing G matrix in al Solver");
        sprintf(str,"%s/D%s",name, suffix);
        bsscr_writeMat( D, str, "Writing D matrix in al Solver");
        sprintf(str,"%s/f%s",name, suffix);
        bsscr_writeVec( f, str, "Writing f vector in al Solver");
        sprintf(str,"%s/h%s",name, suffix);
        bsscr_writeVec( h, str, "Writing h vector in al Solver");
        sprintf(str,"%s/Shat%s",name,suffix);
        bsscr_writeMat( approxS, str, "Writing Shat matrix in al Solver");
        if(C){
          sprintf(str,"%s/C%s",name,suffix);
          bsscr_writeMat( C, str, "Writing C matrix in al Solver");
        }
    }

    mg_active=PETSC_TRUE;
    PetscOptionsGetTruth( PETSC_NULL ,"-A11_mg_active", &mg_active, &found );
    bsscrp_self->solver->mg_active=mg_active;

    penaltyNumber = bsscrp_self->solver->penaltyNumber;

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /******  GET K2   ****************************************************************************************/

    if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
        flg=0;
        PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
        if(flg){
            sprintf(str,"%s/K2%s",name, suffix);
            bsscr_writeMat( bsscrp_self->K2, str, "Writing K2 matrix in al Solver");
        }

        K2=bsscrp_self->K2;

        scrSolveTime = MPI_Wtime();
        ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
        scrSolveTime =  MPI_Wtime() - scrSolveTime;

        PetscPrintf( PETSC_COMM_WORLD, "\n\t* K+p*K2 in time: %lf seconds\n\n", scrSolveTime);
        KisJustK=PETSC_FALSE;
        forcecorrection=PETSC_TRUE;
        PetscOptionsGetTruth( PETSC_NULL ,"-force_correction", &forcecorrection, &found );
        if(forcecorrection){
            if(bsscrp_self->f2 && forcecorrection){
                f2=bsscrp_self->f2;
                ierr=VecAXPY(f,penaltyNumber,f2);/*  f <- f +a*f2 */
            }
            else {
                switch (bsscrp_self->k2type) {
                case (K2_GG):
                {
                    VecDuplicate( u, &f3 );
                    MatMult( G, h, f3);   ierr=VecAXPY(f,penaltyNumber,f3);/*  f <- f +a*f2 */
                }
                break;
                case (K2_GMG):
                case (K2_DGMGD):
                {   /* THIS IS THE ONE THAT WE KNOW WORKS PROPERLY */
                    Mat M;
                    Vec Mdiag;
                    VecDuplicate( u, &f3 );
                    M = bsscrp_self->solver->mStiffMat->matrix;
                    MatGetVecs( M, &Mdiag, PETSC_NULL );
                    MatGetDiagonal( M, Mdiag );
                    VecReciprocal(Mdiag);
                    VecPointwiseMult(Mdiag,Mdiag,h);
                    MatMult( G, Mdiag, f3);   ierr=VecAXPY(f,penaltyNumber,f3);/*  f <- f +a*f2 */
                    Stg_VecDestroy(&Mdiag);
                }
                break;
                case (K2_NULL):
                case (K2_SLE):
                {
                    ;
                }
                break;
                }
            }
        }
    }

    /* Create Schur complement matrix */
    MatCreateSchurComplement(K,K,G,D,C, &S);
    MatSchurComplementGetKSP( S, &ksp_inner);
    KSPGetPC( ksp_inner, &pcInner );

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /*********    SET PREFIX FOR INNER/VELOCITY KSP    *************************************************************/

    KSPSetOptionsPrefix( ksp_inner, "A11_" );
    KSPSetFromOptions( ksp_inner );
    Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);

    useNormInfStoppingConditions = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL ,"-A11_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
    if(useNormInfStoppingConditions)
        BSSCR_KSPSetNormInfConvergenceTest( ksp_inner );

    useNormInfMonitor = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
    if(useNormInfMonitor)  KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfMonitor, PETSC_NULL, PETSC_NULL );

    useNormInfMonitor = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_to_norm_2_monitor", &useNormInfMonitor, &found );
    if(useNormInfMonitor)  KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );

    usePreviousGuess = PETSC_FALSE;
//    if(been_here)
//        PetscOptionsGetTruth( PETSC_NULL, "-scr_use_previous_guess", &usePreviousGuess, &found );
    /***************************************************************************************************************/

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /* If multigrid is enabled, set it now. */
    change_A11rhspresolve = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-change_A11rhspresolve", &change_A11rhspresolve, &found );

    if(bsscrp_self->solver->mg_active && !change_A11rhspresolve) { mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx ); }
    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /* create right hand side */

    if(change_A11rhspresolve){
      KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
      Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
      KSPSetOptionsPrefix(ksp_new_inner, "rhsA11_");
      MatSchurComplementSetKSP( S, ksp_new_inner );/* this call destroys the ksp_inner that is already set on S */
      ksp_inner=ksp_new_inner;
      KSPGetPC( ksp_inner, &pcInner );
      KSPSetFromOptions(ksp_inner); /* make sure we are setting up our solver how we want it */
    }

    MatGetVecs( S, PETSC_NULL, &h_hat );
    /* It may be the case that the current velocity solution might not be bad guess for f_tmp? <-- maybe not */
    MatGetVecs( K, PETSC_NULL, &f_tmp );

    /*************************************/
    /*************************************/

//    if (!been_here) {
//        VecDuplicate( u, &uStar );
//        VecSet(uStar,0.0);
//    }
//
//    if(usePreviousGuess) {
//        VecCopy( uStar, f_tmp);
//        KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );
//    }
//    else {
        KSPSetInitialGuessNonzero( ksp_inner, PETSC_FALSE );
//    }

    RHSSetupTime = MPI_Wtime();
    KSPSetUp(ksp_inner);
    RHSSetupTime = MPI_Wtime() - RHSSetupTime;
    bsscrp_self->solver->stats.velocity_presolve_setup_time = RHSSetupTime;

    RHSSolveTime = MPI_Wtime();
    KSPSolve(ksp_inner, f, f_tmp);
    KSPGetConvergedReason( ksp_inner, &reason ); {if (reason < 0) bsscrp_self->solver->fhat_reason=(int)reason; }
    KSPGetIterationNumber( ksp_inner, &bsscrp_self->solver->stats.velocity_presolve_its );
    RHSSolveTime =  MPI_Wtime() - RHSSolveTime;
    bsscrp_self->solver->stats.velocity_presolve_time = RHSSolveTime;

    // VecCopy( f_tmp, uStar);
    // scrSolveTime =  RHSsolveTime;

    MatMult(D, f_tmp, h_hat);
    VecAYPX(h_hat, -1.0, h); /* Computes y = x + alpha y.  h_hat -> h - Gt*K^(-1)*f*/
    Stg_VecDestroy(&f_tmp);

    /*************************************/
    /*************************************/

    if(bsscrp_self->solver->mg_active && change_A11rhspresolve) {
      //Stg_KSPDestroy(&ksp_inner );
      KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
      Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
      KSPSetOptionsPrefix( ksp_new_inner, "A11_" );
      MatSchurComplementSetKSP( S, ksp_new_inner );
      ksp_inner=ksp_new_inner;
      //MatSchurSetKSP( S, ksp_inner );
      KSPGetPC( ksp_inner, &pcInner );
      KSPSetFromOptions( ksp_inner );
      mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx );
    }

    /* create solver for S p = h_hat */
    KSPCreate( PETSC_COMM_WORLD, &ksp_S );
    KSPSetOptionsPrefix( ksp_S, "scr_");

    /* By default use the UW approxS Schur preconditioner -- same as the one used by the Uzawa solver */
    /* Note that if scaling is activated then the approxS matrix has been scaled already */
    /* so no need to rebuild in the case of scaling as we have been doing */
    if(!approxS){
        PetscPrintf( PETSC_COMM_WORLD,  "WARNING approxS is NULL\n");
    }

    Stg_KSPSetOperators( ksp_S, S, S, SAME_NONZERO_PATTERN );
    KSPSetType( ksp_S, "cg" );
    KSPGetPC( ksp_S, &pc_S );
    BSSCR_BSSCR_StokesCreatePCSchur2( K,G,D,C,approxS, pc_S, sym, bsscrp_self );

    flg=0;
    PetscOptionsGetString( PETSC_NULL, "-NN", name, PETSC_MAX_PATH_LEN-1, &flg );
    if(flg){
      Mat Smat, Pmat;
      //MatStructure mstruct;
      Stg_PCGetOperators( pc_S, &Smat, &Pmat, NULL );
      sprintf(str,"%s/Pmat%s",name, suffix);
      bsscr_writeMat( Pmat, str, "Writing Pmat matrix in al Solver");
    }

    uzawastyle=PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-uzawa_style", &uzawastyle, &found );
    if(uzawastyle){
        /* now want to set up the ksp_S->pc to be of type ksp (gmres) by default to match Uzawa */
        KSP pc_ksp;
        KSPGetPC( ksp_S, &pc_S );
        PCSetType(pc_S,PCKSP);
        PCKSPGetKSP( pc_S, &pc_ksp);
        KSPSetType(pc_ksp, "gmres" );
        KSPSetOptionsPrefix( pc_ksp, "scrPCKSP_");
        KSPSetFromOptions( pc_ksp );
    }

    KSPSetFromOptions( ksp_S );
    /* Set specific monitor test */
    KSPGetTolerances( ksp_S, PETSC_NULL, PETSC_NULL, PETSC_NULL, &max_it );

    if(usePreviousGuess) {   /* Note this should actually look at checkpoint information */
        KSPSetInitialGuessNonzero( ksp_S, PETSC_TRUE ); }
    else {
        KSPSetInitialGuessNonzero( ksp_S, PETSC_FALSE ); }

    /***************************************************************************************************************/
    /* Pressure / Velocity Solve */
    /***************************************************************************************************************/

    /***************************************************************************************************************/
    /*******     SET CONVERGENCE TESTS     *************************************************************************/

    useNormInfStoppingConditions = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL ,"-scr_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
    if(useNormInfStoppingConditions)
        BSSCR_KSPSetNormInfConvergenceTest(ksp_S);

    useNormInfMonitor = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
    if(useNormInfMonitor)
        KSPMonitorSet( ksp_S, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /*******   PRESSURE SOLVE   ************************************************************************************/

    //PetscPrintf( PETSC_COMM_WORLD, "\t* KSPSolve( ksp_S, h_hat, p )\n");
    /* if h_hat needs to be fixed up ..take out any nullspace vectors here */
    /* we want to check that there is no "noise" in the null-space in the h vector */
    /* this causes problems when we are trying to solve a Jacobian system when the Residual is almost converged */

    if(bsscrp_self->check_pressureNS){
        bsscrp_self->buildPNS(ksp);/* build and set nullspace vectors on bsscr - which is on ksp (function pointer is set in KSPSetUp_BSSCR) */
    }

    PetscScalar hnorm, gnorm;
    MatNorm(G,NORM_INFINITY,&gnorm);
    VecNorm(h_hat, NORM_2, &hnorm);
    hnorm=hnorm/gnorm;

    if((hnorm < 1e-6) && (hnorm > 1e-20)){
        VecScale(h_hat,1.0/hnorm);
    }
    /* test to see if v or t are in nullspace of G and orthogonalize wrt h_hat if needed */
    KSPRemovePressureNullspace_BSSCR(ksp, h_hat);
    /* set convergence test to use min_it */
    found = PETSC_FALSE;
    min_it = 0;
    PetscOptionsGetInt( PETSC_NULL,"-scr_ksp_set_min_it_converge", &min_it, &found);
    if(found && min_it > 0){
        BSSCR_KSPSetConvergenceMinIts(ksp_S, min_it, bsscrp_self);
    }

    /** Pressure Setup **/
    scrSetupTime = MPI_Wtime();
    KSPSetUp(ksp_S);
    scrSetupTime = MPI_Wtime() - scrSetupTime;
    bsscrp_self->solver->stats.velocity_pressuresolve_setup_time = scrSetupTime;

    /** Pressure Solve **/
    if(get_flops) PetscGetFlops(&flopsA);
    scrSolveTime = MPI_Wtime();
    KSPSolve( ksp_S, h_hat, p );
    scrSolveTime =  MPI_Wtime() - scrSolveTime;

    KSPGetConvergedReason( ksp_S, &reason ); {if (reason < 0) bsscrp_self->solver->outer_reason=(int)reason; }
    /*************************************/
    /*************************************/
#if ( (PETSC_VERSION_MAJOR >= 3) && (PETSC_VERSION_MINOR >= 6 ) && (PETSC_VERSION_SUBMINOR >= 1 ))
    KSPGetTotalIterations(ksp_inner, &bsscrp_self->solver->stats.velocity_pressuresolve_its);
    if(!change_A11rhspresolve){
      bsscrp_self->solver->stats.velocity_pressuresolve_its -= bsscrp_self->solver->stats.velocity_presolve_its;
    }
#else
    bsscrp_self->solver->stats.velocity_pressuresolve_its=-1;
#endif

    if(get_flops) {
      PetscGetFlops(&flopsB);
      bsscrp_self->solver->stats.pressure_flops=(double)(flopsB-flopsA); }
    //PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve( ksp_S, h_hat, p )  Solve Finished in time: %lf seconds\n\n", scrSolveTime);
    bsscrp_self->solver->stats.pressure_time=scrSolveTime;

    /***************************************/
    if((hnorm < 1e-6) && (hnorm > 1e-20)){
        VecScale(h_hat,hnorm);
        VecScale(p,hnorm);
    }
    KSPRemovePressureNullspace_BSSCR(ksp, p);

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /* restore K and f for the Velocity back solve */

    found = PETSC_FALSE;
    restorek = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-restore_K", &restorek, &found);
    //PetscOptionsGetString( PETSC_NULL, "-restore_K", name, PETSC_MAX_PATH_LEN-1, &flg );
    if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
        if(restorek){
            penaltyNumber = -penaltyNumber;
            if(f2) {
                ierr=VecAXPY(f,penaltyNumber,f2);
            }/*  f <- f +a*f2 */
            if(f3) {
                ierr=VecAXPY(f,penaltyNumber,f3);
            }/*  f <- f +a*f3 */
            ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);
            CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
            Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);
            KisJustK=PETSC_TRUE;
        }
    }
    if(f3){ Stg_VecDestroy(&f3 ); } /* always destroy this local vector if was created */

    /* obtain solution for u */

    VecDuplicate( u, &t );
    MatMult( G, p, t);
    VecAYPX( t, -1.0, f ); /*** t <- -t + f   = f - G*p  ***/

    MatSchurComplementGetKSP( S, &ksp_inner );
    a11SingleSolveTime = MPI_Wtime();           /* ----------------------------------  Final V Solve */
    if(usePreviousGuess)
        KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /*******   VELOCITY SOLVE   ************************************************************************************/

    /** Easier to just create a new KSP here if we want to do backsolve diffferently. (getting petsc errors now when switching from fgmres) */

    change_backsolve=PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-change_backsolve", &change_backsolve, &found );
    if(change_backsolve){
      //Stg_KSPDestroy(&ksp_inner );
      KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
      Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
      KSPSetOptionsPrefix(ksp_new_inner, "backsolveA11_");
      KSPSetFromOptions(ksp_new_inner); /* make sure we are setting up our solver how we want it */
      MatSchurComplementSetKSP( S, ksp_new_inner );/* need to give the Schur it's inner ksp back for when we destroy it at end */
      ksp_inner=ksp_new_inner;
    }

    if(get_flops) PetscGetFlops(&flopsA);
    /*************************************/
    /*************************************/

    backsolveSetupTime = MPI_Wtime();
    KSPSetUp(ksp_inner);
    backsolveSetupTime = MPI_Wtime() - backsolveSetupTime;
    bsscrp_self->solver->stats.velocity_backsolve_setup_time = backsolveSetupTime;

    KSPSolve(ksp_inner, t, u);
    KSPGetConvergedReason(ksp_inner, &reason ); {if (reason < 0) bsscrp_self->solver->backsolve_reason=(int)reason; }
    /*************************************/
    /*************************************/
    KSPGetIterationNumber( ksp_inner, &bsscrp_self->solver->stats.velocity_backsolve_its );
    if(get_flops) {
      PetscGetFlops(&flopsB);
      bsscrp_self->solver->stats.velocity_backsolve_flops=(double)(flopsB-flopsA); }
    a11SingleSolveTime = MPI_Wtime() - a11SingleSolveTime;              /* ------------------ Final V Solve */

    bsscrp_self->solver->stats.velocity_presolve_time=RHSSolveTime;
    bsscrp_self->solver->stats.velocity_backsolve_time=a11SingleSolveTime;
    bsscrp_self->solver->stats.velocity_total_its = bsscrp_self->solver->stats.velocity_backsolve_its +
                                                    bsscrp_self->solver->stats.velocity_presolve_its +
                                                    bsscrp_self->solver->stats.velocity_pressuresolve_its;

    flg=0;

    PetscOptionsGetString( PETSC_NULL, "-solutiondumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
    if(flg){
        sprintf(str,"%s/p%s",name,suffix);
        bsscr_writeVec( p, str, "Writing p vector in al Solver");
        sprintf(str,"%s/u%s",name, suffix);
        bsscr_writeVec( u, str, "Writing u vector in al Solver");
        sprintf(str,"%s/h_hat%s",name,suffix);
        bsscr_writeVec( h_hat, str, "Writing h_hat vector in al Solver");

    }

    found = PETSC_FALSE;
    restorek = PETSC_FALSE;
    PetscOptionsGetTruth( PETSC_NULL, "-restore_K_after_solve", &restorek, &found);
    if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
        if(restorek){
            penaltyNumber = -penaltyNumber;
            ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
            KisJustK=PETSC_TRUE;
        }
    }
    /***************************************************************************************************************/
    /***************************************************************************************************************/
    /******     SOLUTION SUMMARY      ******************************************************************************/
    bsscr_summary(bsscrp_self,ksp_S,ksp_inner,K,K2,D,G,C,u,p,f,h,t,penaltyNumber,KisJustK,
                  mgSetupTime, RHSSolveTime, scrSolveTime, a11SingleSolveTime);

    /***************************************************************************************************************/
    /***************************************************************************************************************/
    Stg_VecDestroy(&t );
    Stg_KSPDestroy(&ksp_S );
    Stg_VecDestroy(&h_hat );
    Stg_MatDestroy(&S );//This will destroy ksp_inner: also.. pcInner == pc_MG and is destroyed when ksp_inner is
    been_here = 1;
    PetscFunctionReturn(0);
}


/*
# Program to run a Monte Carlo radiation transfer through the 2D
# simulations of GRB jets.
#
# Python code written by D. Lazzati at Oregonstate, C code written by Tyler Parsotan @ Oregon State
# ver 0.1 July 8, 2015
# ver 1.1 July 20, 2015: added record of number of scatterings, included
# 	all terms in weight. Should now give correct light curves.
# ver 1.2 July 21, 2015: added parameter file to keep track of input
# 	params of each simulation

# ver 2.0 July 22, 2015: corrected the problem that arises when there is
# 	no scattering in the time span of one frame. Fixed output arrays dimension.

# ver 2.1 July 25, 2015: fixed bug that did not make the number of
# 	scattering grow with the number of photons.

# ver 3.0 July 28, 2015: using scipy nearest neighbor interpolation to
# 	speed things up. Gained about factor 2

# ver 3.1 July 29, 2015: added radial spread of photon injection points
# ver 3.2 July 31, 2015: added Gamma to the weight of photons!!!

# ver 4.0 Aug 5, 2015: try to speed up by inverting cycle
# ver 4.1 Aug 8, 2015: add spherical test as an option
# ver 4.2 Aug 9, 2015: saving files appending rather than re-writing
# ver 4.3 Aug 11, 2015: corrected error in the calculation of the local temperature
# ver 4.4 Aug 13, 2015: added cylindrical test
# ver 4.5 Aug 18, 2015: fixd various problems pointed by the cylindrical test
# ver 4.6 Aug 21, 2015: corrected mean free path for large radii

# ver 5.0 Aug 25, 2015: corrected problem with high-T electrons and excess scatterings
# ver 5.1 Aug 25, 2015: cleaned-up coding
# ver 5.2 Sept 3, 2015: fixed problem with number of scatterings for multiple injections
 *
 * ver 6.0 Dec 28, 2016: rewrote the code in C, added checkpoint file so if the code is interrupted all the progress wont be lost, made the code only need to be compiled once for a given MC_XXX directory path
                                            so you just need to supply the sub directory of MC_XXX as a command line argument
*/

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <dirent.h>
#include <math.h>
#include <gsl/gsl_rng.h>
#include "mclib.h"

#define THISRUN "Spherical"
#define FILEPATH "/Volumes/parsotat/Documents/16TI/"
#define FILEROOT "rhd_jet_big_13_hdf5_plt_cnt_"
#define MC_PATH "CMC_16TI_SPHERICAL/"
#define MCPAR "mc.par"

int main(int argc, char **argv)
{
    //compile each time a macro is changed, have to supply the subfolder within the MC_PATH directory as a command line argument to the C program eg. MCRAT 1/

	// Define variables
	char flash_prefix[200]="";
	char flash_file[200]="";
	char mc_dir[200]="" ;
	char mc_file[200]="" ;
    char mc_filename[200]="";
    char mc_operation[200]="";
    char this_run[200]=THISRUN;
    char *cyl="Cylindrical";
    char *sph="Spherical";
    int file_count = 0;
    DIR * dirp;
    struct dirent * entry;
	char spect;//type of spectrum
    char restrt;//restart or not
	double fps, theta_jmin, theta_jmax ;//frames per second of sim, min opening angle of jet, max opening angle of jet in radians
	double inj_radius, ph_weight ;//radius at chich photons are injected into sim
	int frm0,last_frm, frm2, photon_num ;//frame starting from, last frame of sim, frame of last injection, number of photons (not necessary any more)
	double *xPtr=NULL,  *yPtr=NULL,  *rPtr=NULL,  *thetaPtr=NULL,  *velxPtr=NULL,  *velyPtr=NULL,  *densPtr=NULL,  *presPtr=NULL,  *gammaPtr=NULL,  *dens_labPtr=NULL;
    double *szxPtr=NULL,*szyPtr=NULL, *tempPtr=NULL; //pointers to hold data from FLASH files
    int num_ph=0, array_num=0, ph_scatt_index=0, max_scatt=0, min_scatt=0,i=0; //number of photons produced in injection algorithm, number of array elleemnts from reading FLASH file, index of photon whch does scattering, generic counter
    double dt_max=0, thescatt=0, accum_time=0;
    double  gamma_infinity=0;
    double time_now=0, time_step=0, avg_scatt=0;
    double ph_dens_labPtr=0, ph_vxPtr=0, ph_vyPtr=0, ph_tempPtr=0;// *ph_cosanglePtr=NULL ;
    int frame=0, scatt_frame=0, frame_scatt_cnt=0, scatt_framestart=0, framestart=0;
    struct photon *phPtr=NULL; //pointer to array of photons

    long seed;
    const gsl_rng_type * T;
    gsl_rng * rand;

    gsl_rng_env_setup();
    T = gsl_rng_ranlxs0;
    rand = gsl_rng_alloc (T);

    //make strings of proper directories etc.
	snprintf(flash_prefix,sizeof(flash_prefix),"%s%s",FILEPATH,FILEROOT );
	snprintf(mc_dir,sizeof(flash_prefix),"%s%s%s",FILEPATH,MC_PATH, argv[1]);
	snprintf(mc_file,sizeof(flash_prefix),"%s%s",mc_dir,MCPAR);

    printf(">> mc.py: I am working on path: %s \n",mc_dir);

    printf(">> mc.py:  Reading mc.par\n");

    readMcPar(mc_file, &fps, &theta_jmin, &theta_jmax,&inj_radius,&frm0,&last_frm,&frm2,&photon_num, &ph_weight, &spect, &restrt);

    if (restrt=='c')
    {
        printf(">> mc.py:  Reading checkpoint\n");
        readCheckpoint(mc_dir, &phPtr, &framestart, &scatt_framestart, &num_ph, &restrt, &time_now);
        for (i=0;i<num_ph;i++)
        {
            printf("%e,%e,%e, %e,%e,%e, %e, %e\n",(phPtr+i)->p0, (phPtr+i)->p1, (phPtr+i)->p2, (phPtr+i)->p3, (phPtr+i)->r0, (phPtr+i)->r1, (phPtr+i)->r2, (phPtr+i)->num_scatt );
        }

        if (restrt=='c')
        {
            printf("Starting from photons injected at frame: %d out of %d\n", framestart, frm2);
            printf("Continuing scattering %d photons from frame: %d\n", num_ph, scatt_framestart);
            printf("The time now is: %e\n", time_now);
        }
        else
        {
            printf("Continuing simulation by injecting photons at frame: %d out of %d\n", framestart, frm2); //starting with new photon injection is same as restarting sim
        }
    }
    else
    {
        //remove everything from MC directory to ensure no corruption of data if theres other files there besides the mc.par file
        //for a checkpoint implementation, need to find the latest checkpoint file and read it and not delete the files
        printf(">> mc.py:  Cleaning directory\n");
        dirp = opendir(mc_dir);
        while ((entry = readdir(dirp)) != NULL)
        {
            if (entry->d_type == DT_REG) { /* If the entry is a regular file */
                 file_count++; //count how many files are in dorectory
            }
        }
        //printf("File count %d\n", file_count);

        if (file_count>2)
        {
            for (i=0;i<=last_frm;i++)
            {
                snprintf(mc_filename,sizeof(flash_prefix),"%s%s%d%s", mc_dir,"mcdata_",i,"_P0.dat");
                if( access( mc_filename, F_OK ) != -1 )
                {
                    snprintf(mc_operation,sizeof(flash_prefix),"%s%s%s%d%s","exec rm ", mc_dir,"mcdata_",i,"_*.dat"); //prepares string to remove *.dat in mc_dir
                    //printf("%s",mc_operation);
                    system(mc_operation);
                }
            }
        }
        framestart=frm0; //if restarting then start from parameters given in mc.par file
        scatt_framestart=framestart;
    }

    dt_max=1.0/fps;

    //loop over frames
    //for a checkpoint implementation, start from the last saved "frame" value and go to the saved "frm2" value
    for (frame=framestart;frame<=frm2;frame++)
    {
         if (restrt=='r')
         {
            time_now=frame/fps; //for a checkpoint implmentation, load the saved "time_now" value when reading the ckeckpoint file otherwise calculate it normally
         }

        printf(">> mc.py: Working on Frame %d\n", frame);

        if (restrt=='r')
        {
            //put proper number at the end of the flash file
            if (frame<10)
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%.3d%d",flash_prefix,000,frame);
            }
            else if (frame<100)
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%.2d%d",flash_prefix,00,frame);
            }
            else if (frame<1000)
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%d%d",flash_prefix,0,frame);
            }
            else
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%d",flash_prefix,frame);
            }

            printf(">> mc.py: Opening FLASH file %s\n",flash_file);
            //read in FLASH file
            //for a checkpoint implmentation, dont need to read the file yet
            readAndDecimate(flash_file, inj_radius, &xPtr,  &yPtr,  &szxPtr, &szyPtr, &rPtr,\
                &thetaPtr, &velxPtr,  &velyPtr,  &densPtr,  &presPtr,  &gammaPtr,  &dens_labPtr, &tempPtr, &array_num);

            //check for run type
            if(strcmp(cyl, this_run)==0)
            {
                //printf("In cylindrical prep\n");
                cylindricalPrep(gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num);
            }
            else if (strcmp(sph, this_run)==0)
            {
                sphericalPrep(rPtr, xPtr, yPtr,gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num );
            }

            //determine where to place photons and how many should go in a given place
            //for a checkpoint implmentation, dont need to inject photons, need to load photons' last saved data
            printf(">> mc.py: Injecting photons\n");
            photonInjection(&phPtr, &num_ph, inj_radius, ph_weight, spect, array_num, fps, theta_jmin, theta_jmax, xPtr, yPtr, szxPtr, szyPtr,rPtr,thetaPtr, tempPtr, velxPtr, velyPtr,rand );
            printf("%d\n",num_ph); //num_ph is one more photon than i actually have
            /*
            for (i=0;i<num_ph;i++)
                printf("%e,%e,%e \n",(phPtr+i)->r0, (phPtr+i)->r1, (phPtr+i)->r2 );
            */
        }

        //scatter photons all the way thoughout the jet
        //for a checkpoint implmentation, start from the last saved "scatt_frame" value eh start_frame=frame or start_frame=cont_frame
        if (restrt=='r')
        {
            scatt_framestart=frame; //have to make sure that once the inner loop is done and the outer loop is incrememnted by one the inner loop starts at that new value and not the one read by readCheckpoint()
        }

        for (scatt_frame=scatt_framestart;scatt_frame<=last_frm;scatt_frame++)
        {
            printf(">>\n");
            printf(">> mc.py: Working on photons injected at frame: %d out of %d\n", frame, frm2);
            printf(">> mc.py: %s: Working on frame %d\n", THISRUN, scatt_frame);
            printf(">> mc.py: Opening file...\n");

            //put proper number at the end of the flash file
            if (scatt_frame<10)
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%.3d%d",flash_prefix,000,scatt_frame);
            }
            else if (scatt_frame<100)
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%.2d%d",flash_prefix,00,scatt_frame);
            }
            else if (scatt_frame<1000)
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%d%d",flash_prefix,0,scatt_frame);
            }
            else
            {
                snprintf(flash_file,sizeof(flash_prefix), "%s%d",flash_prefix,scatt_frame);
            }

            readAndDecimate(flash_file, inj_radius, &xPtr,  &yPtr,  &szxPtr, &szyPtr, &rPtr,\
                &thetaPtr, &velxPtr,  &velyPtr,  &densPtr,  &presPtr,  &gammaPtr,  &dens_labPtr, &tempPtr, &array_num);
                //printf("The result of read and decimate are arrays with %d elements\n", array_num);

            //check for run type
            if(strcmp(cyl, this_run)==0)
            {
                printf("In cylindrical prep\n");
                cylindricalPrep(gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num);
            }
            else if (strcmp(sph, this_run)==0)
            {
                sphericalPrep(rPtr, xPtr, yPtr,gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num );
            }

            printf(">> mc.py: propagating and scattering %d photons\n", num_ph);

            frame_scatt_cnt=0;
            while (time_now<((scatt_frame+1)/fps))
            {
                //if simulation time is less than the simulation time of the next frame, keep scattering in this frame
                //go through each photon and find blocks closest to each photon and properties of those blocks to calulate mean free path
                //and choose the photon with the smallest mfp and calculate the timestep

                ph_scatt_index=findNearestPropertiesAndMinMFP(phPtr, num_ph, array_num, &time_step, xPtr,  yPtr, velxPtr,  velyPtr,  dens_labPtr, tempPtr,\
                    &ph_dens_labPtr, &ph_vxPtr, &ph_vyPtr, &ph_tempPtr, rand);

                //printf("In main: %e, %d, %e, %e\n", *(ph_num_scatt+ph_scatt_index), ph_scatt_index, time_step, time_now);
                //printf("In main: %e, %d, %e, %e\n",((phPtr+ph_scatt_index)->num_scatt), ph_scatt_index, time_step, time_now);

                 if (time_step<dt_max)
                {

                    //update number of scatterings and time
                    //(*(ph_num_scatt+ph_scatt_index))+=1;
                    ((phPtr+ph_scatt_index)->num_scatt)+=1;
                    frame_scatt_cnt+=1;
                    time_now+=time_step;

                    updatePhotonPosition(phPtr, num_ph, time_step);

                    //scatter the photon
                    //printf("Passed Parameters: %e, %e, %e\n", (ph_vxPtr), (ph_vyPtr), (ph_tempPtr));

                    photonScatter( (phPtr+ph_scatt_index), (ph_vxPtr), (ph_vyPtr), (ph_tempPtr), rand );

                    if (frame_scatt_cnt%1000 == 0)
                    {
                        printf("Scattering Number: %d\n", frame_scatt_cnt);
                        printf("The local temp is: %e, Local energy: %e\n", (ph_tempPtr),  (ph_tempPtr)*(1.380658e-16/1.6e-9));
                        printf("Average photon energy is: %e keV\n", averagePhotonEnergy(phPtr, num_ph)/1.6e-9); //write function to average over the photons p0*c and then do (/1.6e-9)
                    }

                }
                else
                {
                    time_now+=dt_max;

                    //for each photon update its position based on its momentum

                    updatePhotonPosition(phPtr, num_ph, dt_max);
                }

                //printf("In main 2: %e, %d, %e, %e\n", ((phPtr+ph_scatt_index)->num_scatt), ph_scatt_index, time_step, time_now);

            }

        //get scattering statistics
        phScattStats(phPtr, num_ph, &max_scatt, &min_scatt, &avg_scatt);

        printf("The number of scatterings in this frame is: %d\n", frame_scatt_cnt);
        printf("The last time step was: %lf.\nThe time now is: %lf\n", time_step,time_now);
        printf("The maximum number of scatterings for a photon is: %d\nThe minimum number of scattering for a photon is: %d\n", max_scatt, min_scatt);
        printf("The average number of scatterings thus far is: %lf\n", avg_scatt);

        printPhotons(phPtr, num_ph,  scatt_frame , mc_dir);

        //for a checkpoint implmentation,save the checkpoint file here after every 5 frames or something
        //save the photons data, the scattering number data, the scatt_frame value, and the frame value
        //WHAT IF THE PROGRAM STOPS AFTER THE LAST SCATT_FRAME, DURING THE FIRST SCATT_FRAME OF NEW FRAME VARIABLE - save restrt variable as 'r'
        printf(">> mc.py: Making checkpoint file\n");
        saveCheckpoint(mc_dir, frame, scatt_frame, num_ph, time_now, phPtr, last_frm);

        free(xPtr);free(yPtr);free(szxPtr);free(szyPtr);free(rPtr);free(thetaPtr);free(velxPtr);free(velyPtr);free(densPtr);free(presPtr);
        free(gammaPtr);free(dens_labPtr);free(tempPtr);
        xPtr=NULL; yPtr=NULL;  rPtr=NULL;thetaPtr=NULL;velxPtr=NULL;velyPtr=NULL;densPtr=NULL;presPtr=NULL;gammaPtr=NULL;dens_labPtr=NULL;
        szxPtr=NULL; szyPtr=NULL; tempPtr=NULL;
        }
        restrt='r';//set this to make sure that the next iteration of propogating photons doesnt use the values from the last reading of the checkpoint file
        free(phPtr);
        phPtr=NULL;

    }

    gsl_rng_free (rand);
	return 0;

}


/**
 */
#include <gsl/gsl_vector.h>
#include <gsl/gsl_matrix.h>
#include "aXe_grism.h"
#include <math.h>
#include <string.h>
#include <ctype.h>
#include "drz2pet_utils.h"

/**
 * Function: make_spc_drztable
 * The function generates and fills an ap_pixel vector for one
 * drizzled 2D grism image.
 *
 * Parameters:
 * @param  obs     - pointer to observation structure with image values
 * @param  obs     - pointer to observation structure with image values
 * @param  ob      - pointer to the description of the current beam
 * @param  lambda0 - the starting wavelength in this drizzled image
 * @param  dlambda - the dispersion in this drizzled image
 *
 * Returns:
 * @return table   - pointer to an ap_pixel vector
 */
ap_pixel *
make_spc_drztable(observation * const obs, observation * const wobs,
                  const object *ob, const double lambda0,
                  const double dlambda){

  ap_pixel *table, *cur_ap;

  int npix1, npix2;
  int nsub1=1, nsub2=1;
  int i, j;
  int beamInt = 0;

  npix1 = obs->grism->size1;
  npix2 = obs->grism->size2;

  if (!(table = malloc ((npix1 * npix2 + 1) * sizeof (ap_pixel) * nsub1 * nsub2)))
    return NULL;

  cur_ap = table;

  for (i = 0; i < npix1; i++){
    for (j = 0; j < npix2; j++){

      //* check whether the pixel is within the extraction box
      if (fabs((double)j - ob->beams[beamInt].refpoint.y) > ob->beams[beamInt].width + .5)
        continue;

      //* fill the ap_pixel
      cur_ap->p_x  = i;
      cur_ap->p_y  = j;

      cur_ap->x       = (double)i - ob->beams[beamInt].refpoint.x;
      cur_ap->y       = (double)j - ob->beams[beamInt].refpoint.y;
      cur_ap->dist    = (double)j - ob->beams[beamInt].refpoint.y;
      cur_ap->xs      = (double)i - ob->beams[beamInt].refpoint.x;
      cur_ap->ys      = 0.0;
      cur_ap->dxs     = 0.0;  // be reminded: dxs is the LOCAL TRACEANGLE, not the pixels size..
      cur_ap->xi      = (double)i - ob->beams[beamInt].refpoint.x;
      cur_ap->lambda  = lambda0 + dlambda * ((double)i - ob->beams[beamInt].refpoint.x);
      cur_ap->dlambda = dlambda;

      // transfer the cps values and the weight values
      cur_ap->count  = gsl_matrix_get(obs->grism, i, j);
      cur_ap->weight = gsl_matrix_get(wobs->grism, i, j);

      cur_ap->error  = gsl_matrix_get(obs->pixerrs, i, j);
      cur_ap->contam = gsl_matrix_get(obs->dq, i, j);

      // dq is always set to zero, bad (=unexposed) pixels
      // have exptime=weight=0.0
      cur_ap->dq     = 0;

      // go to the next
      // PET pixel
      cur_ap++;
    }
  }

  cur_ap->p_x = -1;
  cur_ap->p_y = -1;
  cur_ap->count = -1;

  return table;
}

/**
 * Function: get_ID_num
 * The function generates the extension name used in the PET
 * for a 2D drizzled grism image
 *
 * Parameters:
 * @param  grism_image - the name of the 2D drizzled grism image
 * @param  extname     - The ID number of the image
 *
 * Returns:
 * @return WorkPtr     - Pointer to the ID-string
 */
char
*get_ID_num (char *grism_image, char *extname){

  char *WorkPtr;
  char *WorkPtr2;
  char **t_err=NULL;
  char tmp[MAXCHAR];
  char ID[4];

  int len;
  int i=0;

  strcpy(ID,"_ID");
  strcpy(tmp, grism_image);

  t_err = (char **) malloc(sizeof(char *)*1);

  WorkPtr = strstr(tmp, ID)+3;
  WorkPtr2 = WorkPtr + strlen (WorkPtr);
  while (isspace ((int) *WorkPtr2))
    *WorkPtr2-- = '\0';
  WorkPtr2 = WorkPtr2 - 1;
  len = strlen(WorkPtr);
  while  (i++ < len){
    strtol(WorkPtr,t_err,10);
    if (strlen(t_err[0])==0)
      break;
    *WorkPtr2-- = '\0';
  }
  strcpy(extname,WorkPtr);
  free(t_err);
  return WorkPtr;
}

/**
 * Function: normalize_weight
 * The function computes the weights used in the extraction
 * from the exposure times given for each pixel in a column.
 *
 * Parameters:
 * @param  wobs - pointer to the structure which carries the pixel data
 * @param  ob   - pointer to the beam description
 */
void
normalize_weight(observation *wobs, object *ob,
                 const int opt_extr)
{
  gsl_matrix *weight;
  //int i, j;
 // // check for optimal extraction
  if (opt_extr)
    {
      // let the optimal weights compute
      fprintf(stdout, "aXe_DRZ2PET: Use otpimal weighting.\n");
      weight = comp_opt_weight(wobs->pixerrs, wobs->dq, ob);
    }
  else
    {
      // let the exposure time weights compute
      fprintf(stdout, "aXe_DRZ2PET: Use normal, flat  weighting.\n");
      weight = comp_equ_weight(wobs->grism, ob);

      // code lines for exposure time weighting
      //fprintf(stdout, "use exposure time  weighting\n");
      //weight = comp_exp_weight(wobs->grism, ob);
    }

  // replace the weight extension with the new weights
  gsl_matrix_free(wobs->grism);
  wobs->grism = weight;
}

/*
 * Function: comp_equ_weight
 * The function creates and fills a weight map with flat entries.
 * Pixels with non-zero exposure time which which contribute to the
 * extracted spectrum all receive the weight 1.0, all other
 * get the weight 0.0.
 *
 * Parameters:
 * @param exp_map - the matrix with the exposure times
 * @param ob      - the object structure
 *
 * Returns:
 * @return weight - the matrix with the exposure time weights
 */
gsl_matrix *
comp_equ_weight(gsl_matrix *exp_map, const object *ob)
{
  gsl_matrix *weight;
  int i, j;
  int beamInt = 0;

  // allocate the matrix and set it to the default
  weight = gsl_matrix_alloc(exp_map->size1, exp_map->size2);
  gsl_matrix_set_all(weight, 0.0);

   // go over all columns
  for (i=0; i < (int)exp_map->size1; i++)
    // go over all rows
    for (j=0; j < (int)exp_map->size2; j++)
      // check whether the pixel is inside the extraction
      // area and has a positive exposure time
      if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5 &&
          gsl_matrix_get(exp_map, i, j) > 0.0 )
        // set the weight to one
        gsl_matrix_set(weight, i,j,1.0);

  // return the resulting weight map
  return weight;
}

/*
 * Function: comp_exp_weight
 * The function uses the exposure time map and computes
 * the weights which are proportional to the relative
 * exposure time of the pixel. Pixels outside of the extraction
 * area are set to 1000.0 Inside the default is 1.0 in unexposed
 * columns and 0.0 within exposed columns.
 *
 * Parameters:
 * @param exp_map - the matrix with the exposure times
 * @param ob      - the object structure
 *
 * Returns:
 * @return weight - the matrix with the exposure time weights
 */
gsl_matrix *
comp_exp_weight(gsl_matrix *exp_map, const object *ob)
{
  gsl_matrix *weight;
  int i, j;
  double sum, contr, norm, allweight;
  int beamInt = 0;

  // allocate the matrix and set it to the default
  weight = gsl_matrix_alloc(exp_map->size1, exp_map->size2);
  gsl_matrix_set_all(weight, 1000.0);

    //* go over all columns
  for (i=0; i < (int)exp_map->size1; i++)
    {
      sum = 0.0;
      contr = 0.0;
      allweight = 0.0;

      // determine for each column the total exposure time
      // and the number of pixels with non-zero exposure time
      for (j=0; j < (int)exp_map->size2; j++)
        {
          // check whether the pixel is inside the extraction
          // area and has a positive weight
          if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5 &&
              gsl_matrix_get(exp_map, i, j) > 0.0 )
            {
              // sum up the weigth and the number of pixels
              sum = sum + gsl_matrix_get(exp_map, i, j);
              contr = contr + 1.0;
            }
        }

      // check whether the row has exposure time at all
      if (sum > 0.0)
        {
          // if the row is exposed:
          // determine the mean exposure time
          norm = sum / contr;

          // go over each pixel
          for (j=0; j < (int)exp_map->size2; j++)
            {
              // check whether the pixel is inside the extraction are
              if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5)
                {
                  // compute and store the relative exposure time (==weight)
                  // for positive exposure time
                  if (gsl_matrix_get(exp_map, i, j) > 0.0)
                    {
                      gsl_matrix_set(weight, i,j,gsl_matrix_get(exp_map, i, j)/norm);
                      allweight = allweight + gsl_matrix_get(weight, i, j);
                    }
                  // make a default to avoid values <0.0
                  else
                    {
                      gsl_matrix_set(weight, i,j,0.0);
                    }
                }
            }
        }
      // treat unexposed columns differently:
      else
        {

          // go over each row
          for (j=0; j < (int)exp_map->size2; j++)
            {
              // check whether the pixel is inside the extraction are
              if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5)
                {
                  // set inside pixels to the defaulkt 1.0
                  gsl_matrix_set(weight, i,j,1.0);
                  allweight = allweight + gsl_matrix_get(weight, i, j);
                }
            }
        }
    }

  // return the resulting weight map
  return weight;
}

/*
 * Function: comp_opt_weight
 * The function uses the model map and and the variance
 * map and computes optimale weigths according to the
 * Hoorne method.
 * Pixels outside of the extraction
 * area are set to 1000.0
 *
 * Parameters:
 * @param mod_map - the matrix with the model values
 * @param var_map - the matrix with the variance values
 * @param ob      - the object structure
 *
 * Returns:
 * @return weight - the matrix with the exposure time weights
 */
gsl_matrix *
comp_opt_weight(gsl_matrix *mod_map,
                gsl_matrix *var_map, const object *ob)
{
  gsl_matrix *weight;
  int i, j;
  double mod_sum, weight_sum, contr, norm, allweight;
  int beamInt = 0;
  double mod_val;
  //double var_val;

  // allocate the weight matrix and set the default
  weight = gsl_matrix_alloc(mod_map->size1, mod_map->size2);
  gsl_matrix_set_all(weight, 0.0);

  //* go over all columns
  for (i=0; i < (int)mod_map->size1; i++)
    {
      mod_sum = 0.0;
      contr = 0.0;
      allweight = 0.0;
      weight_sum=0.0;

      // determine for each column the total model counts
      // and the number of pixels with non-zero model_counts
      for (j=0; j < (int)mod_map->size2; j++)
        {
          if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5 &&
              gsl_matrix_get(mod_map, i, j) > 0.0 )
            {
              mod_sum = mod_sum + gsl_matrix_get(mod_map, i, j);
              contr = contr + 1.0;
            }
        }

      // check whether the column has model values.
      // normalize the model values and compute
      // optimal weights if yes.
      if (mod_sum > 0.0)
        {
          //* determine the mean model value
          //      norm = mod_sum / contr;
          norm = mod_sum;

          // go over each row
          for (j=0; j < (int)mod_map->size2; j++)
            {
              // check whether the pixel is inside the extraction area
              if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5 &&
                  gsl_matrix_get(mod_map, i, j) > 0.0 )
                {
                  // normalize the model counts
                  mod_val = gsl_matrix_get(mod_map, i, j)/norm;

                  // store the normalized model counts
                  gsl_matrix_set(mod_map, i, j, mod_val);

                  // add up the normalization value for the optimal weights
                  weight_sum = weight_sum + mod_val*mod_val/gsl_matrix_get(var_map, i, j);
                }
            }

          // finally compute and write the weights:
          // go over each pixel
          for (j=0; j < (int)mod_map->size2; j++)
            {
              // check whether the pixel is inside the extraction area
              if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5)
                {
                  if (gsl_matrix_get(mod_map, i, j) > 0.0 )
                    {
                      // compute and set the individual pixel weight
                      gsl_matrix_set(weight, i, j, gsl_matrix_get(mod_map, i, j)/gsl_matrix_get(var_map, i, j)/weight_sum);
                      allweight = allweight + gsl_matrix_get(weight, i, j);
                    }
                  else
                    {
                      // set the default value if no model value is zero
                      gsl_matrix_set(weight, i, j,0.0);
                    }
                }
            }
        }
      // if the column does not have model values at all:
      else
        {
          // go over each row
          for (j=0; j < (int)mod_map->size2; j++)
            {
              // check whether the pixel is within the extraction area
              if (fabs(ob->beams[beamInt].refpoint.y-(double)j) <= ob->beams[beamInt].width+0.5)
                {
                  // set the inside default value
                  gsl_matrix_set(weight, i,j,1.0);
                  allweight = allweight + gsl_matrix_get(mod_map, i, j);
                }
            }
        }
    }

  // return the weight matrix
  return weight;
}


#ifndef GrowthFunctionBase_H
#define GrowthFunctionBase_H

#include <stdio.h>
#include <iostream>
#include <gsl/gsl_linalg.h>
#include <gsl/gsl_blas.h>
#include <array>
#include <vector>

class GrowthFunctionBase{
private:

public:
	GrowthFunctionBase(int id, int type, float initTime, float endTime, bool applyToColumnarLayer, bool applyToPeripodialMembrane, bool applyToBasalECM, bool applyToLateralECM){
		/**
		 *  integer id will set GrowthFunctionBase#Id. \n
		 *  integer type will set GrowthFunctionBase#Type. \n
		 *  floats initTime and endTime will set GrowthFunctionBase#initTime and GrowthFunctionBase#endTime respectively. \n
		 *	booleans applyToColumnarLayer and applyToPeripodialMembrane will set GrowthFunctionBase#applyToColumnarLayer and GrowthFunctionBase#applyToPeripodialMembrane, respectively. \n
		 *   */
		this->Id = id;
		this->Type = type;
		this->initTime = initTime;
		this->endTime = endTime;
		this->applyToColumnarLayer = applyToColumnarLayer;
		this->applyToPeripodialMembrane = applyToPeripodialMembrane;
		this->applyToBasalECM = applyToBasalECM;
		this->applyToLateralECM = applyToLateralECM;
		zMin = 0;
		zMax = 1.0;
	} ///< The constructor of GrowthFunctionBase. Different growth functions will be derived from this class
    ~GrowthFunctionBase(){}

	int Type; 						///< The type of the growth function, 1: uniform growth, 2: Ring shaped growth, 3: Grid based growth, where growth rates read from a separate input file
	int Id;							///< The unique identification number of the growth function
	float initTime;					///< The initiation time of the growth, in seconds
	float endTime;					///< The end time of the growth, in seconds.
	bool applyToColumnarLayer;		///< Boolean stating if the growth should be applied to columnar layer
	bool applyToPeripodialMembrane; ///< Boolean stating if the growth should be applied to peripodial membrane
	bool applyToBasalECM;			///< Boolean stating if the growth should be applied to basal explicit ECM layer
	bool applyToLateralECM ;		///< Boolean stating if the growth should be applied to lateral explicit ECM layer
    bool applyTissueApical;         ///< Boolean stating if the growth should be applied to apical side of the tissue
    bool applyTissueBasal;          ///< Boolean stating if the growth should be applied to basal side of the tissue
    bool applyTissueMidLine;        ///< Boolean stating if the growth should be applied to mid-line side of the tissue
    double zMin;                    ///< Double giving the minimum z fraction of the tissue height that the growth is applied to
    double zMax;                    ///< Double giving the maximum z fraction of the tissue height that the growth is applied to
    std::vector <int> appliedEllipseBandIds;    ///< The vector of integers holding the iIDs of the marker ellipse bands that the growth isapplied to

    void 	ParentErrorMessage(std::string functionName){
        std::cerr<<"You are calling the function: "<<functionName<<" from a parent here, check declaration is via pointers"<<std::endl;
	}
    double 	ParentErrorMessage(std::string functionName, double returnValue){
        std::cerr<<"You are calling the function: "<<functionName<<" from a parent here, check declaration is via pointers"<<std::endl;
		return returnValue;
	}
    int 	ParentErrorMessage(std::string functionName, int returnValue){
        std::cerr<<"You are calling the function: "<<functionName<<" from a parent here, check declaration is via pointers"<<std::endl;
		return returnValue;
	}
    virtual void		writeSummary(std::ofstream&/*saveFileSimulationSummary*/){ParentErrorMessage("writeSummary_no_dt");} /// Virtual function of the parent to write the summary of growthFunction.
    virtual void		writeSummary(std::ofstream&/*saveFileSimulationSummary*/, double /*dt*/){ParentErrorMessage("writeSummary_with_dt");} /// Virtual function of the parent to write the summary of growthFunction.
    virtual void 		getCentre(float &/*centreX*/, float &/*centreY*/){ParentErrorMessage("getCentre");} /// Virtual function of the parent to get centre of the growth function definition.
    virtual float 		getInnerRadius(){return ParentErrorMessage("getInnerRadius",0.0);} /// Virtual function of the parent to get inner radius of the growth function definition.
    virtual float 		getOuterRadius(){return ParentErrorMessage("getOuterRadius",0.0);} /// Virtual function of the parent to get outer radius of the growth function definition.
    virtual std::array<double,3> 		getGrowthRate(){ParentErrorMessage("getGrowthRate");return std::array<double,3>{0.0};} /// Virtual function of the parent to get growth rate of the growth function definition.
    virtual std::array<double,3> 		getShapeChangeRateRate(){ParentErrorMessage("getShapeChangeRateRate");return std::array<double,3>{0.0};} /// Virtual function of the parent to get shape change rate of the growth function definition.
    virtual gsl_matrix* getShearAngleRotationMatrix(){ParentErrorMessage("getShearAngleRotationMatrix"); gsl_matrix* dummy = gsl_matrix_calloc(0,0); return dummy;}  /// Virtual function of the parent to get rotation angle matrix of the growth function definition.
    virtual double 		getShearAngle(){ParentErrorMessage("getShearAngle");return 0.0;} /// Virtual function of the parent to get rotation angle of the growth function definition.
    virtual size_t      getGridX(){return ParentErrorMessage("getGridX",0);} /// Virtual function of the parent to get gridX of the growth function definition.
    virtual size_t      getGridY(){return ParentErrorMessage("getGridY",0);} /// Virtual function of the parent to get gridY of the growth function definition.
    virtual	double 		getGrowthMatrixElement(int /*i*/, int /*j*/, int /*k*/){return ParentErrorMessage("getGrowthMatrixElement",0.0);} /// Virtual function of the parent to get grid matrix element of the growth function definition.
    virtual	double 		getXyShearAngleMatrixElement(int /*i*/, int /*j*/){return ParentErrorMessage("getXyShearhMatrixElement",0.0);} /// Virtual function of the parent to get grid matrix element of the growth function definition.
    virtual bool 		isAspectRatioOverOne(int /*i*/, int /*j*/){return ParentErrorMessage("isAspectRatioOverOne",0);}/// Virtual function of the parent to check aspect ratioof the growth function definition.
    virtual gsl_matrix* getXyShearRotationsMatrixElement(int /*i*/, int /*j*/){ParentErrorMessage("getShearAngleRotationMatrixElement");gsl_matrix* dummy = gsl_matrix_calloc(0,0); return dummy;} //this is used by grid based growth
    virtual void 		getGrowthProfileAt4Corners(int /*IndexX*/, int /*IndexY*/, std::array<double,3>& /*growth0*/, std::array<double,3>&/*growth1*/, std::array<double,3>&/*growth2*/, std::array<double,3>&/*growth3*/, std::array<double,4>&/*angles*/, std::array<bool,4>&/*anglesEliminated*/){ParentErrorMessage("getGrowthProfileAt4Corners");}

    virtual void		setGrowtRate(double /*ex*/, double /*ey*/, double /*ez*/){ParentErrorMessage("setGrowtRate");}/// Virtual function of the parent to set growth rateof the growth function definition.
    virtual void		setGrowthMatrixElement(double /*ex*/, double /*ey*/, double /*ez*/, int /*i*/, int /*j*/){ParentErrorMessage("setGrowtRate");} /// Virtual function of the parent to set growth rateof the growth function definition.
};
#endif


#ifndef NOVUGNO_IMI_INCLUDE_GNOIMI_H_
#define NOVUGNO_IMI_INCLUDE_GNOIMI_H_

#include <algorithm>
#include <cassert>
#include <cmath>
#include <iostream>
#include <iomanip>
#include <fstream>
#include <sstream>
#include <string>
#include <thread>
#include <functional>
#include <vector>
#include <ctime>
#include <chrono>

#include <mutex>
#include <cblas.h>

struct threadInfo;

class GNOIMI {
    public:
    private:
	int D;
	int K;
	int totalLearnCount;
	int learnIterationsCount;
	int L;

	int threadsCount;
	int totalPoints_perThread;
	int Init_chunkSize;

	int* coarseAssigns;
	int* fineAssigns;
	float* alphaNum;
	float* alphaDen;
	float* alpha;

	vector<float*> alphaNumerators;
	vector<float*> alphaDenominators;

	float* coarseVocab;
	float* fineVocab;
	float* fineVocabNum;
	float* fineVocabDen;
	float* coarseVocabNum;
	float* coarseVocabDen;

	vector<float*> fineVocabNumerators;
	vector<float*> fineVocabDenominators;
	vector<float*> coarseVocabNumerators;
	vector<float*> coarseVocabDenominators;

	float* coarseNorms;
	float* fineNorms;
	float* coarseFineProducts;

	float* errors;

	string output_prefix;
};

using namespace std;

int D = 512;
int K = 657;
int totalLearnCount = 260000;
int learnIterationsCount = 20;
int L = 6;

string learnFilename = "./normalized_features_260k.fvecs";
string initCoarseFilename = "./coarse_260k.fvecs";
string initFineFilename = "./fine_260k.fvecs";
string outputFilesPrefix = "./260K_new_";

int threadsCount = 3;
int totalPoints_perThread = 0;
const int Init_chunkSize = 3000;

int* coarseAssigns = (int*)malloc(totalLearnCount * sizeof(int));
int* fineAssigns = (int*)malloc(totalLearnCount * sizeof(int));
float* alphaNum = (float*)malloc(K * K * sizeof(float));
float* alphaDen = (float*)malloc(K * K * sizeof(float));
float* alpha = (float*)malloc(K * K * sizeof(float));

vector<float*> alphaNumerators(threadsCount);
vector<float*> alphaDenominators(threadsCount);

float* coarseVocab = (float*)malloc(D * K * sizeof(float));
float* fineVocab = (float*)malloc(D * K * sizeof(float));
float* fineVocabNum = (float*)malloc(D * K * sizeof(float));
float* fineVocabDen = (float*)malloc(K * sizeof(float));
float* coarseVocabNum = (float*)malloc(D * K * sizeof(float));
float* coarseVocabDen = (float*)malloc(K * sizeof(float));

vector<float*> fineVocabNumerators(threadsCount);
vector<float*> fineVocabDenominators(threadsCount);
vector<float*> coarseVocabNumerators(threadsCount);
vector<float*> coarseVocabDenominators(threadsCount);

float* coarseNorms = (float*)malloc(K * sizeof(float));
float* fineNorms = (float*)malloc(K * sizeof(float));
float* coarseFineProducts = (float*)malloc(K * K * sizeof(float));

float* errors = (float*)malloc(threadsCount * sizeof(float));
mutex mtx;

struct threadInfo {
    long long startId;
    int pointsCount;
    int chunksCount;
    int trainThreadChunkSize;
    int extra_ChunkSize;
};

threadInfo get_threadInfo(const int& threadId) {
    threadInfo t_info;
    t_info.startId = totalPoints_perThread * threadId;
    t_info.trainThreadChunkSize = Init_chunkSize;

    if (totalPoints_perThread * (threadId + 1) <= totalLearnCount) {
	t_info.pointsCount = totalPoints_perThread;
    } else {
	t_info.pointsCount = totalLearnCount - totalPoints_perThread * threadId;
    }

    t_info.extra_ChunkSize = 0;
    if (t_info.pointsCount % t_info.trainThreadChunkSize == 0) {
	t_info.chunksCount = t_info.pointsCount / t_info.trainThreadChunkSize;
    } else {
	t_info.extra_ChunkSize = t_info.pointsCount % t_info.trainThreadChunkSize;
	t_info.chunksCount = (t_info.pointsCount - t_info.extra_ChunkSize) / t_info.trainThreadChunkSize + 1;
    }

    return t_info;
}

void computeOptimalAssignsSubset(int threadId) {

    threadInfo cur_t_info = get_threadInfo(threadId);
    int trainThreadChunkSize = cur_t_info.trainThreadChunkSize;

    errors[threadId] = 0.0;

    FILE* learnStream = fopen(learnFilename.c_str(), "r");
    fseek(learnStream, cur_t_info.startId * (D + 1) * sizeof(float), SEEK_SET);

    float* pointsCoarseTerms = (float*)malloc(trainThreadChunkSize * K * sizeof(float));
    float* pointsFineTerms = (float*)malloc(trainThreadChunkSize * K * sizeof(float));
    float* chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));

    std::vector<std::pair<float, int> > coarseScores(K);

    for(int chunkId = 0; chunkId < cur_t_info.chunksCount; ++chunkId) {
	std::cout << "[Assigns][Thread " << threadId << "] " << "processing chunk " <<  chunkId << " of " << cur_t_info.chunksCount << "\n";

	if (chunkId == cur_t_info.chunksCount - 1 && cur_t_info.extra_ChunkSize != 0) {
	    free(pointsCoarseTerms);
	    free(pointsFineTerms);
	    free(chunkPoints);

	    trainThreadChunkSize = cur_t_info.extra_ChunkSize;
	    pointsCoarseTerms = (float*)malloc(trainThreadChunkSize * K * sizeof(float));
	    pointsFineTerms = (float*)malloc(trainThreadChunkSize * K * sizeof(float));
	    chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));
	}

	fvecs_fread(learnStream, chunkPoints, trainThreadChunkSize, D);
	fmat_mul_full(coarseVocab, chunkPoints, K, trainThreadChunkSize, D, "TN", pointsCoarseTerms);
	fmat_mul_full(fineVocab, chunkPoints, K, trainThreadChunkSize, D, "TN", pointsFineTerms);

	for(int pointId = 0; pointId < trainThreadChunkSize; ++pointId) {
	    cblas_saxpy(K, -1.0, coarseNorms, 1, pointsCoarseTerms + pointId * K, 1);

	    for(int k = 0; k < K; ++k) {
		coarseScores[k].first = (-1.0) * pointsCoarseTerms[pointId * K + k];
		coarseScores[k].second = k;
	    }

	    std::sort(coarseScores.begin(), coarseScores.end());
	    float currentMinScore = 999999999.0;
	    int currentMinCoarseId = -1;
	    int currentMinFineId = -1;

	    for(int l = 0; l < L; ++l) {
		//examine cluster l
		int currentCoarseId = coarseScores[l].second;
		float currentCoarseTerm = coarseScores[l].first;
		for(int currentFineId = 0; currentFineId < K; ++currentFineId) {
		    float alphaFactor = alpha[currentCoarseId * K + currentFineId];
		    float score = currentCoarseTerm + alphaFactor * coarseFineProducts[currentCoarseId * K + currentFineId] +
			(-1.0) * alphaFactor * pointsFineTerms[pointId * K + currentFineId] +
			alphaFactor * alphaFactor * fineNorms[currentFineId];

		    if(score < currentMinScore) {
			currentMinScore = score;
			currentMinCoarseId = currentCoarseId;
			currentMinFineId = currentFineId;
		    }
		}
	    }

	    coarseAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId] = currentMinCoarseId;
	    fineAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId] = currentMinFineId;

	    errors[threadId] += currentMinScore * 2 + 1.0; // point has a norm equals 1.0
	}
    }

    fclose(learnStream);
    free(chunkPoints);
    free(pointsCoarseTerms);
    free(pointsFineTerms);
}

void computeOptimalAlphaSubset(int threadId) {
    memset(alphaNumerators[threadId], 0, K * K * sizeof(float));
    memset(alphaDenominators[threadId], 0, K * K * sizeof(float));

    threadInfo cur_t_info = get_threadInfo(threadId);
    int trainThreadChunkSize = cur_t_info.trainThreadChunkSize;

    FILE* learnStream = fopen(learnFilename.c_str(), "r");
    fseek(learnStream, cur_t_info.startId * (D + 1) * sizeof(float), SEEK_SET);

    float* residual = (float*)malloc(D * sizeof(float));
    float* chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));

    for(int chunkId = 0; chunkId < cur_t_info.chunksCount; ++chunkId) {
	std::cout << "[Alpha][Thread " << threadId << "] " << "processing chunk " <<  chunkId << " of " << cur_t_info.chunksCount << "\n";

	if (chunkId == cur_t_info.chunksCount - 1 && cur_t_info.extra_ChunkSize != 0) {
	    free(chunkPoints);

	    trainThreadChunkSize = cur_t_info.extra_ChunkSize;
	    chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));
	}

	fvecs_fread(learnStream, chunkPoints, trainThreadChunkSize, D);
	for(int pointId = 0; pointId < trainThreadChunkSize; ++pointId) {
	    //S_k
	    int coarseAssign = coarseAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId];
	    //T_l
	    int fineAssign = fineAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId];

	    memcpy(residual, chunkPoints + pointId * D, D * sizeof(float));
	    //P_i - S_k
	    cblas_saxpy(D, -1.0, coarseVocab + coarseAssign * D, 1, residual, 1);

	    //<P_i - S_k, T_l>
	    alphaNumerators[threadId][coarseAssign * K + fineAssign] +=
		cblas_sdot(D, residual, 1, fineVocab + fineAssign * D, 1);

	    //||T||^2
	    alphaDenominators[threadId][coarseAssign * K + fineAssign] += fineNorms[fineAssign] * 2; // we keep halves of norms
	}
    }
    fclose(learnStream);
    free(chunkPoints);
    free(residual);
}

void computeOptimalFineVocabSubset(int threadId) {
    memset(fineVocabNumerators[threadId], 0, K * D * sizeof(float));
    memset(fineVocabDenominators[threadId], 0, K * sizeof(float));

    threadInfo cur_t_info = get_threadInfo(threadId);
    int trainThreadChunkSize = cur_t_info.trainThreadChunkSize;

    FILE* learnStream = fopen(learnFilename.c_str(), "r");
    fseek(learnStream, cur_t_info.startId * (D + 1) * sizeof(float), SEEK_SET);

    float* residual = (float*)malloc(D * sizeof(float));
    float* chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));

    for(int chunkId = 0; chunkId < cur_t_info.chunksCount; ++chunkId) {
	//std::cout << "[Fine vocabs][Thread " << threadId << "] " << "processing chunk " <<  chunkId << " of " << chunksCount << "\n";

	if (chunkId == cur_t_info.chunksCount - 1 && cur_t_info.extra_ChunkSize != 0) {
	    free(chunkPoints);

	    trainThreadChunkSize = cur_t_info.extra_ChunkSize;
	    chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));
	}

	fvecs_fread(learnStream, chunkPoints, trainThreadChunkSize, D);
	for(int pointId = 0; pointId < trainThreadChunkSize; ++pointId) {

	    int coarseAssign = coarseAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId];
	    int fineAssign = fineAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId];

	    float alphaFactor = alpha[coarseAssign * K + fineAssign];
	    memcpy(residual, chunkPoints + pointId * D, D * sizeof(float));

	    cblas_saxpy(D, -1.0, coarseVocab + coarseAssign * D, 1, residual, 1);
	    cblas_saxpy(D, alphaFactor, residual, 1, fineVocabNumerators[threadId] + fineAssign * D, 1);
	    fineVocabDenominators[threadId][fineAssign] += alphaFactor * alphaFactor;
	}
    }

    fclose(learnStream);
    free(chunkPoints);
    free(residual);
}

void computeOptimalCoarseVocabSubset(int threadId) {
    memset(coarseVocabNumerators[threadId], 0, K * D * sizeof(float));
    memset(coarseVocabDenominators[threadId], 0, K * sizeof(float));

    threadInfo cur_t_info = get_threadInfo(threadId);
    int trainThreadChunkSize = cur_t_info.trainThreadChunkSize;

    FILE* learnStream = fopen(learnFilename.c_str(), "r");
    fseek(learnStream, cur_t_info.startId * (D + 1) * sizeof(float), SEEK_SET);

    float* residual = (float*)malloc(D * sizeof(float));
    float* chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));

    for(int chunkId = 0; chunkId < cur_t_info.chunksCount; ++chunkId) {
	//std::cout << "[Coarse vocabs][Thread " << threadId << "] " << "processing chunk " <<  chunkId << " of " << chunksCount << "\n";

	if (chunkId == cur_t_info.chunksCount - 1 && cur_t_info.extra_ChunkSize != 0) {
	    free(chunkPoints);

	    trainThreadChunkSize = cur_t_info.extra_ChunkSize;
	    chunkPoints = (float*)malloc(trainThreadChunkSize * D * sizeof(float));
	}

	fvecs_fread(learnStream, chunkPoints, trainThreadChunkSize, D);
	for(int pointId = 0; pointId < trainThreadChunkSize; ++pointId) {
	    int coarseAssign = coarseAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId];
	    int fineAssign = fineAssigns[cur_t_info.startId + chunkId * cur_t_info.trainThreadChunkSize + pointId];
	    float alphaFactor = alpha[coarseAssign * K + fineAssign];
	    memcpy(residual, chunkPoints + pointId * D, D * sizeof(float));
	    cblas_saxpy(D, -1.0 * alphaFactor, fineVocab + fineAssign * D, 1, residual, 1);
	    cblas_saxpy(D, 1, residual, 1, coarseVocabNumerators[threadId] + coarseAssign * D, 1);
	    coarseVocabDenominators[threadId][coarseAssign] += 1.0;
	}
    }

    fclose(learnStream);
    free(chunkPoints);
    free(residual);
}

int main() {
    totalPoints_perThread = totalLearnCount / threadsCount;

    //increase the threadCount
    if (totalLearnCount % threadsCount != 0) {
       	threadsCount += 1;
	alphaNumerators.resize(threadsCount);
	alphaDenominators.resize(threadsCount);
	fineVocabNumerators.resize(threadsCount);
	fineVocabDenominators.resize(threadsCount);
	coarseVocabNumerators.resize(threadsCount);
	coarseVocabDenominators.resize(threadsCount);
	free(errors);
	errors = (float*)malloc(threadsCount * sizeof(float));
    }

    for(int threadId = 0; threadId < threadsCount; ++threadId) {
	alphaNumerators[threadId] = (float*)malloc(K * K * sizeof(float*));
	alphaDenominators[threadId] = (float*)malloc(K * K * sizeof(float*));
    }

    for(int threadId = 0; threadId < threadsCount; ++threadId) {
	fineVocabNumerators[threadId] = (float*)malloc(K * D * sizeof(float*));
	fineVocabDenominators[threadId] = (float*)malloc(K * sizeof(float*));
    }

    for(int threadId = 0; threadId < threadsCount; ++threadId) {
	coarseVocabNumerators[threadId] = (float*)malloc(K * D * sizeof(float));
	coarseVocabDenominators[threadId] = (float*)malloc(K * sizeof(float));
    }

    // init vocabs
    fvecs_read(initCoarseFilename.c_str(), D, K, coarseVocab);
    fvecs_read(initFineFilename.c_str(), D, K, fineVocab);

    // init alpha
    for(int i = 0; i < K * K; ++i) {
	alpha[i] = 1.0f;
    }

    // learn iterations
    std::cout << "Start learning iterations...\n";
    for(int it = 0; it < learnIterationsCount; ++it) {
	cout << "learning iteration " << it << endl;
	for(int k = 0; k < K; ++k) {
	    coarseNorms[k] = cblas_sdot(D, coarseVocab + k * D, 1, coarseVocab + k * D, 1) / 2;
	    fineNorms[k] = cblas_sdot(D, fineVocab + k * D, 1, fineVocab + k * D, 1) / 2;
	}

	fmat_mul_full(fineVocab, coarseVocab, K, K, D, "TN", coarseFineProducts);

	// update Assigns
	vector<std::thread> workers;
	memset(errors, 0, threadsCount * sizeof(float));
	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers.push_back(std::thread(computeOptimalAssignsSubset, threadId));
	}

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers[threadId].join();
	}

	float totalError = 0.0;
	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    cout << "threadId " << threadId << ": err " << errors[threadId] << endl;
	    totalError += errors[threadId];
	}
	std::cout << "Current reconstruction error... " << totalError / totalLearnCount << "\n";

	// update alpha
	workers.clear();
	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers.push_back(std::thread(computeOptimalAlphaSubset, threadId));
	}
	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers[threadId].join();
	}

	// update fine Vocabs
	workers.clear();
	memset(alphaNum, 0, K * K * sizeof(float));
	memset(alphaDen, 0, K * K * sizeof(float));

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    cblas_saxpy(K * K, 1, alphaNumerators[threadId], 1, alphaNum, 1);
	    cblas_saxpy(K * K, 1, alphaDenominators[threadId], 1, alphaDen, 1);
	}

	for(int i = 0; i < K * K; ++i) {
	    alpha[i] = (alphaDen[i] == 0) ? 1.0 : alphaNum[i] / alphaDen[i];
	}

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers.push_back(std::thread(computeOptimalFineVocabSubset, threadId));
	}

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers[threadId].join();
	}

	// update coarse Vocabs
	workers.clear();
	memset(fineVocabNum, 0, K * D * sizeof(float));
	memset(fineVocabDen, 0, K * sizeof(float));

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    cblas_saxpy(K * D, 1, fineVocabNumerators[threadId], 1, fineVocabNum, 1);
	    cblas_saxpy(K, 1, fineVocabDenominators[threadId], 1, fineVocabDen, 1);
	}

	for(int i = 0; i < K * D; ++i) {
	    fineVocab[i] = (fineVocabDen[i / D] == 0) ? 0 : fineVocabNum[i] / fineVocabDen[i / D];
	}

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers.push_back(std::thread(computeOptimalCoarseVocabSubset, threadId));
	}

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    workers[threadId].join();
	}

	workers.clear();
	memset(coarseVocabNum, 0, K * D * sizeof(float));
	memset(coarseVocabDen, 0, K * sizeof(float));

	for(int threadId = 0; threadId < threadsCount; ++threadId) {
	    cblas_saxpy(K * D, 1, coarseVocabNumerators[threadId], 1, coarseVocabNum, 1);
	    cblas_saxpy(K, 1, coarseVocabDenominators[threadId], 1, coarseVocabDen, 1);
	}

	for(int i = 0; i < K * D; ++i) {
	    coarseVocab[i] = (coarseVocabDen[i / D] == 0) ? 0 : coarseVocabNum[i] / coarseVocabDen[i / D];
	}

	// save current alpha and vocabs
	std::stringstream alphaFilename;
	alphaFilename << outputFilesPrefix << "alpha_" << it << ".fvecs";
	fvecs_write(alphaFilename.str().c_str(), K, K, alpha);

	std::stringstream fineVocabFilename;
	fineVocabFilename << outputFilesPrefix << "fine_" << it << ".fvecs";
	fvecs_write(fineVocabFilename.str().c_str(), D, K, fineVocab);

	std::stringstream coarseVocabFilename;
	coarseVocabFilename << outputFilesPrefix << "coarse_" << it << ".fvecs";
	fvecs_write(coarseVocabFilename.str().c_str(), D, K, coarseVocab);

	cout << "iteration " << it << " finished!" << endl;
    }

    free(coarseAssigns);
    free(fineAssigns);
    free(alphaNum);
    free(alphaDen);
    free(alpha);
    free(coarseVocab);
    free(coarseVocabNum);
    free(coarseVocabDen);
    free(fineVocab);
    free(fineVocabNum);
    free(fineVocabDen);
    free(coarseNorms);
    free(fineNorms);
    free(coarseFineProducts);
    free(errors);
    return 0;
}
#endif  //NOVUGNO_IMI_INCLUDE_GNOIMI_H_


/*
 * Copyright (c) 2010-2018 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2013      Inria. All rights reserved.
 *
 * @precisions normal z -> c d s
 *
 */

#include <lapacke.h>
#include "dplasma.h"
#include "dplasmatypes.h"

#include "map2.h"

static int
dplasma_zlacpy_sum_operator( parsec_execution_stream_t *es,
                             const parsec_tiled_matrix_dc_t *descA,
                             const parsec_tiled_matrix_dc_t *descB,
                             const void *_A, void *_B,
                             PLASMA_enum uplo, int m, int n,
                             void *args )
{
    int tempmm, tempnn, ldam, ldbm;
    const parsec_complex64_t *A = (const parsec_complex64_t*)_A;
    parsec_complex64_t       *B = (parsec_complex64_t*)_B;
    (void)es;
    (void)args;

    tempmm = ((m)==((descA->mt)-1)) ? ((descA->m)-(m*(descA->mb))) : (descA->mb);
    tempnn = ((n)==((descA->nt)-1)) ? ((descA->n)-(n*(descA->nb))) : (descA->nb);
    ldam = BLKLDD( descA, m );
    ldbm = BLKLDD( descB, m );

    parsec_complex64_t* unit_v = (parsec_complex64_t*)malloc(tempnn*sizeof(parsec_complex64_t));
    parsec_complex64_t* int_v = (parsec_complex64_t*)malloc(tempnn*sizeof(parsec_complex64_t));
    for(int i = 0; i < tempnn; i++ ) {
        unit_v[i] = (parsec_complex64_t)1.0;
        int_v[i] = (parsec_complex64_t)(1.0*(i+1));
    }
    parsec_complex64_t* unit_vt = (parsec_complex64_t*)malloc((tempmm+1)*sizeof(parsec_complex64_t));
    parsec_complex64_t* int_vt = (parsec_complex64_t*)malloc((tempmm+1)*sizeof(parsec_complex64_t));
    for(int i = 0; i < tempmm; i++ ) {
        unit_vt[i] = (parsec_complex64_t)1.0;
        int_vt[i] = (parsec_complex64_t)(1.0*(i+1));
    }

    LAPACKE_zlacpy_work(
        LAPACK_COL_MAJOR, lapack_const( PlasmaUpperLower ), tempmm, tempnn, A, ldam, B, ldbm);

    CORE_zgemv(PlasmaNoTrans, tempmm, tempnn,
               (parsec_complex64_t)1.0, B, ldbm,
               unit_v, 1,
               (parsec_complex64_t)0.0, B+ldbm*(tempnn), 1);

    CORE_zgemv(PlasmaNoTrans, tempmm, tempnn,
               (parsec_complex64_t)1.0, B, ldbm,
               int_v, 1,
               (parsec_complex64_t)0.0, B+ldbm*(tempnn+1), 1);

    CORE_zgemv(PlasmaTrans, tempmm, tempnn+1,
               (parsec_complex64_t)1.0, B, ldbm,
               unit_vt, 1,
               (parsec_complex64_t)0.0, B+(tempmm), ldbm);

    CORE_zgemv(PlasmaTrans, tempmm, tempnn+1,
               (parsec_complex64_t)1.0, B, ldbm,
               int_vt, 1,
               (parsec_complex64_t)0.0, B+(tempmm+1), ldbm);

    free(unit_v);
    free(unit_vt);
    free(int_v);
    free(int_vt);

    return 0;
}

/**
 *******************************************************************************
 *
 * @ingroup dplasma_complex64
 *
 * dplasma_zlacpy_New - Generates an object that performs a copy of the matrix A
 * into the matrix B.
 *
 * See dplasma_map2_New() for further information.
 *
 * WARNING: The computations are not done by this call.
 *
 *******************************************************************************
 *
 * @param[in] uplo
 *          Specifies which part of matrix A is copied:
 *          = PlasmaUpperLower: All matrix is referenced.
 *          = PlasmaUpper:      Only upper part is refrenced.
 *          = PlasmaLower:      Only lower part is referenced.
 *
 * @param[in] A
 *          Descriptor of the distributed original matrix A. Any tiled matrix
 *          descriptor can be used. However, if the data is stored in column
 *          major, the tile distribution must match the one of the matrix B.
 *
 * @param[in,out] B
 *          Descriptor of the distributed destination matrix B. Any tiled matrix
 *          descriptor can be used, with no specific storage.
 *
 *******************************************************************************
 *
 * @return
 *          \retval NULL if incorrect parameters are given.
 *          \retval The parsec taskpool describing the operation that can be
 *          enqueued in the runtime. It, then, needs to be
 *          destroy with the corresponding destruct function.
 *
 *******************************************************************************
 *
 * @sa dplasma_zlacpy
 * @sa dplasma_zlacpy_Destruct
 * @sa dplasma_clacpy_New
 * @sa dplasma_dlacpy_New
 * @sa dplasma_slacpy_New
 *
 ******************************************************************************/
parsec_taskpool_t*
dplasma_zlacpy_sum_New( PLASMA_enum uplo,
                        const parsec_tiled_matrix_dc_t *A,
                        parsec_tiled_matrix_dc_t *B)
{
    parsec_taskpool_t* tp;

    tp = dplasma_map2_New(uplo, PlasmaNoTrans, A, B,
                          dplasma_zlacpy_sum_operator, NULL );

    return tp;
}

/**
 *******************************************************************************
 *
 * @ingroup dplasma_complex64
 *
 *  dplasma_zlacpy_Destruct - Free the data structure associated to an object
 *  created with dplasma_zlacpy_New().
 *
 *******************************************************************************
 *
 * @param[in,out] o
 *          On entry, the object to destroy.
 *          On exit, the object cannot be used anymore.
 *
 *******************************************************************************
 *
 * @sa dplasma_zlacpy_New
 * @sa dplasma_zlacpy
 *
 ******************************************************************************/
void
dplasma_zlacpy_sum_Destruct( parsec_taskpool_t *tp )
{
    parsec_taskpool_free(tp);
}

/**
 *******************************************************************************
 *
 * @ingroup dplasma_complex64
 *
 * dplasma_zlacpy - Generates an object that performs a copy of the matrix A
 * into the matrix B.
 *
 * See dplasma_map2() for further information.
 *
 *******************************************************************************
 *
 * @param[in,out] parsec
 *          The PaRSEC context of the application that will run the operation.
 *
 * @param[in] uplo
 *          Specifies which part of matrix A is copied:
 *          = PlasmaUpperLower: All matrix is referenced.
 *          = PlasmaUpper:      Only upper part is refrenced.
 *          = PlasmaLower:      Only lower part is referenced.
 *
 * @param[in] A
 *          Descriptor of the distributed original matrix A. Any tiled matrix
 *          descriptor can be used. However, if the data is stored in column
 *          major, the tile distribution must match the one of the matrix B.
 *
 * @param[in,out] B
 *          Descriptor of the distributed destination matrix B. Any tiled matrix
 *          descriptor can be used, with no specific storage.
 *
 *******************************************************************************
 *
 * @return
 *          \retval -i if the ith parameters is incorrect.
 *          \retval 0 on success.
 *
 *******************************************************************************
 *
 * @sa dplasma_zlacpy_New
 * @sa dplasma_zlacpy_Destruct
 * @sa dplasma_clacpy
 * @sa dplasma_dlacpy
 * @sa dplasma_slacpy
 *
 ******************************************************************************/
int
dplasma_zlacpy_sum( parsec_context_t *parsec,
                PLASMA_enum uplo,
                const parsec_tiled_matrix_dc_t *A,
                parsec_tiled_matrix_dc_t *B)
{
    parsec_taskpool_t *parsec_zlacpy_sum = NULL;

    if ((uplo != PlasmaUpperLower) &&
        (uplo != PlasmaUpper)      &&
        (uplo != PlasmaLower))
    {
        dplasma_error("dplasma_zlacpy_sum", "illegal value of uplo");
        return -2;
    }

    parsec_zlacpy_sum = dplasma_zlacpy_sum_New(uplo, A, B);

    if ( parsec_zlacpy_sum != NULL )
    {
        parsec_context_add_taskpool(parsec, parsec_zlacpy_sum);
        dplasma_wait_until_completion(parsec);
        dplasma_zlacpy_sum_Destruct( parsec_zlacpy_sum );
    }
    return 0;
}