# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

_md := function(lst)
    local i, str;
    str:="";
    if Length(lst) = 0  then
        return str;
    fi;
    str := Concat(str, StringInt(lst[1]));
    for i in [ 2 .. Length(lst) ]  do
        str := Concat(str, "x", StringInt(lst[i]));
    od;
    return str;
end;

#   doSimdDft argument parser
#
#   spiral> doSimdDft();
#   [ [ 4 ], SSE_2x64f, rec(
#      svct := true,
#      oddSizes := false ) ]
#   spiral> doSimdDft(1);
#   [ [ 4, 8 ], SSE_2x64f, rec(
#      svct := true,
#      oddSizes := false ) ]
#   spiral> doSimdDft([4,6]);
#   [ [ 4, 6 ], SSE_2x64f, rec(
#      svct := true,
#      oddSizes := true ) ]
#   spiral> doSimdDft(SSE_4x32f);
#   [ [ 16 ], SSE_4x32f, rec(
#      svct := true,
#      oddSizes := false ) ]
#   spiral> doSimdDft(1, SSE_4x32f);
#   [ [ 16, 32 ], SSE_4x32f, rec(
#      svct := true,
#      oddSizes := false ) ]
#   spiral> doSimdDft([4,6], SSE_4x32f);
#   [ [ 4, 6 ], SSE_4x32f, rec(
#      svct := true,
#      oddSizes := true ) ]
#   spiral> doSimdDft([4,6], SSE_4x32f, rec(svct:=false, oddSizes:=true));
#   [ [ 4, 6 ], SSE_4x32f, rec(
#      svct := false,
#      oddSizes := true ) ]

_parse := function(arglist, multiplier)
    local isa, n, sizes, optrec;

    isa := SIMD_ISA_DB.active()[1];
    optrec := rec(svct := true, oddSizes :=false);

    if Length(arglist) = 0 then
        n := 0;
    elif Length(arglist) = 1 then
        if IsInt(arglist[1]) then
            n := arglist[1];
        elif IsList(arglist[1]) then
            n := arglist[1];
            optrec := rec(svct := true, oddSizes :=true);
        else
            n := 0;
            isa := arglist[1];
        fi;
    elif Length(arglist) = 2 then
        n := arglist[1];
        isa := arglist[2];
        if IsList(arglist[1]) then
            optrec := rec(svct := true, oddSizes :=true);
        fi;
    else
        n := arglist[1];
        isa := arglist[2];
        optrec := arglist[3];
    fi;

    sizes := When(IsList(n), n, List([0..n], i->multiplier*isa.v^2*2^i));

    return [sizes, isa, optrec];
end;

codeSimdDft := function(arg)
    local sizes, opts, isa, res, t,rt, s, c, optrec;
    [sizes, isa, optrec] := _parse(arg, 1);
    opts := SIMDGlobals.getOpts(isa, optrec);

    t := ComplexT(DFT(sizes[1]), opts).withTags(opts.tags);
    rt := RandomRuleTree(t, opts);
    s := SumsRuleTreeOpts(rt, opts);
    c := CodeSumsOpts(s, opts);
    return rec(opts:=opts,t:=t,rt:=rt,s:=s,c:=c);
end;

_doSimd := function(sizes, isa, opts, dpr, name, trafo, mflops, stringint)
    local results, res, i;

    results := [];
    if not IsBound(dpr.hashTable) then dpr.hashTable := HashTableDP(); fi;

    Print(name, ": running ", isa, ": ", sizes, "...\n");
    for i in sizes do
      res := TimedAction(DP(trafo(i), dpr, opts));
      if Length(res[1]) > 0 then
        Add(results, res[1]);
        ExportCodeRuleTree(res[1][1].ruletree,
            Concat(name, "_", isa.name, "_", stringint(i)),
            CopyFields(opts, rec(verbosity := 0, fileinfo := rec(
                cycles := res[1][1].measured,
                flops := mflops(i),
                file := Concat(name, "_", isa.name, "_", stringint(i)),
                algorithm := res[1][1].ruletree
        ))));

        if opts.verify then
            _seqPerfStatsMflopsAcc(Concat(name, "_", isa.name, ".txt"),
                i, mflops(i), res[1][1].measured, res[2], IntDouble(d_log(1e-16+CVerifyRT(res[1][1].ruletree, opts))/d_log(10)));
        else
            _seqPerfStatsMflops(Concat(name, "_", isa.name, ".txt"),
                i, mflops(i), res[1][1].measured, res[2]);
        fi;
        HashSave(dpr.hashTable, Concat(name, "_", isa.name, ".hash"));
      fi;
    od;

    return results;
end;

#F doSimdDft(sizes, isa, optrec)
doSimdDft := function(arg)
    local sizes, opts, dpr, isa, optrec, trafos, dpbench, direction;

    [sizes, isa, optrec] := _parse(arg, 1);

	direction := When(IsBound(optrec.transInverse) and optrec.transInverse, 1, -1);

    opts := SIMDGlobals.getOpts(isa, optrec);
    opts.benchTransforms := List(sizes, k -> ComplexT(DFT(k, direction), opts).withTags(opts.tags));

    if not IsBound(optrec.globalUnrolling) and ForAny(sizes, i -> not IsInt(i/isa.v^2)) then
        optrec.globalUnrolling := 16384;
    fi;
    dpr  := rec(verbosity := 0, timeBaseCases:=true);
    dpbench := spiral.libgen.DPBench(rec((Concat(isa.name, When(opts.vector.conf.interleavedComplex, "_ic", "_sc"))) := opts), dpr);
    if IsBound(optrec.verify) then dpbench.matrixVerify := optrec.verify; fi;

	dpbench.sizes := sizes;

    return dpbench;
end;

#F doSimdWht(sizes, isa, optrec)
doSimdWht := function(arg)
    local sizes, isa, opts, optrec, defrec, dpr, dpbench;

    [sizes, isa, optrec] := _parse(arg, 1);

    defrec := rec(
        oddSizes := false,
        svct := true,
        stdTTensor := true,
        tsplPFA := false,
        splitL:=false,
        realVect := true,
        cplxVect := false,
        useConj := false,
    );
    opts := SIMDGlobals.getOpts(isa, CopyFields(optrec, defrec));
    opts.benchTransforms := List(sizes, k -> WHT(Log2Int(k)).withTags(opts.tags));
    #opts.benchTransforms := List(sizes, k -> GT(WHT(Log2Int(k)), GTPar, GTPar, [4]).withTags(opts.tags));
    #opts.benchTransforms := List(sizes, k -> GT(WHT(Log2Int(k)), GTPar, GTPar, [8]).withTags(opts.tags));
    dpr  := rec(verbosity := 0, timeBaseCases:=true);

    dpbench := spiral.libgen.DPBench(rec((isa.name) := opts), dpr);
    if IsBound(optrec.verify) then dpbench.matrixVerify := optrec.verify; fi;

	dpbench.sizes := sizes;

    return dpbench;
end;

doSimdMddft := function(arg)
    local sizes, opts, tags, dpr, isa, defrec, optrec, dpbench, direction;

    [sizes, isa, optrec] := _parse(arg, 1);

	direction := When(IsBound(optrec.transInverse) and optrec.transInverse, 1, -1);

    # for cannot do larger sizes at this point
    defrec := rec(
        realVect := true,
        cplxVect := false,
        useConj := false,
    );
    opts := SIMDGlobals.getOpts(isa, CopyFields(optrec, defrec));

    opts.benchTransforms := List(sizes, k -> ComplexT(MDDFT(k, direction), opts).withTags(opts.tags));
    dpr  := rec(verbosity := 0, timeBaseCases:=true);

    dpbench := spiral.libgen.DPBench(rec((Concat(isa.name, When(opts.vector.conf.interleavedComplex, "_ic", "_sc"))) := opts), dpr);
    if IsBound(optrec.verify) then dpbench.matrixVerify := optrec.verify; fi;

	dpbench.sizes := sizes;

    return dpbench;
end;

codeSimdSymDFT := function(arg)
    local sizes, opts, dpr, isa, defrec, optrec, t, r, s, c, ttype;

    ttype := arg[1];
    [sizes, isa, optrec] := _parse(Drop(arg, 1), 2);
    defrec := rec(oddSizes := false, svct := true, splitL:=false, realVect := true,
                  cplxVect := false, useConj := false, globalUnrolling := 1024);
    opts := SIMDGlobals.getOpts(isa, CopyFields(optrec, defrec));
    t := ttype(sizes[1]).withTags(opts.tags);
    r := RandomRuleTree(t, opts);
    s := SumsRuleTree(r, opts);
    c := CodeSums(s, opts);
    return rec(opts:=opts, t:=t, rt:=r, s:=s, c:=c);
end;

doSimdSymDFT := function(arg)
    local sizes, opts, dpr, isa, optrec, results, defrec, ur, ttype, dpbench;

    ttype := arg[1];
    [sizes, isa, optrec] := _parse(Drop(arg, 1), 2);

    defrec := rec(
        oddSizes := false,
        svct := true,
        splitL:=false,
        realVect := true,
        cplxVect := false,
        useConj := false,
        globalUnrolling := 1024
    );
    ur := rec(
        globalUnrollingMin := 1024,
        globalUnrollingMax := 1024
    );
    opts := SIMDGlobals.getOpts(isa, CopyFields(defrec, optrec));
    opts.benchTransforms := List(sizes, k -> ttype(k).withTags(Cond(IsBound(optrec.cplxVect) and optrec.cplxVect, opts.cxtags, opts.tags)));
    if not IsBound(optrec.globalUnrolling) and ForAny(sizes, i -> not IsInt(i/isa.v^2)) then
        optrec.globalUnrolling := 16384;
    fi;
    dpr  := rec(verbosity := 0, timeBaseCases:=true);

    dpbench := spiral.libgen.DPBench(rec((isa.name) := opts), dpr);
    if IsBound(optrec.verify) then dpbench.matrixVerify := optrec.verify; fi;

	dpbench.sizes := sizes;

    return dpbench;

#    results := _doSimd(sizes, isa, opts, dpr, arg[1], k -> ttype(k).withTags(opts.tags), i->2*i*d_log(i)/d_log(2) + i, StringInt);
#    return rec(opts:=opts, results:=results);
end;

doSimdSymMDDFT := function(arg)
    local sizes, isa, opts, optrec, defrec, dpr, dpbench, ttype;

    ttype := arg[1];
    [sizes, isa, optrec] := _parse(Drop(arg, 1), 2);

    defrec := rec(svct := true, splitL := false, stdTTensor := true, tsplRader:=false, tsplBluestein:=false,
        tsplPFA:=false, oddSizes:=false, interleavedComplex := false, pushTag := false, flipIxA := true);
    opts := SIMDGlobals.getOpts(isa, CopyFields(optrec, defrec));

    opts.benchTransforms := List(sizes, k -> TTensor(ttype(k), ttype(k)).withTags(opts.tags));
    dpr  := rec(verbosity := 0, timeBaseCases:=true);

    dpbench := spiral.libgen.DPBench(rec((isa.name) := opts), dpr);
    if IsBound(optrec.verify) then dpbench.matrixVerify := optrec.verify; fi;

	dpbench.sizes := sizes;

    return dpbench;
end;

doSimdMdConv := function(arg)
    local sizes, isa, opts, optrec, defrec, dpbench, transforms;

    [sizes, isa, optrec] := _parse(arg, 1);

    defrec := rec();

    opts := SIMDGlobals.getOpts(isa, CopyFields(optrec, defrec));

	transforms := Flat(List(sizes, n -> let(tt := TRConv2D(ImageVar([n, n])).withTags(opts.tags), [tt, tt.forwardTransform()])));

	dpbench := spiral.libgen.DPBench.build(transforms, opts);

	dpbench.sizes := sizes;

    return dpbench;
end;


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

isXchain := (o,dot) ->
    Same(ObjId(o), fTensor) and Same(ObjId(o.child(1)), fId) and
    ForAll(Drop(o.children(),1), c->Same(ObjId(c), fXbase) and Same(dot, c.params[4]));

mergeXchains := function(v, c1, c2)
    local terms, i, t, other;
    terms := [ Copy(c2.child(1)) ];
    for t in Drop(c2.children(),1) do
        Add(terms,
	    fXbase(t.params[1], t.params[2], t.params[3], v));
    od;

    other := c2.child(c2.numChildren());
    for i in [2 .. c1.numChildren()] do
       t := c1.child(i);
       Add(terms,
	   fXbase(t.params[1], t.params[2], t.params[3] + other.params[2] + other.params[3], v));
    od;
    return fTensor(terms);
end;

# ===================================================================
# fXbase and fDot
# ===================================================================
RewriteRules(RulesFuncSimp, rec(
  # fCompose  [[IP, @(1), J], [fTensor, fId, fBase]],
  IP_toXbase := ARule(fCompose,
     [ [ IP, @(1), @(2,J) ], [ fTensor, @(3,fId, e -> e.params[1] = @(1).val / @(2).val.params[1]),
	                                @(4,fBase, e -> e.params[1] = @(2).val.params[1]) ] ],
     e -> let(v:=Ind(),
	 [ fDot(v, fTensor(@(3).val, fXbase(@(2).val.params[1], @(4).val.params[2], 0, v))) ])
  ),

  IP_toYbase := ARule(fCompose,
     [ [ IP, @(1), [@(2,OS), @, -1] ], [ fTensor, @(3,fId, e -> e.params[1] = @(1).val / @(2).val.params[1]),
	                                          @(4,fBase, e -> e.params[1] = @(2).val.params[1]) ] ],
     e -> let(v:=Ind(),
	 [ fDot(v, fTensor(@(3).val, fYbase(@(2).val.params[1], @(4).val.params[2], 0, v))) ])
  ),

 # fCompose, [ fDot, [fTensor, fId, fXbase] ]
 #	     [ fDot, [fTensor, @,   fXbase] ]
 MergeXchains1 := ARule(fCompose,
       [ [ fDot, @(1),
	     [ fTensor, fId, [@(10,fXbase), @, @, zero, @.cond(e->Same(e, @(1).val)) ]]],
	 [ @(30,fDot), @(3),
	     [ fTensor, ..., [@(20,fXbase), @, @,    @, @.cond(e->Same(e, @(3).val)) ]]] ],

 e -> let(v:=IndNR(), f:=@(10).val, g:=@(20).val,
	  dots:=DropLast(@(30).val.params[2].children(), 1),
     [ fDot(v, fTensor(dots,
	               fXbase(g.params[1], g.params[2], g.params[3], v),
		       fXbase(f.params[1], f.params[2], g.params[2] + g.params[3], v))) ])
 ),

 # fDot X fDot -> fDot
 MergeXchains2 := ARule(fCompose,
    [ [ fDot, @(1), @(2).cond(e->isXchain(e,@(1).val)) ],
      [ fDot, @(3), @(4).cond(e->isXchain(e,@(3).val)) ] ],
 e -> let(v:=Ind(),
	 [ fDot(v, fTensor( mergeXchains(v, @(2).val, @(4).val))) ])
 )
));

Class(RulesDot, RuleSet);
RewriteRules(RulesDot, rec(
  IP_toYbase := ARule(fCompose,
     [ [ IP, @(1), [@(2,OS), @, -1] ], [ fTensor, @(3,fId, e -> e.params[1] = @(1).val / @(2).val.params[1]),
	                                          @(4,fBase, e -> e.params[1] = @(2).val.params[1]) ] ],
     e -> let(v:=Ind(),
	 [ fDot(v, fTensor(@(3).val, fYbase(@(2).val.params[1], @(4).val.params[2], 0, v))) ])
  )
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

NewRulesFor(TTensorInd, rec(
#   base cases
#   I x A
    dsA_base_vec_push := rec(
        info := "IxA base",
        forTransposition := false,
        applicable := nt -> IsParPar(nt.params) and nt.isTag(1, AVecReg),
        children := nt -> let(_krnl := nt.params[1].withTags(nt.getTags()), krnl := When(_krnl.isReal(), _krnl.setWrap(VWrapId), _krnl.setWrap(VWrapTRC(nt.firstTag().isa))),
            [[ krnl, InfoNt(nt.params[2]) ]]),
        apply := (nt, c, cnt) -> IDirSum(cnt[2].params[1], c[1])
    ),
#   A x Iv
    L_dsA_L_base_vec := rec(
        info := "AxI base",
        forTransposition := false,
        applicable := nt -> IsVecVec(nt.params) and nt.isTag(1, AVecReg) and nt.params[2].range = nt.getTags()[1].v,
        children := nt -> let(jv := Ind(nt.getTags()[1].v),
            [[ SubstVars(Copy(nt.params[1]), rec((nt.params[2].id) := jv)).setWrap(VWrap(nt.firstTag().isa)).withTags(Drop(nt.getTags(), 1)),
            InfoNt(jv) ]]),
        apply := (nt, c, cnt) -> let(v := nt.getTags()[1].v, jv := cnt[2].params[1],
            VTensorInd(c[1], jv))
    ),
#   A x In
    L_dsA_L_vec := rec(
        info := "AxI base",
        forTransposition := false,
        applicable := nt -> IsVecVec(nt.params) and nt.isTag(1, AVecReg) and nt.params[2].range > nt.getTags()[1].v,
        children := nt -> let(v := nt.getTags()[1].v, jv := Ind(nt.getTags()[1].v), j := Ind(nt.params[2].range/nt.getTags()[1].v),
            [[ TTensorInd(
                SubstVars(Copy(nt.params[1]), rec((nt.params[2].id) := j*V(v)+jv)),
                j, AVec, AVec).setWrap(VWrap(nt.firstTag().isa)).withTags(Drop(nt.getTags(), 1)), InfoNt(jv) ]]),
        apply := (nt, c, cnt) -> let(jv := cnt[2].params[1],
            VTensorInd(c[1], jv))
    )
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

# -------------------------------------------------------------------------------
# NOTE: Make at()/lambda() methods consistent
#
#  Either implement at via lambda consistently or the other way around
#  Currently some classes implement at() via lambda(), and some lambda() via at()
#  This is error prone!
# -------------------------------------------------------------------------------

#################################################################################
#F VData( <diagfunc>, <vlen> ) - new diagonal gen. function returning vectors
#F
#F Given a diagonal generating function F with domain N, and range (=element
#F type) T, VData(F, v) is a new "vector" diagonal function with domain N/v,
#F and range (=element type) T.vectorType(v).
#F
#F It uses vpack(_, _, ...) pseudo-instruction to pack scalars into vectors.
#F This instrucion has to be converted into architecture specific sequence
#F by the backend.
#F
Class(VData, Function, rec(
   __call__ := (self, func, v) >> WithBases(self, rec(
       func := Checked(IsFunction(func) or IsFuncExp(func), func),
       v := v,
       operations := PrintOps)),

   print := self >> Print(self.name, "(", self.func, ", ", self.v, ")"),
   rChildren := self >> [ self.func, self.v ],
   rSetChild := rSetChildFields("func", "v"),

   at := (self, i) >> let(
       n := self.func.domain(), v := self.v, f := self.func.lambda(),
       ApplyFunc(vpack, List([0..v-1], j->f.at(v*i + j)))),

   lambda := self >> let(
       n := self.func.domain(), v := self.v, f := self.func.lambda(),
       i := Ind(n/v),
       Lambda(i, ApplyFunc(vpack, List([0..v-1], j->f.at(v*i + j))))),

   domain := self >> div(self.func.domain(), self.v),
   range := self >> TVect(self.func.range(), self.v),
   free := self >> self.func.free()
));

#################################################################################
#F VDup( <diagfunc>, <vlen> ) - new diagonal gen. function returning vectors
#F
#F
#F Given a diagonal generating function F with domain N, and range (=element
#F type) T, VData(F, v) is a new "vector" diagonal function with domain N,
#F and range (=element type) T.vectorType(v). Every vector element is the
#F replicated original scalar.
#F
#F It uses vdup(<elt>, <vlen>) pseudo-instruction to duplicate scalars
#F to form vectors.  This instrucion has to be converted into
#F architecture specific sequence by the backend.
#F
Class(VDup, Function, rec(
   __call__ := (self, func, v) >> WithBases(self, rec(
       func := Checked(IsFunction(func) or IsFuncExp(func), func),
       v := v,
       operations := PrintOps)),

   print := self >> Print(self.name, "(", self.func, ", ", self.v, ")"),
   rChildren := self >> [ self.func, self.v ],
   rSetChild := rSetChildFields("func", "v"),

   at := (self, i) >> vdup(self.func.at(i), self.v),

   lambda := self >> let(
       i := Ind(self.func.domain()),
       Lambda(i, vdup(self.func.at(i), self.v))),

   domain := self >> self.func.domain(),
   range := self >> TVect(self.func.range(), self.v),
   free := self >> self.func.free()
));

#################################################################################
#F VDupOnline( <diagfunc>, <vlen> ) - is an online version of VDup.
#F
#F VDupOnline semantically the same as VDup(), but unlike VDup() it is pulled out
#F of fPrecompute and executed at compute time.
#F

Class(VDupOnline, VDup, rec(
    range := (self) >> let( t := self.func.range(),
        Cond( IsVecT(t), TVect(t.t, t.size*self.v), TVect(t, self.v)))
));

#################################################################################
#F RCVData( <cplx_vdiagfunc> ) - func returning real vecs given complex vec func
#F
#F Given a diagonal generating function F with domain N, and range of
#F of complex vectors TVect(CT, v), RCVData(F) is a new real vector diagonal
#F function with domain N and range TVect(CT.realType(), 2*v)
#F
Class(RCVData, Function, rec(
   __call__ := (self, func) >> WithBases(self, rec(
       func := Checked((IsFunction(func) or IsFuncExp(func)) and IsVecT(func.range()), func),
       operations := PrintOps)),

   print := self >> Print(self.name, "(", self.func, ")"),
   rChildren := self >> [ self.func ],
   rSetChild := rSetChildFields("func"),

   at := (self, i) >> let(
       t := self.func.range(),
       f := self.func.lambda(),
       Cond(IsVecT(t),
	    ApplyFunc(vpack, ConcatList([0..t.size-1], j -> [ re(velem(f.at(i), j)), im(velem(f.at(i), j)) ])),
	    ApplyFunc(vpack, [ re(f.at(i)), im(f.at(i)) ]))
   ),

   lambda := self >> let(j := Ind(self.domain()), Lambda(j, self.at(j))),
   domain := self >> self.func.domain(),
   range := self >> let(t := self.func.range(),
       Cond(IsVecT(t), TVect(t.t.realType(), 2*t.size), TVect(t.realType(), 2))),
   free := self >> self.func.free()
));

#################################################################################
#F RCVDataSplit(<cplx_func>, <v>)
#F
#F Given a complex function f RCVDataSplit(f, v)
#F is a new function that returns v-way vectors of the form
#F [ re0, re0, re1, re1, ... ], [-im0, im0, -im1, im1, ...]
#F
Class(RCVDataSplit, Function, rec(
   __call__ := (self, cplx_func, v) >> WithBases(self, rec(
       func := Checked(IsFunction(cplx_func) or IsFuncExp(cplx_func), cplx_func),
       v := v,
       operations := PrintOps)),

   print := self >> Print(self.name, "(", self.func, ", ", self.v, ")"),
   rChildren := self >> [ self.func, self.v ],
   rSetChild := rSetChildFields("func", "v"),

   lambda := self >> let(
       i := Ind(self.domain()),
       n := self.func.domain(),
       v := self.v,
       f := self.func.lambda(),
       vars := DropLast(f.mkVars(), 1),
       ip := idiv(i, 2),
       Lambda(vars :: [i],
       cond(imod(i, 2),
            ApplyFunc(vpack, ConcatList([0..v/2-1], j ->
	        [ -im(f.at(vars :: [(v/2)*ip+j])), im(f.at(vars :: [(v/2)*ip+j])) ])),
            ApplyFunc(vpack, ConcatList([0..v/2-1], j ->
		[  re(f.at(vars :: [(v/2)*ip+j])), re(f.at(vars :: [(v/2)*ip+j])) ]))))
   ),

   at := (self, i) >> self.lambda().at(i),

   # each cpx element -> 2 duplicated reals (r r -i i) = 4 reals -> packed into vectors of length <v>
   domain := self >> let(d := self.func.domain(),
       Cond(IsType(d), d, 4 * self.func.domain() / self.v)),

   range := self >> let(t := self.func.range(), TVect(t.realType(), self.v)),
   free := self >> self.func.free()
));

#################################################################################
#F RCVDataSplitVec( <cplx_vdiagfunc> )
#F
#F Given a v/2-way complex vector function f RCVDataSplitVec(f, v)
#F is a new function that returns v-way real vectors of the form
#F [ re0, re0, re1, re1, ... ], [-im0, im0, -im1, im1, ...]
#F
Class(RCVDataSplitVec, RCVData, rec(
   # lambda := <inherited from RCVData, implemented via at>
   #
   at := (self, i) >> let(
       v := self.func.range().size,
       f := self.func.lambda(),
       ip := idiv(i, 2),
       cond(imod(i, 2),
            ApplyFunc(vpack, ConcatList([0..v-1], j ->
		        [ -im(velem(f.at(ip), j)), im(velem(f.at(ip), j)) ])),
            ApplyFunc(vpack, ConcatList([0..v-1], j ->
		        [  re(velem(f.at(ip), j)), re(velem(f.at(ip), j)) ])))
   ),

   # each cpx element -> 2 duplicated reals (r r -i i) = 4 reals
   #  -> packed into vectors of length <v>
   domain := self >> let(d:=self.func.domain(),
       Cond(IsType(d), d, 2 * self.func.domain())),
));

#################################################################################
#F VDataRDup(<real-vector-func>)
#F
#F Returns new function of twice the vector length, with every element
#F duplicated.
#F
#F
Class(VDataRDup, RCVData, rec(
   # lambda := <inherited from RCVData, implemented via at>
   #
   at := (self, i) >> let(
       v := self.func.range().size,
       f := self.func.lambda(),
       ApplyFunc(vpack, ConcatList([0..v-1],
	           j -> [ velem(f.at(i), j), velem(f.at(i), j) ]))
   ),

   # domain doesn't change, but output vectors get longer (ie range is different)
   domain := self >> self.func.domain(),
));

#################################################################################
#F fStretch( <diagfunc>, num, den ) - new diagonal gen. function stretching the old one
#F
Class(fStretch, Function, rec(
   __call__ := (self, func, num, den) >> WithBases(self, rec(
       func := Checked(IsFunction(func) or IsFuncExp(func), func),
       den := den,
       num := num,
       operations := PrintOps)),

   print := self >> Print(self.__name__, "(", self.func, ", ", self.num, ", ", self.den, ")"),
   rChildren := self >> [ self.func, self.num, self.den ],
   rSetChild := rSetChildFields("func", "num", "den"),

   at := (self, i) >> self.lambda().at(i).eval(),

   lambda := self >> let(
       n := self.func.domain(),
       f := self.func.lambda(),
       i := Ind(n * self.num / self.den),
       i_old := self.den * idiv(i, self.num) + imod(i, self.num),
       Lambda(i, cond(leq(imod(i, self.num), self.den - 1), f.at(i_old), 0))
   ),

   domain := self >> self.func.domain() * self.num/self.den,
   range := self >> self.func.range(),
   free := self >> self.func.free()
));

#F ==========================================================================
#F VDiag(<func>, <v>)
#F
#F Vectorized diagonal. <func> must be a plain scalar function.
#F
#F Diag(f) == VDiag(f, v)
#F
Class(VDiag, Diag, rec(
    abbrevs := [
	(f, v) -> Checked(IsInt(v),
	    Cond(IsList(f), let(ff := List(f, toExpArg),
		    [ FList(UnifyTypes(List(f, x -> x.t)), f), v]),
		 IsFunction(f),
		    [ f, v ]))
    ],

    new := (self, f, v) >> SPL(WithBases(self, rec(element := f, v := v))).setDims(),

    print := (self, indent, indentStep) >> Print(self.name, "(", self.element, ", ", self.v, ")"),
    needInterleavedLeft := False,
    needInterleavedRight := False,
    transpose := self >> self,
    conjTranspose := self >> ObjId(self)(FConj(self.element), self.v),
    # self.v is not exposed in rChildren
    from_rChildren := (self, rch) >> ObjId(self)(rch[1], self.v),
));

#################################################################################
#F VDiag_x_I(<func>, <v>)
#F
#F VTensor(Diag(f), v) == VDiag_x_I(f, v)
#F
Class(VDiag_x_I, VDiag, rec(
    toAMat := self >> Tensor(Diag(self.element), I(self.v)).toAMat(),
    dims := self >> self.v * Diag(self.element).dims(),
    needInterleavedLeft := False,
    needInterleavedRight := False,
    transpose := self >> self
));

#################################################################################
#F VRCDiag(<func>, <v>)
#F
#F Semantically equivalent to (I tensor L) * RCDiag * (I tensor L), and vectorized.
#F
#F Expects <func> and Input/output data to be in vector-interleaved complex format.
#F I.e. that packing is r0,r1,r2,...,r_(v-1); i0,i1,..,i_(v-1); ... into vectors.
#F
#F Example, given a complex function <f> (range = TComplex), and vector length <v>:
#F (different SIMD slots perform subsequent complex mults)
#F
#F    VRCDiag(VData(fCompose(RCData(f.domain()), fTensor(fId(d.domain()/v), L(2*v, 2))), v), v)
#F
#F Example, given a complex function <f> (range = TComplex), and vector length <v>:
#F (different SIMD slots perform SAME complex mults)
#F
#F    VRCDiag(VDup(RCData(f), v), v)
#F
Declare(VRCDiag);
Class(VRCDiag, VDiag, rec(
    print := (self, indent, indentStep) >> Print(self.name, "(", self.element, ", ", self.v, ")"),

    toAMat := meth(self)
        local elts, k, res, r, i;
        elts := self.element.tolist();
        Constraint(Length(elts) mod 2 = 0);
        res := [];
        for k in [ 1 .. Length(elts)/2 ] do
            r := EvalScalar(elts[2 * k - 1]);
            i := EvalScalar(elts[2 * k]);
            Add(res, BlockMat([[Diag(r), Diag(-i)], [Diag(i), Diag(r)]]).toAMat());
        od;
        return DirectSumAMat(res);
    end,

    toloop := self >> let(
        func := self.element, # r,i,r,i,...
        j    := Ind(idiv(func.domain(),2)),
        re   := func.at(2*j),
        im   := func.at(2*j+1),
        ISum(j, j.range,
                VScat(H(2*j.range, 2, 2*j, 1), self.v) *
                VBlk([[re, -im],
                      [im,  re]], self.v) *
                VGath(H(2*j.range, 2, 2*j, 1), self.v))),

    dims := self >> let(n:=self.element.domain() * self.v, [n,n]),  # data stores r,i pairs
    needInterleavedLeft := False,
    needInterleavedRight := False,
    transpose := self >> VRCDiag(FRConj(self.element), self.v)
));

#################################################################################
#F RCVDiag(<func>, <v>)
#F
#F Semantically equivalent to RCDiag, but vectorized.
#F Expects <func> to pack r0,i0,r1,i1,.. into vectors. Input/Output data format is same.
#F
#F Example, given a complex function <f> (range = TComplex), and vector length <v>:
#F (different SIMD slots perform subsequent complex mults)
#F
#F    RCDiag(RCData(f)) == RCVDiag(VData(RCData(f), v), v)
#F
#F Example, given a complex function <f> (range = TComplex), and vector length <v>:
#F (different SIMD slots perform SAME complex mults)
#F
#F    RCDiag(RCData(f)) == RCVDiag(RCVData(VDup(f, v/2)), v)
#F
#F Example, given a complex vector function <f> (range = TVect(TComplex, v/2))
#F
#F    RCVDiag(RCVData(f), v),
#F
Class(RCVDiag, VDiag, rec(
    print := (self, indent, indentStep) >> Print(self.name, "(", self.element, ", ", self.v, ")"),

    toAMat := self >> RCDiagonalAMat(
	ConcatList(self.element.tolist(), x -> Cond(IsValue(x), x.v, List([1..self.v], i->velem(x,i)))),
	IdentityPermAMat(2)),

    dims := self >> let(n:=self.element.domain() * self.v, [n,n]),  # data stores r,i pairs
    needInterleavedLeft := True,
    needInterleavedRight := True,

    transpose := self >> self  # this is invalid! need to conjugate complex function
));

#################################################################################
#F RCVDiagSplit(<func>, <v>)
#F
#F Semantically equivalent to RCDiag, but vectorized, and expects different <func>
#F data format than RCVDiag.
#F
#F Expects <func> to pack r0,r0,r1,r1,...; -i0,i0,-i1,i1,...; ... into vectors.
#F Expects Input/Output data to pack r0,i0,r1,i1,.. into vectors.
#F
#F Example, given a complex function <f> (range = TComplex), and vector length <v/2>:
#F (different SIMD slots perform SAME complex mults)
#F
#F  RCVDiagSplit(RCVDataSplitVec(VDup(f, v/2)), v);
#F
#F Example, given a complex function <f> (range = TComplex), and vector length <v/2>:
#F (different SIMD slots perform different complex mults)
#F
#F  RCVDiagSplit(RCVDataSplit(f, v), v);
#F
Class(RCVDiagSplit, VDiag, rec(
    print := (self, indent, indentStep) >> Print(self.name, "(", self.element, ", ", self.v, ")"),

    toAMat := meth(self)
        local elts, k, res, r, i, p;
        elts := self.element.tolist();
        Constraint(Length(elts) mod 2 = 0);
        res := [];
        p := Tensor(I(self.v/2), J(2));
        for k in [ 1 .. Length(elts)/2 ] do
            r := EvalScalar(elts[2 * k - 1]);
            i := EvalScalar(elts[2 * k]);
            Add(res, SUM(Diag(r), Diag(i)*p).toAMat());
        od;
        return DirectSumAMat(res);
    end,

    dims := self >> let(n:=self.element.domain() * (self.v/2), [n,n]),  # data stores r,i pairs
    needInterleavedLeft := True,
    needInterleavedRight := True,
    transpose := self >> self   # this is invalid! need to conjugate complex function
));

#################################################################################
#F VScale(<spl>, <constant>, <v>)
#F
#F Vecorized version of Scale
#F
Declare(VScale);
Class(VScale, BaseMat, SumsBase, rec(
    new := (self, spl, s, v) >> SPL(WithBases(self,
        rec(dimensions := spl.dims(), _children:=[spl], s:=s, v := v))),
    #-----------------------------------------------------------------------
    dims := self >> self.dimensions,
    child := (self, i) >> self._children[i],
    rChildren := self >> [self._children[1], self.s, self.v],
    rSetChild := meth(self, n, what)
        if n=1 then self._children[1] := what;
        elif n=2 then self.s := what;
        elif n=3 then self.v := what;
        else Error("<n> must be in [1..3]");
        fi;
    end,
    #-----------------------------------------------------------------------
    print := (self, i, is) >> Print(self.name, "(", self.child(1).print(i+is,is), ", ", self.s, ", ", self.v, ")"),
    unroll := self >> self,
    transpose := self >> VScale(self.child(1).transpose(), self.s, self.v),
    isReal := self >> self.child(1).isReal(),
    #-----------------------------------------------------------------------
    toAMat := self >> AMatMat(MatSPL(Scale(self.s, self.child(1))))
));

#################################################################################
# Below are the version of above objects hacked up for online twiddle computation
# The difference is in the codegen, as it uses a different style of underlying
# function, that is recursive.
#
# NOTE: Find a cleaner and more general solution
# NOTE: document this
# NOTE: .toAMat() does not work for the below object, because self.func is different
#
Class(RCVOnlineDiag, RCVDiag);
Class(VOnlineDiag, VDiag);
Class(VOnlineDiag_x_I, VDiag_x_I);


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

_AVXINTRIN := () ->  ["<immintrin.h>"];
_MM_MALLOC := () -> When(not LocalConfig.osinfo.isDarwin(), ["<include/mm_malloc.h>"],[]);

Class(AVX_Intel, SIMD_ISA, rec(
    unparser := AVXUnparser,
    arch := "Intel_AVX",
    info := "Intel AVX architecture definition base",
    declareConstants := false,

    file := "avx",
    #arrayDataModifier := "static __declspec(align(32))",
    #arrayBufModifier  := "static __declspec(align(32))",
	alignmentBytes := 32,

    autolib := rec(
        includes := () -> _NMMINTRIN() :: _SMMINTRIN() :: _TMMINTRIN() :: _PMMINTRIN()
                       :: _EMMINTRIN() :: _XMMINTRIN() :: _AVXINTRIN(),
        timerIncludes := () -> ["<include/sp_rdtsc.h>"],
        cFlags := "-mavx",
    ),

    simpIndicesInside := AVX_LDST.getList(),
));

#==============================================================================================
# 4-way double precision real

Class(AVX_4x64f, AVX_Intel, rec(
    info     := "AVX 4 x 64-bit double",
    countrec := rec(
        ops := [
            [ add, sub, addsub_4x64f, chslo_2x64f, chshi_2x64f, addsub_2x64f, hadd_2x64f ],
	    [ mul ],
	    [ fmaddsub_4x64f],
            [ vunpacklo_4x64f,  vunpackhi_4x64f,  vshuffle_4x64f,  vperm2_4x64f,    vpermf128_4x64f,
              vuunpacklo_4x64f, vuunpackhi_4x64f, vushuffle_4x64f, vuperm2_4x64f,   vupermf128_4x64f,
              vunpacklo_2x64f,  vunpackhi_2x64f,  vshuffle_2x64f,  vushuffle_2x64f, vperm_4x64f,
              vinsert_2l_4x64f, vextract_2l_4x64f, vblend_4x64f],
            [ vloadmask_4x64f,  vstoremask_4x64f, vload1sd_2x64f,  vload_1l_2x64f,  vload_1h_2x64f,
	      vloadu_2x64f,     vstore_1l_2x64f,  vstore_1h_2x64f, vstoreu_2x64f ],
            [ deref, vloadu_4x64f, vstoreu_4x64f ],
            Value      # Value without [] is a keyword in countOps !!
        ],
        printstrings := ["[adds]", "[mults]", "[fmas]", "[vperms]", "[svldst]", "[vldst]", "[vval]"],
        arithcost := (self, opcount) >> opcount[1]+opcount[2]+2*opcount[3]
    ),

    includes     := () -> ["<include/omega64.h>"] :: _AVXINTRIN(),
    active       := true,
    isFixedPoint := false,
    isFloat      := true,

    v       := 4,
    t       := TVect(T_Real(64), 4),
    ctype   := "double",
    instr   := [ vunpacklo_4x64f, vunpackhi_4x64f, vshuffle_4x64f, vperm2_4x64f, vpermf128_4x64f, vperm_4x64f, vblend_4x64f ],
               #vuunpacklo_4x64f, vuunpackhi_4x64f, vushuffle_4x64f, vuperm2_4x64f, vupermf128_4x64f
    bits    := 64,
    splopts := rec(precision := "double"),

    freshU  := self >> var.fresh_t("U", self.t),

    dupload := (y, x) -> assign(y, vdup(x, 4)),
    mul_cx := (self, opts) >>
        ((y, x, c) -> let( u1 := self.freshU(), u2 := self.freshU(), u3 := self.freshU(),
            decl([u1, u2, u3], chain(
                assign(u1, mul(x, vunpacklo_4x64f(c, c))),
                assign(u2, vshuffle_4x64f(x, x, [2,1,2,1])),
                assign(u3, mul(u2, vunpackhi_4x64f(c, c))),
                assign(y, addsub_4x64f(u1, u3))))
        )),

    mul_cx_conj := (self, opts) >>
        ((y,x,c) -> let(
		u1 := self.freshU(), u2 := self.freshU(), v := self.freshU(),
                decl([u1, u2, v], chain(
                    assign(u1,             mul(x, vunpackhi_4x64f(c, c))),
                    assign(u2, mul(vshuffle_4x64f(x, x, [2,1,2,1]), vunpacklo_4x64f(c, c))),
                    assign(v, addsub_4x64f(u2, u1)),
                    assign(y, vshuffle_4x64f(v, v, [2,1,2,1]))
                )))),

    svload_init := (vt) -> [
        # load using subvectors of length 1
	#
	[
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1], chain(
                assign(u1, vload1sd_2x64f(x[1].toPtr(vt.t))),
                assign(y, vinsert_2l_4x64f(vt.zero(), u1, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)), u2 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1, u2], chain(
                assign(u1, vload1sd_2x64f(x[1].toPtr(vt.t))),
                assign(u2, vload_1h_2x64f(u1, x[2].toPtr(vt.t))),
                assign(y, vinsert_2l_4x64f(vt.zero(), u2, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)), u2 := var.fresh_t("U", TVect(vt.t, 2)),
                        u3 := var.fresh_t("U", TVect(vt.t, 2)), v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, v1], chain(
                assign(u1, vload1sd_2x64f(x[1].toPtr(vt.t))),
                assign(u2, vload_1h_2x64f(u1, x[2].toPtr(vt.t))),
                assign(u3, vload1sd_2x64f(x[3].toPtr(vt.t))),
                assign(v1, vinsert_2l_4x64f(vt.zero(), u2, [0])),
                assign(y, vinsert_2l_4x64f(v1, u3, [1]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)), u2 := var.fresh_t("U", TVect(vt.t, 2)),
                        u3 := var.fresh_t("U", TVect(vt.t, 2)), u4 := var.fresh_t("U", TVect(vt.t, 2)),
                        v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, v1], chain(
                assign(u1, vload1sd_2x64f(x[1].toPtr(vt.t))),
                assign(u2, vload_1h_2x64f(u1, x[2].toPtr(vt.t))),
                assign(u3, vload1sd_2x64f(x[3].toPtr(vt.t))),
                assign(u4, vload_1h_2x64f(u3, x[4].toPtr(vt.t))),
                assign(v1, vinsert_2l_4x64f(vt.zero(), u2, [0])),
                assign(y, vinsert_2l_4x64f(v1, u4, [1]))
            )))
        ],
        # load using subvectors of length 2
	#
        [
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1], chain(
                assign(u1, vloadu_2x64f(x[1].toPtr(vt.t))),
                assign(y, vinsert_2l_4x64f(vt.zero(), u1, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)), u2 := var.fresh_t("U", TVect(vt.t, 2)),
                        v1 := var.fresh_t("U", vt),
            decl([u1, u2, v1], chain(
                assign(u1, vloadu_2x64f(x[1].toPtr(vt.t))),
                assign(u2, vloadu_2x64f(x[2].toPtr(vt.t))),
                assign(v1, vinsert_2l_4x64f(vt.zero(), u1, [0])),
                assign(y, vinsert_2l_4x64f(v1, u2, [1]))
            )))
        ]
    ],
    svstore_init := (vt) -> [
        # store using subvectors of length 1
        [
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1], chain(
                assign(u1, vextract_2l_4x64f(x, [0])),
                vstore_1l_2x64f(y[1].toPtr(vt.t), u1)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1], chain(
                assign(u1, vextract_2l_4x64f(x, [0])),
                vstore_1l_2x64f(y[1].toPtr(vt.t), u1),
                vstore_1h_2x64f(y[2].toPtr(vt.t), u1)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)), u2 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1, u2], chain(
                assign(u1, vextract_2l_4x64f(x, [0])),
                vstore_1l_2x64f(y[1].toPtr(vt.t), u1),
                vstore_1h_2x64f(y[2].toPtr(vt.t), u1),
                assign(u2, vextract_2l_4x64f(x, [1])),
                vstore_1l_2x64f(y[3].toPtr(vt.t), u2)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)), u2 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1, u2], chain(
                assign(u1, vextract_2l_4x64f(x, [0])),
                vstore_1l_2x64f(y[1].toPtr(vt.t), u1),
                vstore_1h_2x64f(y[2].toPtr(vt.t), u1),
                assign(u2, vextract_2l_4x64f(x, [1])),
                vstore_1l_2x64f(y[3].toPtr(vt.t), u2),
                vstore_1h_2x64f(y[4].toPtr(vt.t), u2)
            )))
        ],
        # store using subvectors of length 2
        [
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1], chain(
                assign(u1, vextract_2l_4x64f(x, [0])),
                vstoreu_2x64f(y[1].toPtr(vt.t), u1)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 2)), u2 := var.fresh_t("U", TVect(vt.t, 2)),
            decl([u1, u2], chain(
                assign(u1, vextract_2l_4x64f(x, [0])),
                vstoreu_2x64f(y[1].toPtr(vt.t), u1),
                assign(u2, vextract_2l_4x64f(x, [1])),
                vstoreu_2x64f(y[2].toPtr(vt.t), u2)
            )))
        ]],

    svload  := ~.svload_init(~.t),
    svstore := ~.svstore_init(~.t),

    loadc := (self, sv, opts) >> Cond(
        sv = 4,
            ((y,x) -> assign(y, vloadu_4x64f(x.toPtr(self.t.t)))),
        sv = 2,
            ((y,x) -> assign(y, vinsert_2l_4x64f(self.t.zero(), vloadu_2x64f(x.toPtr(self.t.t)), [0]))),
        # else
	Checked( sv=1,
	    ((y,x) -> assign(y, vinsert_2l_4x64f(self.t.zero(), vload1sd_2x64f(x.toPtr(self.t.t)), [0]))))),

    loadc_align := (self, sv, align, opts) >> Cond(
        align = 0 and sv = 8,
            ((y,x,addr) -> assign(y, deref(nth(x,addr).toPtr(self.t)))),
        align mod 2 = 0 and sv = 2,
            ((y,x,addr) -> assign(y, vinsert_2l_4x64f(self.t.zero(), deref(nth(x,addr).toPtr(TVect(self.t.t, 2))), [0]))),
        sv = 4,
            ((y,x,addr) -> assign(y, vloadu_4x64f(nth(x, addr).toPtr(self.t.t)))),
        sv = 2,
            ((y,x,addr) -> assign(y, vinsert_2l_4x64f(self.t.zero(), vloadu_2x64f(nth(x, addr).toPtr(self.t.t)), [0]))),
        # else
        Checked( sv=1,
	    ((y,x,addr) -> assign(y, vinsert_2l_4x64f(self.t.zero(), vload1sd_2x64f(nth(x, addr).toPtr(self.t.t)), [0]))))),

    storec_init := (vt) -> [
        ((y,x) -> vstore_1l_2x64f(y.toPtr(vt.t), vextract_2l_4x64f(x, [0]))),
        ((y,x) -> vstoreu_2x64f(y.toPtr(vt.t), vextract_2l_4x64f(x, [0]))),
        ((y,x) -> vstoremask_4x64f(y.toPtr(vt.t), x, vstoremask_4x64f.mask(3))),
        ((y,x) -> vstoreu_4x64f(y.toPtr(vt.t), x))
    ],

    storec := ~.storec_init(~.t),

    swap_cx := (y, x, opts) -> assign(y, vushuffle_4x64f(x, [2,1,2,1])),
    RCVIxJ2 := (y, x, opts) -> assign(y, vupermf128_4x64f(x, [2,1])),

    bin_shrev := (self, y, x, opts) >> let(
        u1 := self.freshU(), u2 := self.freshU(),
        decl( [u1, u2], chain(
            assign(u1, vupermf128_4x64f(x[2], [2,1])),
            assign(u2, vblend_4x64f(x[1], x[2], [1,1,2,2])),
            assign(y,  vblend_4x64f(u1, u2, [2,1,2,1]))))),
));

#==============================================================================================
Class(AVX_8x32f, AVX_Intel, rec(
    info     := "AVX 8 x 32-bit float",
    countrec := rec(
        ops := [
            [ add, sub, addsub_8x32f, addsub_4x32f, hadd_4x32f ],
	    [ mul ],
	    [ fmaddsub_8x32f ],
            [ vunpacklo_8x32f,  vunpackhi_8x32f,   vpermf128_8x32f,  vperm2_8x32f,  vshuffle_8x32f,
              vinsert_4l_8x32f, vextract_4l_8x32f, vperm_8x32f,      vpermv_8x32f,
              vunpacklo_4x32f,  vunpackhi_4x32f,   vshuffle_4x32f,   vushuffle_4x32f,
              vuunpacklo_8x32f, vuunpackhi_8x32f,  vupermf128_8x32f, vushuffle_8x32f,
              vldup_8x32f, vhdup_8x32f, vblend_8x32f ],
            [ vloadmask_8x32f, vstoremask_8x32f,
              vload1_4x32f,    vload_2l_4x32f,  vload_2h_4x32f,  vloadu_4x32f,  vloadu2_4x32f,
              vstore1_4x32f,   vstore_2l_4x32f, vstore_2h_4x32f, vstoreu_4x32f, vstoreu2_4x32f,
              vstoremsk_4x32f ],
            [ deref, vloadu_8x32f, vstoreu_8x32f ],
            Value   # Value without [] is a keyword in countOps !!
        ],
        printstrings := ["[adds]", "[mults]", "[fmas]", "[vperms]", "[svldst]", "[vldst]", "[vval]"],
        arithcost := (self, opcount) >> opcount[1]+opcount[2]+2*opcount[3]
    ),

    includes     := () -> ["<include/omega32.h>"] :: _AVXINTRIN(),
    active       := true,
    isFixedPoint := false,
    isFloat      := true,

    v       := 8,
    t       := TVect(T_Real(32), 8),
    ctype   := "float",
    instr   := [ vunpacklo_8x32f,  vunpackhi_8x32f,  vpermf128_8x32f,  vshuffle_8x32f,  vperm_8x32f,
                 vuunpacklo_8x32f, vuunpackhi_8x32f, vupermf128_8x32f, vushuffle_8x32f, vblend_8x32f,
                 vcxtr_8x32f],

    bits    := 32,
    splopts := rec(precision := "single"),

    freshU  := self >> var.fresh_t("U", self.t),

    dupload := (y, x) -> assign(y, vdup(x, 8)),

    mul_cx := (self, opts) >>
        ((y, x, c) -> let( u1 := self.freshU(), u2 := self.freshU(), u3 := self.freshU(),
            decl([u1, u2, u3], chain(
                assign(u1, mul(x, vldup_8x32f(c))),
                assign(u2, vshuffle_8x32f(x, x, [2,1,4,3])),
                assign(u3, mul(u2, vhdup_8x32f(c))),
                assign(y,  addsub_8x32f(u1, u3))))
        )),

    mul_cx_conj := (self, opts) >>
        ((y,x,c) -> let( u1 := self.freshU(), u2 := self.freshU(), v := self.freshU(),
            decl([u1, u2, v], chain(
                assign(u1, mul(x, vhdup_8x32f(c))),
                assign(u2, mul(vshuffle_8x32f(x, x, [2,1,4,3]), vldup_8x32f(c))),
                assign(v, addsub_8x32f(u2, u1)),
                assign(y, vshuffle_8x32f(v, v, [2,1,4,3]))))
        )),

    svload_init := (vt) -> [
        # Load using subvectors of length 1.
	#
	[
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(y,  vinsert_4l_8x32f(vt.zero(), u1, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(u2, vinsert_4x32f(u1, deref(x[2].toPtr(T_Int(32))), 2)),
                assign(y,  vinsert_4l_8x32f(vt.zero(), u2, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2, u3], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(u2, vinsert_4x32f(u1, deref(x[2].toPtr(T_Int(32))), 2)),
                assign(u3, vinsert_4x32f(u2, deref(x[3].toPtr(T_Int(32))), 3)),
                assign(y,  vinsert_4l_8x32f(vt.zero(), u3, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)), u4 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2, u3, u4], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(u2, vinsert_4x32f(u1, deref(x[2].toPtr(T_Int(32))), 2)),
                assign(u3, vinsert_4x32f(u2, deref(x[3].toPtr(T_Int(32))), 3)),
                assign(u4, vinsert_4x32f(u3, deref(x[4].toPtr(T_Int(32))), 4)),
                assign(y,  vinsert_4l_8x32f(vt.zero(), u4, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)), u4 := var.fresh_t("U", TVect(vt.t, 4)),
                     u5 := var.fresh_t("U", TVect(vt.t, 4)),
                     v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, u4, u5, v1], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(u2, vinsert_4x32f(u1, deref(x[2].toPtr(T_Int(32))), 2)),
                assign(u3, vinsert_4x32f(u2, deref(x[3].toPtr(T_Int(32))), 3)),
                assign(u4, vinsert_4x32f(u3, deref(x[4].toPtr(T_Int(32))), 4)),
                assign(v1, vinsert_4l_8x32f(vt.zero(), u4, [0])),
                assign(u5, vload1_4x32f(x[5].toPtr(vt.t))),
                assign(y,  vinsert_4l_8x32f(v1, u5, [1]))
            ))),

        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)), u4 := var.fresh_t("U", TVect(vt.t, 4)),
                     u5 := var.fresh_t("U", TVect(vt.t, 4)), u6 := var.fresh_t("U", TVect(vt.t, 4)),
                     v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, u4, u5, u6, v1], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(u2, vinsert_4x32f(u1, deref(x[2].toPtr(T_Int(32))), 2)),
                assign(u3, vinsert_4x32f(u2, deref(x[3].toPtr(T_Int(32))), 3)),
                assign(u4, vinsert_4x32f(u3, deref(x[4].toPtr(T_Int(32))), 4)),
                assign(v1, vinsert_4l_8x32f(vt.zero(), u4, [0])),
                assign(u5, vload1_4x32f(x[5].toPtr(vt.t))),
                assign(u6, vinsert_4x32f(u5, deref(x[6].toPtr(T_Int(32))), 2)),
                assign(y,  vinsert_4l_8x32f(v1, u6, [1]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)), u4 := var.fresh_t("U", TVect(vt.t, 4)),
                     u5 := var.fresh_t("U", TVect(vt.t, 4)), u6 := var.fresh_t("U", TVect(vt.t, 4)),
                     u7 := var.fresh_t("U", TVect(vt.t, 4)),
                     v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, u4, u5, u6, u7, v1], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(u2, vinsert_4x32f(u1, deref(x[2].toPtr(T_Int(32))), 2)),
                assign(u3, vinsert_4x32f(u2, deref(x[3].toPtr(T_Int(32))), 3)),
                assign(u4, vinsert_4x32f(u3, deref(x[4].toPtr(T_Int(32))), 4)),
                assign(v1, vinsert_4l_8x32f(vt.zero(), u4, [0])),
                assign(u5, vload1_4x32f(x[5].toPtr(vt.t))),
                assign(u6, vinsert_4x32f(u5, deref(x[6].toPtr(T_Int(32))), 2)),
                assign(u7, vinsert_4x32f(u6, deref(x[7].toPtr(T_Int(32))), 3)),
                assign(y,  vinsert_4l_8x32f(v1, u7, [1]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)), u4 := var.fresh_t("U", TVect(vt.t, 4)),
                     u5 := var.fresh_t("U", TVect(vt.t, 4)), u6 := var.fresh_t("U", TVect(vt.t, 4)),
                     u7 := var.fresh_t("U", TVect(vt.t, 4)), u8 := var.fresh_t("U", TVect(vt.t, 4)),
                     v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, u4, u5, u6, u7, u8, v1], chain(
                assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                assign(u2, vinsert_4x32f(u1, deref(x[2].toPtr(T_Int(32))), 2)),
                assign(u3, vinsert_4x32f(u2, deref(x[3].toPtr(T_Int(32))), 3)),
                assign(u4, vinsert_4x32f(u3, deref(x[4].toPtr(T_Int(32))), 4)),
                assign(v1, vinsert_4l_8x32f(vt.zero(), u4, [0])),
                assign(u5, vload1_4x32f(x[5].toPtr(vt.t))),
                assign(u6, vinsert_4x32f(u5, deref(x[6].toPtr(T_Int(32))), 2)),
                assign(u7, vinsert_4x32f(u6, deref(x[7].toPtr(T_Int(32))), 3)),
                assign(u8, vinsert_4x32f(u7, deref(x[8].toPtr(T_Int(32))), 4)),
                assign(y,  vinsert_4l_8x32f(v1, u8, [1]))
            )))
        ],
        # Load using subvectors of length 2
	#
        [
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vload_2l_4x32f(vzero_4x32f(), x[1].toPtr(TVect(vt.t, 2)))),
                assign(y,  vinsert_4l_8x32f(vt.zero(), u1, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vload_2l_4x32f(vzero_4x32f(), x[1].toPtr(TVect(vt.t, 2)))),
                assign(u2, vload_2h_4x32f(u1, x[2].toPtr(TVect(vt.t, 2)))),
                assign(y,  vinsert_4l_8x32f(vt.zero(), u2, [0]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)),
                     z1 := var.fresh_t("Z", TVect(vt.t, 4)),
                     v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, v1], chain(
                assign(z1, vzero_4x32f()),
                assign(u1, vload_2l_4x32f(z1, x[1].toPtr(TVect(vt.t, 2)))),
                assign(u2, vload_2h_4x32f(u1, x[2].toPtr(TVect(vt.t, 2)))),
                assign(v1, vinsert_4l_8x32f(vt.zero(), u2, [0])),
                assign(u3, vload_2l_4x32f(z1, x[3].toPtr(TVect(vt.t, 2)))),
                assign(y,  vinsert_4l_8x32f(v1, u3, [1]))
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
                     u3 := var.fresh_t("U", TVect(vt.t, 4)), u4 := var.fresh_t("U", TVect(vt.t, 4)),
                     z1 := var.fresh_t("Z", TVect(vt.t, 4)),
                     v1 := var.fresh_t("U", vt),
            decl([u1, u2, u3, u4, v1], chain(
                assign(z1, vzero_4x32f()),
                assign(u1, vload_2l_4x32f(z1, x[1].toPtr(TVect(vt.t, 2)))),
                assign(u2, vload_2h_4x32f(u1, x[2].toPtr(TVect(vt.t, 2)))),
                assign(v1, vinsert_4l_8x32f(vt.zero(), u2, [0])),
                assign(u3, vload_2l_4x32f(z1, x[3].toPtr(TVect(vt.t, 2)))),
                assign(u4, vload_2h_4x32f(u3, x[4].toPtr(TVect(vt.t, 2)))),
                assign(y,  vinsert_4l_8x32f(v1, u4, [1]))
            ))),
        ],
        # Load using subvectors of length 4.
	#
        [
        (y,x,opts) ->
            assign(y, vinsert_4l_8x32f(vt.zero(), vloadu_4x32f(x[1].toPtr(vt.t)), [0])),

        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vinsert_4l_8x32f(vt.zero(), vloadu_4x32f(x[1].toPtr(vt.t)), [0])),
                assign(y,  vinsert_4l_8x32f(u1,        vloadu_4x32f(x[2].toPtr(vt.t)), [1]))
            )))
        ]
    ],

    svstore_init := (vt) -> [
        # Store using subvectors of length 1.
	#
        [
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1),
                vextract_4x32f(y[2].toPtr(T_Int(32)), u1, 2)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1),
                vextract_4x32f(y[2].toPtr(T_Int(32)), u1, 2),
                vextract_4x32f(y[3].toPtr(T_Int(32)), u1, 3)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1),
                vextract_4x32f(y[2].toPtr(T_Int(32)), u1, 2),
                vextract_4x32f(y[3].toPtr(T_Int(32)), u1, 3),
                vextract_4x32f(y[4].toPtr(T_Int(32)), u1, 4)
            ))),

        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1),
                vextract_4x32f(y[2].toPtr(T_Int(32)), u1, 2),
                vextract_4x32f(y[3].toPtr(T_Int(32)), u1, 3),
                vextract_4x32f(y[4].toPtr(T_Int(32)), u1, 4),
                assign(u2, vextract_4l_8x32f(x, [1])),
                vstore1_4x32f(y[5].toPtr(vt.t), u2)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1),
                vextract_4x32f(y[2].toPtr(T_Int(32)), u1, 2),
                vextract_4x32f(y[3].toPtr(T_Int(32)), u1, 3),
                vextract_4x32f(y[4].toPtr(T_Int(32)), u1, 4),
                assign(u2, vextract_4l_8x32f(x, [1])),
                vstore1_4x32f(y[5].toPtr(vt.t), u2),
                vextract_4x32f(y[6].toPtr(T_Int(32)), u2, 2)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1),
                vextract_4x32f(y[2].toPtr(T_Int(32)), u1, 2),
                vextract_4x32f(y[3].toPtr(T_Int(32)), u1, 3),
                vextract_4x32f(y[4].toPtr(T_Int(32)), u1, 4),
                assign(u2, vextract_4l_8x32f(x, [1])),
                vstore1_4x32f(y[5].toPtr(vt.t), u2),
                vextract_4x32f(y[6].toPtr(T_Int(32)), u2, 2),
                vextract_4x32f(y[7].toPtr(T_Int(32)), u2, 3)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore1_4x32f(y[1].toPtr(vt.t), u1),
                vextract_4x32f(y[2].toPtr(T_Int(32)), u1, 2),
                vextract_4x32f(y[3].toPtr(T_Int(32)), u1, 3),
                vextract_4x32f(y[4].toPtr(T_Int(32)), u1, 4),
                assign(u2, vextract_4l_8x32f(x, [1])),
                vstore1_4x32f(y[5].toPtr(vt.t), u2),
                vextract_4x32f(y[6].toPtr(T_Int(32)), u2, 2),
                vextract_4x32f(y[7].toPtr(T_Int(32)), u2, 3),
                vextract_4x32f(y[8].toPtr(T_Int(32)), u2, 4)
            )))
        ],
        # Store using subvectors of length 2.
	#
        [
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore_2l_4x32f(y[1].toPtr(TVect(vt.t, 2)), u1)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore_2l_4x32f(y[1].toPtr(TVect(vt.t, 2)), u1),
                vstore_2h_4x32f(y[2].toPtr(TVect(vt.t, 2)), u1)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore_2l_4x32f(y[1].toPtr(TVect(vt.t, 2)), u1),
                vstore_2h_4x32f(y[2].toPtr(TVect(vt.t, 2)), u1),
                assign(u2, vextract_4l_8x32f(x, [1])),
                vstore_2l_4x32f(y[3].toPtr(TVect(vt.t, 2)), u2)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstore_2l_4x32f(y[1].toPtr(TVect(vt.t, 2)), u1),
                vstore_2h_4x32f(y[2].toPtr(TVect(vt.t, 2)), u1),
                assign(u2, vextract_4l_8x32f(x, [1])),
                vstore_2l_4x32f(y[3].toPtr(TVect(vt.t, 2)), u2),
                vstore_2h_4x32f(y[4].toPtr(TVect(vt.t, 2)), u2)
            )))
        ],
        # Store using subvectors of length 4.
	#
        [
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstoreu_4x32f(y[1].toPtr(vt.t), u1)
            ))),
        (y,x,opts) -> let(u1 := var.fresh_t("U", TVect(vt.t, 4)), u2 := var.fresh_t("U", TVect(vt.t, 4)),
            decl([u1, u2], chain(
                assign(u1, vextract_4l_8x32f(x, [0])),
                vstoreu_4x32f(y[1].toPtr(vt.t), u1),
                assign(u2, vextract_4l_8x32f(x, [1])),
                vstoreu_4x32f(y[2].toPtr(vt.t), u2)
            )))
        ]
    ],

    svload  := ~.svload_init(~.t),
    svstore := ~.svstore_init(~.t),

    loadc := (self, sv, opts) >> Cond(
        sv = 8,
            ((y,x) -> assign(y, vloadu_8x32f(x.toPtr(self.t.t)))),
        sv = 4,
            ((y,x) -> assign(y, vinsert_4l_8x32f(self.t.zero(), vloadu_4x32f(x.toPtr(self.t.t)), [0]))),
        sv = 2,
            ((y,x) -> assign(y, vinsert_4l_8x32f(self.t.zero(), vload_2l_4x32f(vzero_4x32f(), x.toPtr(TVect(self.t.t, 2))), [0]))),
        # else
            ((y,x) -> assign(y, vinsert_4l_8x32f(self.t.zero(), vload1_4x32f(x.toPtr(self.t.t)), [0])))),

    loadc_align := (self, sv, align, opts) >> Cond(
        align = 0 and sv = 8,
            ((y,x,addr) -> assign(y, deref(nth(x,addr).toPtr(self.t)))),
        align mod 4 = 0 and sv = 4,
            ((y,x,addr) -> assign(y, vinsert_4l_8x32f(self.t.zero(), deref(nth(x,addr).toPtr(TVect(self.t.t, 4))), [0]))),
        sv = 8,
            ((y,x,addr) -> assign(y, vloadu_8x32f(nth(x,addr).toPtr(self.t.t)))),
        sv = 4,
            ((y,x,addr) -> assign(y, vinsert_4l_8x32f(self.t.zero(), vloadu_4x32f(nth(x,addr).toPtr(self.t.t)), [0]))),
        sv = 2,
            ((y,x,addr) -> assign(y, vinsert_4l_8x32f(self.t.zero(), vload_2l_4x32f(vzero_4x32f(), nth(x,addr).toPtr(TVect(self.t.t, 2))), [0]))),
        # else
            ((y,x,addr) -> assign(y, vinsert_4l_8x32f(self.t.zero(), vload1_4x32f(nth(x,addr).toPtr(self.t.t)), [0])))),

    storec_init := (vt) -> [
        ((y,x) -> vstore1_4x32f(y.toPtr(vt.t), vextract_4l_8x32f(x, [0]))),
        ((y,x) -> vstore_2l_4x32f(y.toPtr(TVect(vt.t, 2)), vextract_4l_8x32f(x, [0]))),
        ((y,x) -> vstoremask_8x32f(y.toPtr(vt.t), x, vstoremask_8x32f.mask(3))),
        ((y,x) -> vstoreu_4x32f(y.toPtr(vt.t), vextract_4l_8x32f(x, [0]))),
        ((y,x) -> vstoremask_8x32f(y.toPtr(vt.t), x, vstoremask_8x32f.mask(5))),
        ((y,x) -> vstoremask_8x32f(y.toPtr(vt.t), x, vstoremask_8x32f.mask(6))),
        ((y,x) -> vstoremask_8x32f(y.toPtr(vt.t), x, vstoremask_8x32f.mask(7))),
        ((y,x) -> vstoreu_8x32f(y.toPtr(vt.t), x))
    ],

    storec  := ~.storec_init(~.t),

    swap_cx := (y, x, opts) -> assign(y, vshuffle_8x32f(x, x, [2,1,4,3])),
    RCVIxJ2 := (y, x, opts) -> assign(y, vperm_8x32f(x, [3,4,1,2])),

    # VJxI implements VJxI(m,v) which is J(m) /tensor I(v/m)
    VJxI := (m, y, x, opts) -> Cond( m=1, assign(y, x),
                                     m=2, assign(y, vupermf128_8x32f(x, [2,1])),
                                     m=4, assign(y, vperm_8x32f(vupermf128_8x32f(x, [2,1]), [3,4,1,2])),
                                     m=8, assign(y, vperm_8x32f(vupermf128_8x32f(x, [2,1]), [4,3,2,1])),
                                     Error("unexpected m")),

    bin_shrev := (self, y, x, opts) >> let(
        u1 := self.freshU(), u2 := self.freshU(), u3 := self.freshU(),
        decl( [u1, u2, u3], chain(
            assign(u1, vupermf128_8x32f(x[2], [2,1])),                # [a,b,c,d,e,f,g,h] -> [e,f,g,h,a,b,c,d]
            assign(u2, vblend_8x32f(x[1], x[2], [1,1,1,1,2,2,2,2])),  # -> [x11,x12,x13,x14,e,f,g,h]
            assign(u3, vushuffle_8x32f(u1, [1,4,3,2])),               # [e,f,g,h,a,b,c,d] -> [e,h,g,f,a,d,c,b]
            assign(y,  vblend_8x32f(u3, u2, [2,1,1,1,2,1,1,1]))))),   # -> [x11,h,g,f,e,d,c,b]

));

#==============================================================================================

SIMD_ISA_DB.addISA(AVX_4x64f);
SIMD_ISA_DB.addISA(AVX_8x32f);


##  Copyright (c) 2018-2021, Carnegie Mellon University
##  See LICENSE for details

RewriteRules(RulesRC, rec(
    RC_ISumAcc := Rule([RC, @(1, ISumAcc)], e -> ISumAcc(@(1).val.var, @(1).val.domain, RC(@(1).val.child(1)))),
    RC_COND := Rule([RC, @(1, COND)], e -> ApplyFunc(COND, [ @(1).val.cond ]::List(@(1).val._children, i->RC(i)))),
));

RewriteRules(RulesFuncSimp, rec(
    H_fTensor_fBase_fId := ARule(fCompose, [@(1, H, e->e.domain()+1 = e.range() and e.params[4] = 1),
        [@(0, fTensor), @(2, fBase), @(3, fId)]],
        e-> [fAdd(@(1).val.range(), @(3).val.domain(), @(1).val.params[3] + @(0).val.at(0))]
    ),
    fTensor_fBase_fId_H := Rule(@(1, fTensor, e->(
            ForAll(e.children(), c->ObjId(c) in [fBase, fId, H] and When(ObjId(c) = H, c.domain() = 1, true))
            and ForAny(e.children(), c->ObjId(c) = H))),
        e->fTensor(List(@(1).val.children(), c->When(ObjId(c) = H, fBase(c.params[1], c.params[3]), c)))
    )
));

RewriteRules(RulesSums, rec(
    Scat_ISumAcc := ARule(Compose,
       [ @(1, Scat), @(2, ISumAcc) ],
        e -> [ CopyFields(@(2).val,
        rec(_children :=  List(@(2).val._children, c -> ScatAcc(@(1).val.func) * c), dimensions := [Rows(@(1).val), Cols(@(2).val)] )) ]),

    PropagateBB := Rule([@(1,BB), @(2,Compose, e->ForAny(e.children(), i->ObjId(i)=COND))],
        e -> Compose(List(@(2).val.children(), i->Cond(ObjId(i) = COND, ApplyFunc(COND, [i.cond]::List(i.children(), j->BB(j))), BB(i))))),

    MergeBB := ARule(Compose, [@(1, BB), @(2, BB)], e-> [ BB(@(1).val.child(1) * @(2).val.child(1)) ]),

    COND_true := Rule(@(1, COND, e->e.cond = V(true)), e -> @(1).val.child(1)),

    eq_false := Rule([@(1, eq), [add, @(3, var, e->e.t=TInt), @(2, Value, e->e.v >0) ], @(0, Value, e->e.v=0)], e-> V(false)),

    eq_false2 := Rule([@(1, eq), @(2, add, e->ForAny(e.args, j->IsValue(j) and j.v > 0)), @(0, Value, e->e.v=0)], e-> V(false)),

    COND_false := Rule(@(1, COND, e->e.cond = V(false)), e -> @(1).val.child(2)),

    DiagISumAccLeft := ARule( Compose, [ @(1, ISumAcc, canReorder), @(2, RightPull) ],
        e -> [ ISumAcc(@1.val.var, @1.val.domain, @1.val.child(1) * @2.val).attrs(@(1).val) ]),

    COND_Diag :=  ARule( Compose, [ @(1, COND), @(2, [Diag, RCDiag]) ],
        e -> [ CopyFields(@(1).val, rec(_children := List(@(1).val._children, c -> c * @(2).val),
                dimensions := [Rows(@(1).val), Cols(@(2).val)] )) ])
));

RewriteRules(RulesStrengthReduce, rec(
    mul_cond := Rule([@(1, mul), [cond, @(2), @(3, Value), @(4, Value)], @(5)],
        e->cond(@(2).val, @(5).val*@(3).val, @(5).val*@(4).val)),

    eq_add := Rule([eq, [add, @(1,Value), @(2)], @(3,Value)], e->eq(@(2).val, @(3).val-@(1).val)),

    eq_false := Rule([@(1, eq), [add, @(3, var, e->e.t=TInt), @(2, Value, e->e.v >0) ], @(0, Value, e->e.v=0)], e-> V(false)),

    eq_false2 := Rule([@(1, eq), @(2, add, e->ForAny(e.args, j->IsValue(j) and j.v > 0)), @(0, Value, e->e.v=0)], e-> V(false)),

    fix_cond := Rule([eq, [mul, @(0,Value, e->e.t=TInt and e.v>0), @(1,var,IsLoopIndex)], @(2, Value, e->e.t=TInt and e.v<0)], e->V(false)),
));

fixISumAccPDLoop := function(s, opts)
    local ss, vr, scat, dim, krnl, lvar, it0, itn, sct, isum, its, sx, scand, scats;

    while Length(Collect(s, ISumAcc)) >0  do
        scand := Collect(s, ISumAcc);
        ss := scand[1];
        vr := ss.var;
        scats := Collect(ss, ScatAcc);
        if Length(scats) > 1 then Error("There should be only one ScatAcc in an ISumAcc..."); fi;
        scat := scats[1];
        dim := Cols(scat);

        krnl := ss.child(1);
        lvar := Ind(vr.range-1);

        it0 := RulesSums(SubstTopDown(SubstVars(Copy(krnl), rec((vr.id) := V(0))), ScatAcc, e-> Scat(fId(dim))));
        itn := RulesSums(RulesSums(SubstTopDown(SubstVars(Copy(krnl), rec((vr.id) := lvar+1)), ScatAcc, e-> ScatAcc(fAdd(dim, dim, 0)))));
        sct := Scat(Collect(krnl, ScatAcc)[1].func);
        isum := ISum(lvar, itn);
        isum.doNotMarkBB := true;
        its := BB(sct) * SUM(it0, isum);

        s := SubstTopDown(s, ss, e->its);
        s := MergedRuleSet(RulesSums, RulesDiag, RulesDiagStandalone, RulesStrengthReduce, RulesRC)(s);
    od;

    return s;
end;

fixScatter := function(s, opts)
    local tags, scts, sct, f, fl, vars, fll, rng, it, itspace, vals, vby2, vby2d, vby2d2, accf2, flnew, sctnew, srec;

    tags := rec();
    if IsBound(s.ruletree) then tags.ruletree  := s.ruletree; fi;

    # very brittle and danger of infinite loop
    scts := Collect(s, @(1, Scat, e->Cols(e) = 4 and Length(Collect(e, BH)) > 0 and ForAny(e.free(), i->i.t <> TInt)));
    while Length(scts) > 0  do
        sct := scts[1];

        f := sct.func;
        fl := f.lambda();
        vars := Filtered(fl.free(), i->i.t = TInt);
        fll := fl.tolist();
        rng := List(vars, i->i.range);
        itspace := Cartesian(List(rng, i->[1..i]));

        vals := [];
        for it in itspace do
            srec := rec();
            for i in [1..Length(vars)] do
               srec := CopyFields(srec, rec((vars[i].id):=V(it[i]-1)));
            od;
            vals := vals :: List(Copy(fll), e->EvalScalar(SubstVars(e, srec)));
        od;

        vby2 := List([1..Length(vals)/2], i->vals[2*i-1]/2);
        vby2d := fTensor(FData(List(vby2, i->V(i))), fId(2));

        vby2d2 := FData(List(vby2, i->V(i)));
        accf2 := fTensor(List(vars, fBase) :: [fId(fl.vars[1].range/2)]);
        flnew := fTensor(fCompose(vby2d2, accf2).lambda().setRange(Rows(sct)/2), fId(2));

        sctnew := Scat(flnew);
        s := SubstTopDownNR(s, sct, e->sctnew);
        # very brittle and danger of infinite loop
        scts := Collect(s, @(1, Scat, e->Cols(e) = 4 and Length(Collect(e, BH)) > 0 and ForAny(e.free(), i->i.t <> TInt)));
    od;
    s := CopyFields(tags, s);

    return s;
end;

fixGather := function(s, opts)
    local tags, gaths, gath, f, fl, vars, fll, rng, it, itspace, vals, vby2, vby2d, vby2d2, accf2, flnew, gathnew, srec;

    tags := rec();
    if IsBound(s.ruletree) then tags.ruletree  := s.ruletree; fi;

    # very brittle and danger of infinite loop
    gaths := Collect(s, @(1, Gath, e->Rows(e) = 4 and Length(Collect(e, BH)) > 0 and ForAny(e.free(), i->i.t <> TInt)));
    while Length(gaths) > 0  do
        gath := gaths[1];

        f := gath.func;
        fl := f.lambda();
        vars := Filtered(fl.free(), i->i.t = TInt);
        fll := fl.tolist();
        rng := List(vars, i->i.range);
        itspace := Cartesian(List(rng, i->[1..i]));

        vals := [];
        for it in itspace do
            srec := rec();
            for i in [1..Length(vars)] do
               srec := CopyFields(srec, rec((vars[i].id):=V(it[i]-1)));
            od;
            vals := vals :: List(Copy(fll), e->EvalScalar(SubstVars(e, srec)));
        od;

        vby2 := List([1..Length(vals)/2], i->vals[2*i-1]/2);
        vby2d := fTensor(FData(List(vby2, i->V(i))), fId(2));

        vby2d2 := FData(List(vby2, i->V(i)));
        accf2 := fTensor(List(vars, fBase) :: [fId(fl.vars[1].range/2)]);
        flnew := fTensor(fCompose(vby2d2, accf2).lambda().setRange(Cols(gath)/2), fId(2));

        gathnew := Gath(flnew);
        s := SubstTopDownNR(s, gath, e->gathnew);
        # very brittle and danger of infinite loop
        gaths := Collect(s, @(1, Gath, e->Rows(e) = 4 and Length(Collect(e, BH)) > 0 and ForAny(e.free(), i->i.t <> TInt)));
    od;
    s := CopyFields(tags, s);
    return s;
end;

fixInitFunc := function(c, opts)
    local cis, ci, ci2, fc, freei, offending, myd, tags;

    tags := rec();
    if IsBound(c.dimensions) then tags.dimensions  := c.dimensions; fi;
    if IsBound(c.ruletree) then tags.ruletree  := c.ruletree; fi;

    cis := Collect(c, @(1, func, e->e.id="init"));

    for ci in cis do
        fc := c.free();
        freei := ci.free();
        offending := Filtered(fc, i->i in freei);
        for myd in offending do
           ci2 := func(ci.ret, ci.id, ci.params,
               data(myd, myd.value, ci.cmd));
           c := SubstTopDown(c, @(1, func, e->e = ci), e->ci2);
        od;
    od;

    c := CopyFields(tags, c);
    return c;
end;

fixUpSigmaSPL := function(s, opts)
    s := fixISumAccPDLoop(s, opts);
    s := fixScatter(s, opts);
    s := fixGather(s, opts);

    s := SubstTopDown(s, [@(1,RCDiag, e->e.dims()=[2,2]),
           @(2, FDataOfs, e->IsBound(e.var.value) and e.var.value = V(Flat(List([1..e.var.value.t.size / 2], i->[V(1.0), V(0.0)])))), @(3, I)], e->I(2));

    s := SubstTopDown(s, [@(1,RCDiag, e->e.dims()=[4,4]), @(2, FDataOfs,
            e->IsBound(e.var.value) and ForAll([1..e.var.value.t.size/4], i->e.var.value.v[4*i-3] = V(1.0)) and ForAll([1..e.var.value.t.size/4],
            i->e.var.value.v[4*i- 2] = V(0.0))), @(3, I)],
        e->let(vals := V(Flat(List([1..@(2).val.var.value.t.size/4], i->[@(2).val.var.value.v[4*i-1], @(2).val.var.value.v[4*i]]))),
            dt := Dat(vals.t).setValue(vals),
            fdo := FDataOfs(dt, 2, V(2) * @(2).val.ofs.free()[1]),
            rcdnew  := RCDiag(fdo, I(2)),
            SumsSPL(DirectSum(I(2), rcdnew), opts))
    );
    return s;
end;

fixReplicatedData := function(c, opts)
    local datas, d, replicas, cc, r, tags;

    tags := rec();
    if IsBound(c.dimensions) then tags.dimensions  := c.dimensions; fi;
    if IsBound(c.ruletree) then tags.ruletree  := c.ruletree; fi;

    c  := SubstBottomUp(c, @(1, data, e->IsArray(e.value.t) and ObjId(e.cmd)=data and IsArray(e.cmd.value.t) and e.value.t.size < e.cmd.value.t.size),
        e->data(@(1).val.cmd.var, @(1).val.cmd.value, data(@(1).val.var, @(1).val.value, @(1).val.cmd.cmd)));

    datas := Collect(c, data);
    for d in datas do
        replicas := Collect(d.cmd, @(1, data,
            e->e.value.t.size <= d.value.t.size and e.value.v = d.value.v{[1..e.value.t.size]}));
        for r in replicas do
            cc := r.cmd;
            cc := SubstVars(cc, rec((r.var.id) := d.var));
            c := SubstTopDown(c, @(1, data, e->e.var = r.var), e->cc);
        od;
    od;
    c := CopyFields(tags, c);
    return c;
end;

scalarizeAccumulator := function(c, opts)
    local scand, svr, svars, tags;

    tags := rec();
    if IsBound(c.dimensions) then tags.dimensions  := c.dimensions; fi;
    if IsBound(c.ruletree) then tags.ruletree  := c.ruletree; fi;

    scand := Filtered(Set(Flat(List(Collect(c, decl), i->i.vars))), e->IsArray(e.t) and ForAll(Collect(c, @(1, nth, f-> f.loc = e)), g->IsValue(g.idx)));
    for svr in scand do
        svars := List([1..svr.t.size], e->var.fresh_t("q", svr.t.t));
        c := SubstTopDown(c, @(1, decl, e->svr in e.vars), e-> decl(svars :: Filtered(@(1).val.vars, i-> i <>svr), @(1).val.cmd));
        c := SubstTopDown(c, @(1, nth, e->e.loc = svr), e->svars[e.idx.v+1]);
    od;
    c := CopyFields(tags, c);
    return c;
end;