adding more concat tests

adding more concat tests

use NumSuch,
    Norm,
    LinearAlgebra,
    Viterbi,
    Core,
    Charcoal;

class NumSuchTest : UnitTest {
  var nv: int = 8,
      D: domain(2) = {1..nv, 1..nv},
      SD: sparse subdomain(D),
      X: [SD] real;

  var vn: [1..0] string;

  proc setUp() {
    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    SD += (1,2); X[1,2] = 1;
    SD += (1,3); X[1,3] = 1;
    SD += (1,4); X[1,4] = 1;
    SD += (2,2); X[2,2] = 1;
    SD += (2,4); X[2,4] = 1;
    SD += (3,4); X[3,4] = 1;
    SD += (4,5); X[4,5] = 1;
    SD += (5,6); X[5,6] = 1;
    SD += (6,7); X[6,7] = 1;
    SD += (6,8); X[6,8] = 1;
    SD += (7,8); X[7,8] = 1;

    var t: Timer;
    t.start();
    return t;
  }

  proc tearDown(ref t: Timer) {
    return super.tearDown(t);
  }

  proc init(verbose:bool) {
    super.init(verbose=verbose);
    this.complete();
  }

  proc testMatrixOperators() {
    var t = this.setUp("Matrix Ops");
    var vn2: [1..0] string;
    vn2.push_back("star lord");
    vn2.push_back("gamora");
    vn2.push_back("groot");
    vn2.push_back("drax");
    vn2.push_back("rocket");
    vn2.push_back("mantis");
    vn2.push_back("yondu");
    vn2.push_back("nebula");

    //for n in vn do vn2.push_back(n);
    //vn2.push_back("nebula");
    var nm = new NamedMatrix(X=X, names=vn2);
    nm.set(i=2, j=7, w=13.1);
    assertRealEquals("Can set a new entry in matrix", expected=13.1, actual=nm.get(2,7));
    nm.remove(i=2, j=7);
    assertBoolEquals("Can remove an entry from the matrix", expected=false, actual=nm.SD.member(2,7));

    // Same thing with names instead
    nm.set("star lord", "yondu", w=13.1);
    assertRealEquals("Can set a new entry in matrix by name", expected=13.1, actual=nm.get("star lord","yondu"));
    nm.remove("star lord", "yondu");
    assertBoolEquals("Can remove an entry from the matrix by name", expected=false, actual=nm.SD.member(2,7));

    this.tearDown(t);
  }

  proc tropicalTesting() {
    var t = this.setUp("Tropicals");
    var nv: int = 8,
        D: domain(2) = {1..nv, 1..nv},
        SD = CSRDomain(D),
        X: [SD] real;

    SD += (1,2); X[1,2] = 1;
    SD += (1,3); X[1,3] = 1;
    SD += (1,4); X[1,4] = 1;
    SD += (2,2); X[2,2] = 1;
    SD += (2,4); X[2,4] = 1;
    SD += (3,4); X[3,4] = 1;
    SD += (4,5); X[4,5] = 1;
    SD += (5,6); X[5,6] = 1;
    SD += (6,7); X[6,7] = 1;
    SD += (6,8); X[6,8] = 1;
    SD += (7,8); X[7,8] = 1;

    var X2 = tropic(X,X);
//    writeln(X2);
//    writeln(X2.domain);
//    tropicLimit(X,X);
//    var R = tropic(X,X);
    //writeln(&& reduce (X2 == X));

    var X3 = tropic(X2,X);
    writeln(X3);
    writeln(X3.domain);
    var X4 = tropic(X3,X);
    writeln(X4);
    writeln(X4.domain);
    var X5 = tropic(X4,X);
    writeln(X5);
    writeln(X5.domain);
    var X6 = tropic(X5,X);
    writeln(X6);
    writeln(X6.domain);

  //  writeln(sparseEq(X6,X));
  //  writeln(sparseEq(X6,X5));

    writeln(tropicLimit(X,X));
    this.tearDown(t);
/*

    if R.domain == X.domain {
      var same = && reduce (R == X);
      if same {
        writeln("Arrays Equal");
        writeln(R);
      } else {
        writeln("Arrays Not Equal");
        writeln(tropic(R,X));
      }
    } else {
      writeln("Domains Not Equal");
      writeln(tropic(R,X));
    }*/
  }

  proc testIndexSort() {
    var t = this.setUp("IndexSort");
    var p: bool = true;
    const Arr = [7, 10, 23, 1];
    const Idx = [2.2, 3.3, 1.1, 4.4];

    var a: [1..4] real = for x in indexSort(arr=Arr, idx=Idx) do x;
    var expected: [1..4] real = [23, 7, 10, 1];
    assertArrayEquals(msg="IndexSort", expected=expected, actual=a);

    var b: [1..4] real = for x in indexSort(arr=Arr, idx=Idx, reverse=true) do x;
    var e: [1..4] real = [1, 10, 7, 23];
    assertArrayEquals(msg="IndexSort reversed", expected=e, actual=b);
    this.tearDown(t);
  }

  proc testNamedMatrix() {
    var t = this.setUp("Named Matrix");
    var nm = new NamedMatrix(X=X);
    nm.set(1,3, 17.0);

    assertIntEquals(msg="nrows() set correctly", expected=8, actual=nm.nrows());
    assertIntEquals(msg="ncols() set correctly", expected=8, actual=nm.ncols());
    assertIntEquals(msg="Number of non-zeroes", expected=11:int, actual=11:int );
    assertRealEquals(msg="X.sparsity", expected=0.171875:real, actual=nm.sparsity());
    assertRealEquals(msg="Max of row 1", expected=17.0:real, actual=nm.rowMax(1));
    var e: [1..8] real = [17.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, NAN];
    assertArrayEquals(msg="Max of all rows", expected=e, actual=nm.rowMax());

    var DY = {1..3, 1..4},
        SDY = CSRDomain(DY),
        Y:[SDY] real;
    SDY +=(1,3); Y[1,3] = 1;
    var nm2 = new NamedMatrix(X=Y);
    assertIntEquals("grid2seq(tuple) works on the matrix", expected=9, actual=nm2.grid2seq((3,1)));
    assertIntEquals("grid2seq(i,j) works on the matrix", expected=9, actual=nm2.grid2seq(3,1));
    var sg = nm2.seq2grid(7);
    assertIntEquals("seq2grid works on the matrix", expected=2, actual=sg[1]);
    assertIntEquals("seq2grid works on the matrix", expected=3, actual=sg[2]);

    var dX: [DY] real;
    fillRandom(dX);
    var dnm = new NamedMatrix(X=dX);
    assertIntEquals("Dense NamedMatrix is all non-zeroes", expected=12, actual=dnm.nnz());

    this.tearDown(t);
  }

  proc testSetRowNames() {
    var t = this.setUp("SetRowNames");
    var nm = new NamedMatrix(X=X);
    var vn2: [1..0] string;
    for n in vn do vn2.push_back(n);
    vn2.push_back("stakar ogord");

    try {
      nm.setRowNames(vn2);
      assertThrowsError(msg="Set Row Names wrong size NOT ENFORCED", passed=false, new Error());
    } catch err: DimensionMatchError {
      assertThrowsError(msg="Set Row Names wrong size ENFORCED",  passed=true, err=err);
    } catch err: Error {
      assertThrowsError(msg="Set Row Names wrong size ENFORCED",  passed=true, err=err);
    }

    try {
      nm.setRowNames(vn2);
      assertIntEquals(msg="Set Row Names right size ENFORCED",  expected=vn.size, actual=nm.nrows());
    } catch {
      assertIntEquals(msg="Set Row Names right size failed",  expected=8, actual=nm.nrows());
    }

    this.tearDown(t);
  }

  proc testSetColNames() {
    var vn: [1..0] string;
    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    var nm = new NamedMatrix(X=X);
    try {
      nm.setColNames(vn);
      assertIntEquals(msg="Set Col Names right size ENFORCED",  expected=vn.size, actual=nm.ncols());
    } catch {
      assertIntEquals(msg="Set Col Names right size",  expected=vn.size, actual=nm.ncols());
    }
  }

  proc testNamedMatrixInitWithNames() {
    try {
      var nm = new NamedMatrix(rownames = vn, colnames=vn);
      assertIntEquals(msg="Initialize X with names only (rows)", expected=vn.size, actual=nm.nrows());
      assertIntEquals(msg="Initialize X with names only (cols)", expected=vn.size, actual=nm.ncols());
    } catch e: DimensionMatchError {
      writeln(e);
    } catch e: Error {
      writeln(e);
    }
  }

  proc testSetByName() {
    var t = this.setUp("SetByName");
    try {
      var nm = new NamedMatrix(rownames = vn, colnames=vn);
      nm.set("star lord", "yondu", 17.0);
      assertRealEquals(msg="Set/Get by name", expected=17.0, actual=nm.get("star lord", "yondu"));
    } catch e: DimensionMatchError {
      writeln(e);
    } catch e: Error {
      writeln(e);
    }
    this.tearDown(t);
  }

  proc testArgMaxMin() {
    var t = this.setUp("ArgMaxMin");
    var x: [1..3] real = [1.1, 3.3, 2.2],
        y: [1..3,1..3] real = ((1,0.1,0.5), (0.3,0.2,2), (0.7,3,0)),
        xSD: sparse subdomain(x.domain),
        z: [xSD] real,
        vn: [1..0] string;
    xSD += 2; z[2] = 1.3;
    xSD += 3; z[3] = 0.8;

    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    assertBoolEquals("z is sparse", expected=true, actual=isSparseArr(z));
    assertIntEquals("argmax1d(z)", expected=2, actual=argmax1d(z));

    assertIntEquals("argmax(x)", expected=2, actual=argmax(x));
    assertIntArrayEquals("argmax(y, ?)", expected=[3,2], actual=argmax(y));
    assertIntArrayEquals("argmax(y,0)", expected=[3,2], actual=argmax(y, axis=0));
    assertIntArrayEquals("argmax(y,1)", expected=[1,3,2], actual=argmax(y, axis=1));
    assertIntArrayEquals("argmax(y,2)", expected=[1,3,2], actual=argmax(y, axis=2));

    assertIntEquals("argmin(x)", expected=1, actual=argmin(x));
    assertIntArrayEquals("argmin(y, ?)", expected=[3,3], actual=argmin(y));
    assertIntArrayEquals("argmin(y,0)", expected=[3,3], actual=argmin(y, axis=0));
    assertIntArrayEquals("argmin(y,1)", expected=[2,2,3], actual=argmin(y, axis=1));
    assertIntArrayEquals("argmin(y,2)", expected=[2,1,3], actual=argmin(y, axis=2));

    var SD2: sparse subdomain(D),
        X2:[SD2] real;
    SD2 += (1,2); X2[1,2] = 1;
    SD2 += (3,1); X2[3,1] = 3;
    SD2 += (3,4); X2[3,4] = 4;
    SD2 += (3,6); X2[3,6] = 1;
    var m = new NamedMatrix(X=X2, names=vn);
    assertRealEquals("m rowMax by row number", expected=4, actual=m.rowMax(3));
    assertRealEquals("m rowMax by row name", expected=4, actual=m.rowMax("groot"));
    var e: [1..4] real = [3.0, 1.0, 4.0, 1.0];
    assertArrayEquals("m colMax", expected=e, actual=m.colMax());
    var f: [1..4] int = [1,2,3,4];
    //writeln(m.rowArgMax());
    //assertIntArrayEquals("m rowArgMax", expected=f, actual=m.rowArgMax());
    //assertIntEquals("m rowArgMax(3)", expected=4, actual=m.rowArgMax(3));
    /* This interface is kind of annoying, I know what row I'm sending in so I
       should just get the answer back */
    assertIntEquals("m rowArgMax(3)", expected=4, actual=m.rowArgMax(3)[2]);
    assertIntEquals("m rowArgMax('groot')", expected=4, actual=m.rowArgMax('groot')[2]);

    assertIntEquals("m rowArgMin(3)", expected=6, actual=m.rowArgMin(3)[2]);
    assertIntEquals("m rowArgMin('groot')", expected=6, actual=m.rowArgMin('groot')[2]);

    assertRealEquals("m rowMin(3)", expected=1, actual=m.rowMin(3));
    assertRealEquals("m rowMin('groot')", expected=1, actual=m.rowMin('groot'));

    assertRealEquals("m colMin(6)", expected=1, actual=m.colMin(6));
    assertRealEquals("m colMin('mantis')", expected=1, actual=m.colMin('mantis'));

    assertRealEquals("m colMax(6)", expected=1, actual=m.colMax(6));
    assertRealEquals("m colMax('mantis')", expected=1, actual=m.colMax('mantis'));

    assertIntEquals("m colArgMax(6)", expected=6, actual=m.colArgMax(6)[2]);
    assertIntEquals("m colArgMax('mantis')", expected=6, actual=m.colArgMax('mantis')[2]);

    assertIntEquals("m colArgMin(6)", expected=6, actual=m.colArgMin(6)[2]);
    assertIntEquals("m colArgMin('mantis')", expected=6, actual=m.colArgMin('mantis')[2]);

    this.tearDown(t);
  }

  proc testCosineDistance() {
    var t = this.setUp("Cosine Distance");
    var X = Matrix(
      [3.0, 0.2, 0.0, 0.7, 0.1],
      [0.2, 2.0, 0.3, 0.0, 0.0],
      [0.0, 0.3, 3.0, 0.9, 0.6],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.1, 0.0, 0.6, 0.0, 2.0]
    );

    var Y = Matrix(
      [3.0, 0.2, 0.0, 0.7, 0.1],
      [0.2, 2.0, 0.3, 0.0, 0.0],
      [0.0, 0.3, 3.0, 0.9, 0.6],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.1, 0.0, 0.6, 0.0, 2.0]
    );

    const xxTarget = Matrix(
      [0.0, 0.974619, 0.992338, 0.930778, 0.988007],
      [0.974619, 0.0, 0.964601, 0.981269, 0.988919],
      [0.992338, 0.964601, 0.0, 0.917246, 0.93309],
      [0.930778, 0.981269, 0.917246, 0.0, 0.973663],
      [0.988007, 0.988919, 0.93309, 0.973663, 0.0]);

    var V = cosineDistance(X);
    const a = V-xxTarget;
    const aa = norm(a);
    assertRealApproximates(msg="Cosine Distance(X) norm", expected=1.39884e-06, actual=aa);

    var V2 = cosineDistance(X,Y);
    const cosimXYtarget = Matrix(
      [0.895178, 0.974619, 0.992338, 0.930778, 0.930778, 0.988007],
      [0.974619, 0.757869, 0.964601, 0.981269, 0.981269, 0.988919],
      [0.992338, 0.964601, 0.902534, 0.917246, 0.917246, 0.93309],
      [0.930778, 0.981269, 0.917246, 0.811321, 0.811321, 0.973663],
      [0.988007, 0.988919, 0.93309,  0.973663, 0.973663, 0.771167]
      );
    const b = V2-cosimXYtarget;
    const bb = norm(b);
    assertRealApproximates(msg="Cosine Distance(X,Y) norm", expected=1.53583e-06, actual=bb);
    this.tearDown(t);
  }

  /*
   Does not assert anything, LabeledMatrix may be deprecated
   */
  proc testLabelMatrix() {
    var t = this.setUp("Label Matrix");
    var Y = Matrix(
      [3.0, 0.2, 0.0, 0.7, 0.1],
      [0.2, 2.0, 0.3, 0.0, 0.0],
      [0.0, 0.3, 3.0, 0.9, 0.6],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.1, 0.0, 0.6, 0.0, 2.0]
    );
    var L = new LabelMatrix();
    L.fromMatrix(Y);
    assertRealEquals(msg="Label Matrix entry (1,4)", expected=0.7, actual=L.data(1,4));
    this.tearDown(t);
  }

  proc testECDF() {
    var t = this.setUp("ECDF");
    var x = [3,3,1,4];
    var ecdf = new ECDF(x);
    assertIntEquals("ECDF: Number of observations", expected=4, actual=ecdf.nobs);

    var y = [3.0, 55.0, 0.5, 1.5];
    var d = ecdf(y);
    assertArrayEquals("ECDF: Output", expected=[0.75,1.0,0.0,0.25], actual=d);
    this.tearDown(t);
  }

  proc testBiMap() {
    var t = this.setUp("BiMap");
    var bm = new BiMap();
    bm.add("bob");
    bm.add("chuck");
    bm.add("bob");
    bm.add('ethel', 78);
    bm.add('frank', 3);

    assertIntEquals("BIMAP redundant entry ignored", expected=4, actual=bm.keys.size);
    for k in bm.keys {
      assertStringEquals("BIMAP name is retrieved", k, bm.idx[bm.ids[k]]);
      assertIntEquals("BIMAP id is retrieved", bm.ids[k], bm.get(k));
    }
    assertIntEquals("BIMPA max is set", expected=78, actual=bm.max());

    var abm = new BiMap();
    abm.add("one",1);
    abm.add("two",2);
    var bbm = new BiMap();
    bbm.add("three",3);
    bbm.add("four",4);

    var cbm = abm.uni(bbm);
    assertIntEquals("BIMAP union size", expected=4, actual=cbm.keys.size);
    assertIntEquals("BIMAP union max", expected=4, actual=cbm.max());
    this.tearDown(t);
  }

  proc testViterbi() {
    var t = this.setUp("Viterbi");
    var obs = ["normal", "cold", "dizzy"];
    var states = ["health", "fever"];
    var initP = [0.6, 0.4];

    /* (Hidden) State transition matrix
             Healthy Fever
    Healthy  0.7     0.3
      Fever  0.4     0.6
     */
    var A: [1..2, 1..2] real = Matrix(
      [0.7, 0.3],
      [0.4, 0.6]
    );

    /*
     State emission probabilities
             Normal Cold Dizzy
    Healthy  0.5    0.4  0.1
      Fever  0.1    0.3  0.6
     */
     var B: [1..2, 1..3] real = Matrix(
       [0.5, 0.4, 0.1],
       [0.1, 0.3, 0.6]
    );
    var v = Viterbi(obs, states, initP, transitionProbabilities=A, emissionProbabilities=B);
    assertStringArrayEquals("Viterbi states output", expected=["health","health","fever"], actual=v);
    this.tearDown(t);
  }

  proc testLetters() {
    var t= this.setUp("Letters");
    var ltrs = letters(28);
    assertStringEquals("First in letters is A", expected="A", actual=ltrs[1]);
    assertStringEquals("28th in letters is BB", expected="BB", actual=ltrs[28]);

    var gn = gridNames(7);
    assertIntEquals("Grid names(7) has 49 entries", expected=49, actual=gn.size);
    assertStringEquals("Last grid name is 'G7'", expected="G7", actual=gn[49]);
    var gn2 = gridNames(i=3, j=5);
    assertRealEquals("gridNames(3,5) has 15 entries", expected=15, actual=gn2.size);
    assertStringEquals("Last grid name is 'C5'", expected="C5", actual=gn2[15]);
    this.tearDown(t);
  }

  proc testChoice() {
    var t = this.setUp("Choice");
    var x = [1,2,3,4,5,6,7,8,9,10];
    var y = choice(a=x,size=2);
    assertIntEquals("Can pick two choices, no replacement, no p", expected=2, actual=y.size);
    var z = choice(x,size=2,replace=true);
    assertIntEquals("Can pick two choices, with replacement, no p", expected=2, actual=z.size);

    var s = [1, 2, 3, 4, 5];
    // Note, this does not need to be normalized but it is easier to see
    var p = [0.5, 0.25, 0.13, 0.09, 0.03];
    var m = chooseMultinomial(a=s, replace=false, size=3, p=p);
    assertIntEquals("Multinomial w/o replace returns correct number of results", expected=3, actual=m.size);
    var n = chooseMultinomial(a=s, replace=true, size=3, p=p);
    assertIntEquals("Multinomial w/ replace returns correct number of results", expected=3, actual=n.size);
    var o = choice(a=s, replace=false, size=3, p=p);

    this.tearDown(t);
  }

  proc testRandomGenerators() {
    var t= this.setUp("Random Generators");
    var n: int = 100000;
    var x: [1..n] int;
    //for i in 1..n { x[i] = randInt(1,6); }
    for i in 1..n do x[i] = randInt(1,6);
    var mu = + reduce x;
    assertRealApproximates("Average of 100,000 randInt(1,6) is about 3.5"
      , expected=3.5, actual=mu:real/n:real, error=0.01);

    var threeD6 = for i in 1..n do nds(n=3, s=6);
    assertRealApproximates("Average of 100,000 three d6 is about 10.5"
      , expected=10.5, actual=(+reduce threeD6):real/n:real, error=0.012);

    // Check random reals
    var rrls: [1..n] real;
    for i in 1..n do rrls[i] = rand(a=2,b=7);
    assertRealApproximates("Average of 100,000 random(2,7) is about 4.5"
      , expected=4.5, actual=(+reduce rrls):real/n:real, error=0.01);

    var sq = seq(3,7);
    assertIntEquals("First element of sequence is 3", expected=3, actual=sq[1]);
    assertIntEquals("Last element of sequence is 7", expected=7, actual=sq[5]);

    var sq2 = seq(3,7, stride=5);
    assertIntEquals("First element of sequence(stride=5) is 3", expected=3, actual=sq2[1]);
    assertIntEquals("Last element of sequence(stride=5) is 7", expected=23, actual=sq2[5]);

    this.tearDown(t);
  }

  proc testPersistance() {

  }

  proc testMatrixMakers() {
    var t = this.setUp("Matrix makers");
    var n: int = 5,
        d: domain(2),
        X: [d] real;

    d =  {1..n, 1..n};
    const o = ones(X.domain);
    assertRealEquals(" Norm of o is 5", expected=5.0, actual=norm(o));

    const p = ones(X.domain, v=7.0);
    assertRealEquals(" Norm of p is 7", expected=35.0, actual=norm(p));

    this.tearDown(t);
  }

  proc testRowColSums() {
    var t = this.setUp("RowColSums");
    var X = Matrix( [5,2] ,[6,1] ,[-1,6] ,[1,1] );
    assertArrayEquals("rowSums of X", expected=[7.0, 7.0, 5.0, 2.0], actual=rowSums(X));
    assertArrayEquals("colSums of X", expected=[11.0, 10.0], actual=colSums(X));

    this.tearDown(t);
  }

  proc testConcat() {
    var t = this.setUp("Concat");
    var x: [1..3] real = [1.1, 2.2, 3.3],
        y: [9..11] real = [9.9, 10.10, 11.11];

    var z = concat(x,y);
    var zz: [1..6] real = [1.1, 2.2, 3.3, 9.9, 10.10, 11.11];
    assertIntEquals(msg="Concatenated real array is 6 wide", expected=6, actual=z.size);
    assertArrayEquals(msg="Check concat gives correct array", expected=zz, actual=z);

    // Now integer Arrays
    var r: [1..3] int = [0, 1, 0],
        s: [8..10] int = [1, 1, 0];

    var u = concat(r, s);
    assertIntArrayEquals(msg="Concat gets integer arrays"
      ,expected = [0,1,0,1,1,0], actual=concat(r,s));

    var a = [3,2,1],
        b = [4,5,6];
    assertIntArrayEquals(msg="Concat gets integer arrays (no domain)"
       ,expected=[3,2,1,4,5,6], actual=concat(a,b));

    this.tearDown(t);
  }

  proc testNamedDenseMatrix() {
    var t = this.setUp("Named Dense Matrix");
    var dom: domain(2) = {1..7,1..6};
    var W: [dom] real;
    fillRandom(W);
    var nm = new NamedMatrix(X=W);
    //writeln(nm.X);

    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    SD += (1,2); X[1,2] = 1;
    SD += (1,3); X[1,3] = 1;
    SD += (1,4); X[1,4] = 1;
    SD += (2,2); X[2,2] = 1;
    SD += (2,4); X[2,4] = 1;
    SD += (3,4); X[3,4] = 1;
    SD += (4,5); X[4,5] = 1;
    SD += (5,6); X[5,6] = 1;
    SD += (6,7); X[6,7] = 1;
    SD += (6,8); X[6,8] = 1;
    SD += (7,8); X[7,8] = 1;

    //writeln(X);

    var nm2 = new NamedMatrix(X=X);
    //writeln(nm2.X);

    this.tearDown(t);
  }

  proc run() {
    super.run();
    testNamedDenseMatrix();
    testRowColSums();
    testMatrixMakers();
    testMatrixOperators();
//    tropicalTesting();
    testIndexSort();
    testNamedMatrix();
    testSetRowNames();
    testSetColNames();
//    testNamedMatrixInitWithNames();
    testSetByName();
      testArgMaxMin();
//    testCosineDistance();
//    testLabelMatrix();
    testECDF();
    testBiMap();
    testViterbi();
    testLetters();
    testChoice();
    testRandomGenerators();
    testConcat();
    return 0;
  }
}

proc main(args: [] string) : int {
  var t = new NumSuchTest(verbose=false);
  var ret = t.run();
  t.report();
  return ret;
}

use NumSuch,
    Norm,
    LinearAlgebra,
    Viterbi,
    Core,
    Charcoal;

class NumSuchTest : UnitTest {
  var nv: int = 8,
      D: domain(2) = {1..nv, 1..nv},
      SD: sparse subdomain(D),
      X: [SD] real;

  var vn: [1..0] string;

  proc setUp() {
    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    SD += (1,2); X[1,2] = 1;
    SD += (1,3); X[1,3] = 1;
    SD += (1,4); X[1,4] = 1;
    SD += (2,2); X[2,2] = 1;
    SD += (2,4); X[2,4] = 1;
    SD += (3,4); X[3,4] = 1;
    SD += (4,5); X[4,5] = 1;
    SD += (5,6); X[5,6] = 1;
    SD += (6,7); X[6,7] = 1;
    SD += (6,8); X[6,8] = 1;
    SD += (7,8); X[7,8] = 1;

    var t: Timer;
    t.start();
    return t;
  }

  proc tearDown(ref t: Timer) {
    return super.tearDown(t);
  }

  proc init(verbose:bool) {
    super.init(verbose=verbose);
    this.complete();
  }

  proc testMatrixOperators() {
    var t = this.setUp("Matrix Ops");
    var vn2: [1..0] string;
    vn2.push_back("star lord");
    vn2.push_back("gamora");
    vn2.push_back("groot");
    vn2.push_back("drax");
    vn2.push_back("rocket");
    vn2.push_back("mantis");
    vn2.push_back("yondu");
    vn2.push_back("nebula");

    //for n in vn do vn2.push_back(n);
    //vn2.push_back("nebula");
    var nm = new NamedMatrix(X=X, names=vn2);
    nm.set(i=2, j=7, w=13.1);
    assertRealEquals("Can set a new entry in matrix", expected=13.1, actual=nm.get(2,7));
    nm.remove(i=2, j=7);
    assertBoolEquals("Can remove an entry from the matrix", expected=false, actual=nm.SD.member(2,7));

    // Same thing with names instead
    nm.set("star lord", "yondu", w=13.1);
    assertRealEquals("Can set a new entry in matrix by name", expected=13.1, actual=nm.get("star lord","yondu"));
    nm.remove("star lord", "yondu");
    assertBoolEquals("Can remove an entry from the matrix by name", expected=false, actual=nm.SD.member(2,7));

    this.tearDown(t);
  }

  proc tropicalTesting() {
    var t = this.setUp("Tropicals");
    var nv: int = 8,
        D: domain(2) = {1..nv, 1..nv},
        SD = CSRDomain(D),
        X: [SD] real;

    SD += (1,2); X[1,2] = 1;
    SD += (1,3); X[1,3] = 1;
    SD += (1,4); X[1,4] = 1;
    SD += (2,2); X[2,2] = 1;
    SD += (2,4); X[2,4] = 1;
    SD += (3,4); X[3,4] = 1;
    SD += (4,5); X[4,5] = 1;
    SD += (5,6); X[5,6] = 1;
    SD += (6,7); X[6,7] = 1;
    SD += (6,8); X[6,8] = 1;
    SD += (7,8); X[7,8] = 1;

    var X2 = tropic(X,X);
//    writeln(X2);
//    writeln(X2.domain);
//    tropicLimit(X,X);
//    var R = tropic(X,X);
    //writeln(&& reduce (X2 == X));

    var X3 = tropic(X2,X);
    writeln(X3);
    writeln(X3.domain);
    var X4 = tropic(X3,X);
    writeln(X4);
    writeln(X4.domain);
    var X5 = tropic(X4,X);
    writeln(X5);
    writeln(X5.domain);
    var X6 = tropic(X5,X);
    writeln(X6);
    writeln(X6.domain);

  //  writeln(sparseEq(X6,X));
  //  writeln(sparseEq(X6,X5));

    writeln(tropicLimit(X,X));
    this.tearDown(t);
/*

    if R.domain == X.domain {
      var same = && reduce (R == X);
      if same {
        writeln("Arrays Equal");
        writeln(R);
      } else {
        writeln("Arrays Not Equal");
        writeln(tropic(R,X));
      }
    } else {
      writeln("Domains Not Equal");
      writeln(tropic(R,X));
    }*/
  }

  proc testIndexSort() {
    var t = this.setUp("IndexSort");
    var p: bool = true;
    const Arr = [7, 10, 23, 1];
    const Idx = [2.2, 3.3, 1.1, 4.4];

    var a: [1..4] real = for x in indexSort(arr=Arr, idx=Idx) do x;
    var expected: [1..4] real = [23, 7, 10, 1];
    assertArrayEquals(msg="IndexSort", expected=expected, actual=a);

    var b: [1..4] real = for x in indexSort(arr=Arr, idx=Idx, reverse=true) do x;
    var e: [1..4] real = [1, 10, 7, 23];
    assertArrayEquals(msg="IndexSort reversed", expected=e, actual=b);
    this.tearDown(t);
  }

  proc testNamedMatrix() {
    var t = this.setUp("Named Matrix");
    var nm = new NamedMatrix(X=X);
    nm.set(1,3, 17.0);

    assertIntEquals(msg="nrows() set correctly", expected=8, actual=nm.nrows());
    assertIntEquals(msg="ncols() set correctly", expected=8, actual=nm.ncols());
    assertIntEquals(msg="Number of non-zeroes", expected=11:int, actual=11:int );
    assertRealEquals(msg="X.sparsity", expected=0.171875:real, actual=nm.sparsity());
    assertRealEquals(msg="Max of row 1", expected=17.0:real, actual=nm.rowMax(1));
    var e: [1..8] real = [17.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, NAN];
    assertArrayEquals(msg="Max of all rows", expected=e, actual=nm.rowMax());

    var DY = {1..3, 1..4},
        SDY = CSRDomain(DY),
        Y:[SDY] real;
    SDY +=(1,3); Y[1,3] = 1;
    var nm2 = new NamedMatrix(X=Y);
    assertIntEquals("grid2seq(tuple) works on the matrix", expected=9, actual=nm2.grid2seq((3,1)));
    assertIntEquals("grid2seq(i,j) works on the matrix", expected=9, actual=nm2.grid2seq(3,1));
    var sg = nm2.seq2grid(7);
    assertIntEquals("seq2grid works on the matrix", expected=2, actual=sg[1]);
    assertIntEquals("seq2grid works on the matrix", expected=3, actual=sg[2]);

    var dX: [DY] real;
    fillRandom(dX);
    var dnm = new NamedMatrix(X=dX);
    assertIntEquals("Dense NamedMatrix is all non-zeroes", expected=12, actual=dnm.nnz());

    this.tearDown(t);
  }

  proc testSetRowNames() {
    var t = this.setUp("SetRowNames");
    var nm = new NamedMatrix(X=X);
    var vn2: [1..0] string;
    for n in vn do vn2.push_back(n);
    vn2.push_back("stakar ogord");

    try {
      nm.setRowNames(vn2);
      assertThrowsError(msg="Set Row Names wrong size NOT ENFORCED", passed=false, new Error());
    } catch err: DimensionMatchError {
      assertThrowsError(msg="Set Row Names wrong size ENFORCED",  passed=true, err=err);
    } catch err: Error {
      assertThrowsError(msg="Set Row Names wrong size ENFORCED",  passed=true, err=err);
    }

    try {
      nm.setRowNames(vn2);
      assertIntEquals(msg="Set Row Names right size ENFORCED",  expected=vn.size, actual=nm.nrows());
    } catch {
      assertIntEquals(msg="Set Row Names right size failed",  expected=8, actual=nm.nrows());
    }

    this.tearDown(t);
  }

  proc testSetColNames() {
    var vn: [1..0] string;
    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    var nm = new NamedMatrix(X=X);
    try {
      nm.setColNames(vn);
      assertIntEquals(msg="Set Col Names right size ENFORCED",  expected=vn.size, actual=nm.ncols());
    } catch {
      assertIntEquals(msg="Set Col Names right size",  expected=vn.size, actual=nm.ncols());
    }
  }

  proc testNamedMatrixInitWithNames() {
    try {
      var nm = new NamedMatrix(rownames = vn, colnames=vn);
      assertIntEquals(msg="Initialize X with names only (rows)", expected=vn.size, actual=nm.nrows());
      assertIntEquals(msg="Initialize X with names only (cols)", expected=vn.size, actual=nm.ncols());
    } catch e: DimensionMatchError {
      writeln(e);
    } catch e: Error {
      writeln(e);
    }
  }

  proc testSetByName() {
    var t = this.setUp("SetByName");
    try {
      var nm = new NamedMatrix(rownames = vn, colnames=vn);
      nm.set("star lord", "yondu", 17.0);
      assertRealEquals(msg="Set/Get by name", expected=17.0, actual=nm.get("star lord", "yondu"));
    } catch e: DimensionMatchError {
      writeln(e);
    } catch e: Error {
      writeln(e);
    }
    this.tearDown(t);
  }

  proc testArgMaxMin() {
    var t = this.setUp("ArgMaxMin");
    var x: [1..3] real = [1.1, 3.3, 2.2],
        y: [1..3,1..3] real = ((1,0.1,0.5), (0.3,0.2,2), (0.7,3,0)),
        xSD: sparse subdomain(x.domain),
        z: [xSD] real,
        vn: [1..0] string;
    xSD += 2; z[2] = 1.3;
    xSD += 3; z[3] = 0.8;

    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    assertBoolEquals("z is sparse", expected=true, actual=isSparseArr(z));
    assertIntEquals("argmax1d(z)", expected=2, actual=argmax1d(z));

    assertIntEquals("argmax(x)", expected=2, actual=argmax(x));
    assertIntArrayEquals("argmax(y, ?)", expected=[3,2], actual=argmax(y));
    assertIntArrayEquals("argmax(y,0)", expected=[3,2], actual=argmax(y, axis=0));
    assertIntArrayEquals("argmax(y,1)", expected=[1,3,2], actual=argmax(y, axis=1));
    assertIntArrayEquals("argmax(y,2)", expected=[1,3,2], actual=argmax(y, axis=2));

    assertIntEquals("argmin(x)", expected=1, actual=argmin(x));
    assertIntArrayEquals("argmin(y, ?)", expected=[3,3], actual=argmin(y));
    assertIntArrayEquals("argmin(y,0)", expected=[3,3], actual=argmin(y, axis=0));
    assertIntArrayEquals("argmin(y,1)", expected=[2,2,3], actual=argmin(y, axis=1));
    assertIntArrayEquals("argmin(y,2)", expected=[2,1,3], actual=argmin(y, axis=2));

    var SD2: sparse subdomain(D),
        X2:[SD2] real;
    SD2 += (1,2); X2[1,2] = 1;
    SD2 += (3,1); X2[3,1] = 3;
    SD2 += (3,4); X2[3,4] = 4;
    SD2 += (3,6); X2[3,6] = 1;
    var m = new NamedMatrix(X=X2, names=vn);
    assertRealEquals("m rowMax by row number", expected=4, actual=m.rowMax(3));
    assertRealEquals("m rowMax by row name", expected=4, actual=m.rowMax("groot"));
    var e: [1..4] real = [3.0, 1.0, 4.0, 1.0];
    assertArrayEquals("m colMax", expected=e, actual=m.colMax());
    var f: [1..4] int = [1,2,3,4];
    //writeln(m.rowArgMax());
    //assertIntArrayEquals("m rowArgMax", expected=f, actual=m.rowArgMax());
    //assertIntEquals("m rowArgMax(3)", expected=4, actual=m.rowArgMax(3));
    /* This interface is kind of annoying, I know what row I'm sending in so I
       should just get the answer back */
    assertIntEquals("m rowArgMax(3)", expected=4, actual=m.rowArgMax(3)[2]);
    assertIntEquals("m rowArgMax('groot')", expected=4, actual=m.rowArgMax('groot')[2]);

    assertIntEquals("m rowArgMin(3)", expected=6, actual=m.rowArgMin(3)[2]);
    assertIntEquals("m rowArgMin('groot')", expected=6, actual=m.rowArgMin('groot')[2]);

    assertRealEquals("m rowMin(3)", expected=1, actual=m.rowMin(3));
    assertRealEquals("m rowMin('groot')", expected=1, actual=m.rowMin('groot'));

    assertRealEquals("m colMin(6)", expected=1, actual=m.colMin(6));
    assertRealEquals("m colMin('mantis')", expected=1, actual=m.colMin('mantis'));

    assertRealEquals("m colMax(6)", expected=1, actual=m.colMax(6));
    assertRealEquals("m colMax('mantis')", expected=1, actual=m.colMax('mantis'));

    assertIntEquals("m colArgMax(6)", expected=6, actual=m.colArgMax(6)[2]);
    assertIntEquals("m colArgMax('mantis')", expected=6, actual=m.colArgMax('mantis')[2]);

    assertIntEquals("m colArgMin(6)", expected=6, actual=m.colArgMin(6)[2]);
    assertIntEquals("m colArgMin('mantis')", expected=6, actual=m.colArgMin('mantis')[2]);

    this.tearDown(t);
  }

  proc testCosineDistance() {
    var t = this.setUp("Cosine Distance");
    var X = Matrix(
      [3.0, 0.2, 0.0, 0.7, 0.1],
      [0.2, 2.0, 0.3, 0.0, 0.0],
      [0.0, 0.3, 3.0, 0.9, 0.6],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.1, 0.0, 0.6, 0.0, 2.0]
    );

    var Y = Matrix(
      [3.0, 0.2, 0.0, 0.7, 0.1],
      [0.2, 2.0, 0.3, 0.0, 0.0],
      [0.0, 0.3, 3.0, 0.9, 0.6],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.1, 0.0, 0.6, 0.0, 2.0]
    );

    const xxTarget = Matrix(
      [0.0, 0.974619, 0.992338, 0.930778, 0.988007],
      [0.974619, 0.0, 0.964601, 0.981269, 0.988919],
      [0.992338, 0.964601, 0.0, 0.917246, 0.93309],
      [0.930778, 0.981269, 0.917246, 0.0, 0.973663],
      [0.988007, 0.988919, 0.93309, 0.973663, 0.0]);

    var V = cosineDistance(X);
    const a = V-xxTarget;
    const aa = norm(a);
    assertRealApproximates(msg="Cosine Distance(X) norm", expected=1.39884e-06, actual=aa);

    var V2 = cosineDistance(X,Y);
    const cosimXYtarget = Matrix(
      [0.895178, 0.974619, 0.992338, 0.930778, 0.930778, 0.988007],
      [0.974619, 0.757869, 0.964601, 0.981269, 0.981269, 0.988919],
      [0.992338, 0.964601, 0.902534, 0.917246, 0.917246, 0.93309],
      [0.930778, 0.981269, 0.917246, 0.811321, 0.811321, 0.973663],
      [0.988007, 0.988919, 0.93309,  0.973663, 0.973663, 0.771167]
      );
    const b = V2-cosimXYtarget;
    const bb = norm(b);
    assertRealApproximates(msg="Cosine Distance(X,Y) norm", expected=1.53583e-06, actual=bb);
    this.tearDown(t);
  }

  /*
   Does not assert anything, LabeledMatrix may be deprecated
   */
  proc testLabelMatrix() {
    var t = this.setUp("Label Matrix");
    var Y = Matrix(
      [3.0, 0.2, 0.0, 0.7, 0.1],
      [0.2, 2.0, 0.3, 0.0, 0.0],
      [0.0, 0.3, 3.0, 0.9, 0.6],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.7, 0.0, 0.9, 2.0, 0.0],
      [0.1, 0.0, 0.6, 0.0, 2.0]
    );
    var L = new LabelMatrix();
    L.fromMatrix(Y);
    assertRealEquals(msg="Label Matrix entry (1,4)", expected=0.7, actual=L.data(1,4));
    this.tearDown(t);
  }

  proc testECDF() {
    var t = this.setUp("ECDF");
    var x = [3,3,1,4];
    var ecdf = new ECDF(x);
    assertIntEquals("ECDF: Number of observations", expected=4, actual=ecdf.nobs);

    var y = [3.0, 55.0, 0.5, 1.5];
    var d = ecdf(y);
    assertArrayEquals("ECDF: Output", expected=[0.75,1.0,0.0,0.25], actual=d);
    this.tearDown(t);
  }

  proc testBiMap() {
    var t = this.setUp("BiMap");
    var bm = new BiMap();
    bm.add("bob");
    bm.add("chuck");
    bm.add("bob");
    bm.add('ethel', 78);
    bm.add('frank', 3);

    assertIntEquals("BIMAP redundant entry ignored", expected=4, actual=bm.keys.size);
    for k in bm.keys {
      assertStringEquals("BIMAP name is retrieved", k, bm.idx[bm.ids[k]]);
      assertIntEquals("BIMAP id is retrieved", bm.ids[k], bm.get(k));
    }
    assertIntEquals("BIMPA max is set", expected=78, actual=bm.max());

    var abm = new BiMap();
    abm.add("one",1);
    abm.add("two",2);
    var bbm = new BiMap();
    bbm.add("three",3);
    bbm.add("four",4);

    var cbm = abm.uni(bbm);
    assertIntEquals("BIMAP union size", expected=4, actual=cbm.keys.size);
    assertIntEquals("BIMAP union max", expected=4, actual=cbm.max());
    this.tearDown(t);
  }

  proc testViterbi() {
    var t = this.setUp("Viterbi");
    var obs = ["normal", "cold", "dizzy"];
    var states = ["health", "fever"];
    var initP = [0.6, 0.4];

    /* (Hidden) State transition matrix
             Healthy Fever
    Healthy  0.7     0.3
      Fever  0.4     0.6
     */
    var A: [1..2, 1..2] real = Matrix(
      [0.7, 0.3],
      [0.4, 0.6]
    );

    /*
     State emission probabilities
             Normal Cold Dizzy
    Healthy  0.5    0.4  0.1
      Fever  0.1    0.3  0.6
     */
     var B: [1..2, 1..3] real = Matrix(
       [0.5, 0.4, 0.1],
       [0.1, 0.3, 0.6]
    );
    var v = Viterbi(obs, states, initP, transitionProbabilities=A, emissionProbabilities=B);
    assertStringArrayEquals("Viterbi states output", expected=["health","health","fever"], actual=v);
    this.tearDown(t);
  }

  proc testLetters() {
    var t= this.setUp("Letters");
    var ltrs = letters(28);
    assertStringEquals("First in letters is A", expected="A", actual=ltrs[1]);
    assertStringEquals("28th in letters is BB", expected="BB", actual=ltrs[28]);

    var gn = gridNames(7);
    assertIntEquals("Grid names(7) has 49 entries", expected=49, actual=gn.size);
    assertStringEquals("Last grid name is 'G7'", expected="G7", actual=gn[49]);
    var gn2 = gridNames(i=3, j=5);
    assertRealEquals("gridNames(3,5) has 15 entries", expected=15, actual=gn2.size);
    assertStringEquals("Last grid name is 'C5'", expected="C5", actual=gn2[15]);
    this.tearDown(t);
  }

  proc testChoice() {
    var t = this.setUp("Choice");
    var x = [1,2,3,4,5,6,7,8,9,10];
    var y = choice(a=x,size=2);
    assertIntEquals("Can pick two choices, no replacement, no p", expected=2, actual=y.size);
    var z = choice(x,size=2,replace=true);
    assertIntEquals("Can pick two choices, with replacement, no p", expected=2, actual=z.size);

    var s = [1, 2, 3, 4, 5];
    // Note, this does not need to be normalized but it is easier to see
    var p = [0.5, 0.25, 0.13, 0.09, 0.03];
    var m = chooseMultinomial(a=s, replace=false, size=3, p=p);
    assertIntEquals("Multinomial w/o replace returns correct number of results", expected=3, actual=m.size);
    var n = chooseMultinomial(a=s, replace=true, size=3, p=p);
    assertIntEquals("Multinomial w/ replace returns correct number of results", expected=3, actual=n.size);
    var o = choice(a=s, replace=false, size=3, p=p);

    this.tearDown(t);
  }

  proc testRandomGenerators() {
    var t= this.setUp("Random Generators");
    var n: int = 100000;
    var x: [1..n] int;
    //for i in 1..n { x[i] = randInt(1,6); }
    for i in 1..n do x[i] = randInt(1,6);
    var mu = + reduce x;
    assertRealApproximates("Average of 100,000 randInt(1,6) is about 3.5"
      , expected=3.5, actual=mu:real/n:real, error=0.01);

    var threeD6 = for i in 1..n do nds(n=3, s=6);
    assertRealApproximates("Average of 100,000 three d6 is about 10.5"
      , expected=10.5, actual=(+reduce threeD6):real/n:real, error=0.012);

    // Check random reals
    var rrls: [1..n] real;
    for i in 1..n do rrls[i] = rand(a=2,b=7);
    assertRealApproximates("Average of 100,000 random(2,7) is about 4.5"
      , expected=4.5, actual=(+reduce rrls):real/n:real, error=0.01);

    var sq = seq(3,7);
    assertIntEquals("First element of sequence is 3", expected=3, actual=sq[1]);
    assertIntEquals("Last element of sequence is 7", expected=7, actual=sq[5]);

    var sq2 = seq(3,7, stride=5);
    assertIntEquals("First element of sequence(stride=5) is 3", expected=3, actual=sq2[1]);
    assertIntEquals("Last element of sequence(stride=5) is 7", expected=23, actual=sq2[5]);

    this.tearDown(t);
  }

  proc testPersistance() {

  }

  proc testMatrixMakers() {
    var t = this.setUp("Matrix makers");
    var n: int = 5,
        d: domain(2),
        X: [d] real;

    d =  {1..n, 1..n};
    const o = ones(X.domain);
    assertRealEquals(" Norm of o is 5", expected=5.0, actual=norm(o));

    const p = ones(X.domain, v=7.0);
    assertRealEquals(" Norm of p is 7", expected=35.0, actual=norm(p));

    this.tearDown(t);
  }

  proc testRowColSums() {
    var t = this.setUp("RowColSums");
    var X = Matrix( [5,2] ,[6,1] ,[-1,6] ,[1,1] );
    assertArrayEquals("rowSums of X", expected=[7.0, 7.0, 5.0, 2.0], actual=rowSums(X));
    assertArrayEquals("colSums of X", expected=[11.0, 10.0], actual=colSums(X));

    this.tearDown(t);
  }

  proc testConcat() {
    var t = this.setUp("Concat");
    var x: [1..3] real = [1.1, 2.2, 3.3],
        y: [9..11] real = [9.9, 10.10, 11.11];

    var z = concat(x,y);
    var zz: [1..6] real = [1.1, 2.2, 3.3, 9.9, 10.10, 11.11];
    assertIntEquals(msg="Concatenated real array is 6 wide", expected=6, actual=z.size);
    assertArrayEquals(msg="Check concat gives correct array", expected=zz, actual=z);

    this.tearDown(t);
  }

  proc testNamedDenseMatrix() {
    var t = this.setUp("Named Dense Matrix");
    var dom: domain(2) = {1..7,1..6};
    var W: [dom] real;
    fillRandom(W);
    var nm = new NamedMatrix(X=W);
    //writeln(nm.X);

    vn.push_back("star lord");
    vn.push_back("gamora");
    vn.push_back("groot");
    vn.push_back("drax");
    vn.push_back("rocket");
    vn.push_back("mantis");
    vn.push_back("yondu");
    vn.push_back("nebula");

    SD += (1,2); X[1,2] = 1;
    SD += (1,3); X[1,3] = 1;
    SD += (1,4); X[1,4] = 1;
    SD += (2,2); X[2,2] = 1;
    SD += (2,4); X[2,4] = 1;
    SD += (3,4); X[3,4] = 1;
    SD += (4,5); X[4,5] = 1;
    SD += (5,6); X[5,6] = 1;
    SD += (6,7); X[6,7] = 1;
    SD += (6,8); X[6,8] = 1;
    SD += (7,8); X[7,8] = 1;

    //writeln(X);

    var nm2 = new NamedMatrix(X=X);
    //writeln(nm2.X);

    this.tearDown(t);
  }

  proc run() {
    super.run();
    testNamedDenseMatrix();
    testRowColSums();
    testMatrixMakers();
    testMatrixOperators();
//    tropicalTesting();
    testIndexSort();
    testNamedMatrix();
    testSetRowNames();
    testSetColNames();
//    testNamedMatrixInitWithNames();
    testSetByName();
      testArgMaxMin();
//    testCosineDistance();
//    testLabelMatrix();
    testECDF();
    testBiMap();
    testViterbi();
    testLetters();
    testChoice();
    testRandomGenerators();
    testConcat();
    return 0;
  }
}

proc main(args: [] string) : int {
  var t = new NumSuchTest(verbose=false);
  var ret = t.run();
  t.report();
  return ret;
}


Updating blockLU test to use the swap operator.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@9859 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// Block LU with pivoting.
// Magic square of dimension 10, is used as test matrix A.

// Using array alias operators to implement LU block algorithm
// with pivoting.
// In this version, A1D is a range, not a 1D domain.  The iterator,
// IterateByBlocks returns ranges, not subdomains of A1D.  Temporary
// ranges, slice1 and slice2 are used to define what should be
// subdomains sliced by indefinite ranges.
//
// The maxIndex reduction routine and the swap operator are not
// implemented yet and need to be coded.  Since passing array aliases
// with slicing to functions doesn't seem to be fully supported yet,
// the pivot calculation and the swapping of pivot rows is coded inline
// in the algorithm.

def main() {
  config const inputfile = "Amagic10.dat";
  var Adat = file(inputfile,path='./',mode='r');
  Adat.open;

  const n = readSize(Adat);
  config var blk = readBlk(Adat);

  // The blocksize cannot exceed the size of n.  Reset to n if this happens.
  blk = min(blk,n);

  var A1D = 1..n;
  const A2D = [A1D,A1D];
  var A: [A2D] real;
  initA(A,Adat);
  Adat.close;

  var slice0, slice1, slice2: range;
  var piv: [A1D] int;
  var ind, temp:int;
  var rtemp: real;

  [i in A1D] piv(i) = i;

  writeln("Unfactored Matrix:");
  writeln(A);
  writeln();

  blockLU(A, piv, blk);

  writeln("Factored Matrix:");
  writeln(A);
  writeln();

  writeln("Pivot Vector:");
  writeln(piv);
}

def blockLU(x: [?D], piv, blk) where (D.rank != 2) {
  compilerError("blockLU factors a matrix.  The first
input parameter to blockLU must be a two-dimensional array.");
}

def blockLU(A: [?D], piv: [D(1)], blk) where (D.rank == 2){
  var slice0, slice1, slice2: range;
  var ind: index(D);

  if ((D(1).low != D(2).low) | (D(1).high != D(2).high)) then
    halt("error: blockLU requires square matrix with same dimensions");

  var A1D = D(1);
  for (UnfactoredInds,CurrentBlockInds,TrailingBlockInds)
    in IterateByBlocks(A1D,blk) {

    var A1 => A[UnfactoredInds,CurrentBlockInds];
    var A2 => A[UnfactoredInds,TrailingBlockInds];
    var A12 => A[CurrentBlockInds,TrailingBlockInds];
    var A22 => A[TrailingBlockInds,TrailingBlockInds];

// LU factorization of A1
    for k in CurrentBlockInds {
//    temporaries used instead of subdomains with indefinite ranges.
      slice0 = k..UnfactoredInds.high;
      slice1 = k+1..UnfactoredInds.high;
      slice2 = k+1..CurrentBlockInds.high;

      ind = amaxIndex(A(slice0,k..k));

      if (ind(1) != k) {
        piv(k) <=> piv(ind(1));
        A(k..k,A1D) <=> A(ind(1)..ind(1),A1D);
      }

      if (A1(k,k) != 0.0) {
        forall i in slice1 {
          A1(i,k) = A1(i,k)/A1(k,k);
        }
        forall (i,j) in [slice1,slice2] {
          A1(i,j) -= A1(i,k)*A1(k,j);
        }
      }
      else halt("zero pivot encountered");
    }

// Update of A12.
    forall j in TrailingBlockInds {
      for k in CurrentBlockInds {
//    temporary range used instead of subdomain with indefinite range
        slice2 = k+1..CurrentBlockInds.high;
        forall i in slice2 {
          A12(i,j) -= A1(i,k)*A12(k,j);
        }
      }
    }
// Update of A22 -= A12*A21.
    forall (i,j) in [TrailingBlockInds, TrailingBlockInds] {
      for k in CurrentBlockInds {
        A22(i,j) -= A1(i,k)*A12(k,j);
      }
    }
  }
}

iterator IterateByBlocks(D:range,blksize) {
  var n = D.length;
  var lo,hi: int;

  for i in D by blksize {
    lo = i;
    hi = min(i + blksize-1,n);
    yield (lo..n,i..hi,hi+1..n);
  }
}

def amaxIndex(A:[?D]) {
   var (_,ind) = maxloc reduce (abs(A),D);
   return ind;
}

def readSize(Adat) {
  var n: int;

  Adat.read(n);
  return n;
}

def readBlk(Adat) {
  var blk: int;

  Adat.read(blk);
  return blk;
}

def initA(A,Adat){

  for ij in A.domain {
    Adat.read(A(ij));
  }
}

// Block LU with pivoting.
// Magic square of dimension 10, is used as test matrix A.

// Using array alias operators to implement LU block algorithm
// with pivoting.
// In this version, A1D is a range, not a 1D domain.  The iterator,
// IterateByBlocks returns ranges, not subdomains of A1D.  Temporary
// ranges, slice1 and slice2 are used to define what should be
// subdomains sliced by indefinite ranges.
//
// The maxIndex reduction routine and the swap operator are not
// implemented yet and need to be coded.  Since passing array aliases
// with slicing to functions doesn't seem to be fully supported yet,
// the pivot calculation and the swapping of pivot rows is coded inline
// in the algorithm.

def main() {
  config const inputfile = "Amagic10.dat";
  var Adat = file(inputfile,path='./',mode='r');
  Adat.open;

  const n = readSize(Adat);
  config var blk = readBlk(Adat);

  // The blocksize cannot exceed the size of n.  Reset to n if this happens.
  blk = min(blk,n);

  var A1D = 1..n;
  const A2D = [A1D,A1D];
  var A: [A2D] real;
  initA(A,Adat);
  Adat.close;

  var slice0, slice1, slice2: range;
  var piv: [A1D] int;
  var ind, temp:int;
  var rtemp: real;

  [i in A1D] piv(i) = i;

  writeln("Unfactored Matrix:");
  writeln(A);
  writeln();

  blockLU(A, piv, blk);

  writeln("Factored Matrix:");
  writeln(A);
  writeln();

  writeln("Pivot Vector:");
  writeln(piv);
}

def blockLU(A: [?D], piv: [D(1)], blk) {
  var slice0, slice1, slice2: range;
  var rtemp: A.eltType;

  if (D.rank != 2) then
    halt("error: blockLU requires 2D array");

  if ((D(1).low != D(2).low) | (D(1).high != D(2).high)) then
    halt("error: blockLU requires square matrix with same dimensions");

  var A1D = D(1);
  for (UnfactoredInds,CurrentBlockInds,TrailingBlockInds)
    in IterateByBlocks(A1D,blk) {

    var A1 => A[UnfactoredInds,CurrentBlockInds];
    var A2 => A[UnfactoredInds,TrailingBlockInds];
    var A12 => A[CurrentBlockInds,TrailingBlockInds];
    var A22 => A[TrailingBlockInds,TrailingBlockInds];

// LU factorization of A1
    for k in CurrentBlockInds {
//    temporaries used instead of subdomains with indefinite ranges.
      slice0 = k..UnfactoredInds.high;
      slice1 = k+1..UnfactoredInds.high;
      slice2 = k+1..CurrentBlockInds.high;

//    ind = maxIndex reduce (A1(UnfactoredInds(k..),k));
      var (ind,_) = amaxIndex(A(slice0,k..k));

      if (ind != k) {
//        piv(k) <==> piv(ind);
//        A(k,..) <==> A(ind,..);
        var temp = piv(k);
        piv(k) = piv(ind);
        piv(ind) = temp;
        for i in A1D {
          rtemp = A(k,i);
          A(k,i) = A(ind,i);
          A(ind,i) = rtemp;
        }
      }
      if (A1(k,k) != 0.0) {
        forall i in slice1 {
          A1(i,k) = A1(i,k)/A1(k,k);
        }
        forall (i,j) in [slice1,slice2] {
          A1(i,j) -= A1(i,k)*A1(k,j);
        }
      }
      else halt("zero pivot encountered");
    }

// Update of A12.
    forall j in TrailingBlockInds {
      for k in CurrentBlockInds {
//    temporary range used instead of subdomain with indefinite range
        slice2 = k+1..CurrentBlockInds.high;
        forall i in slice2 {
          A12(i,j) -= A1(i,k)*A12(k,j);
        }
      }
    }
// Update of A22 -= A12*A21.
    forall (i,j) in [TrailingBlockInds, TrailingBlockInds] {
      for k in CurrentBlockInds {
        A22(i,j) -= A1(i,k)*A12(k,j);
      }
    }
  }
}

iterator IterateByBlocks(D:range,blksize) {
  var n = D.length;
  var lo,hi: int;

  for i in D by blksize {
    lo = i;
    hi = min(i + blksize-1,n);
    yield (lo..n,i..hi,hi+1..n);
  }
}

def amaxIndex(A:[?D]) {
   var (_,ind) = maxloc reduce (abs(A),D);
   return ind;
}

def readSize(Adat) {
  var n: int;

  Adat.read(n);
  return n;
}

def readBlk(Adat) {
  var blk: int;

  Adat.read(blk);
  return blk;
}

def initA(A,Adat){

  for ij in A.domain {
    Adat.read(A(ij));
  }
}


Quick fix to GASNet associative domains/arrays

This is a workaround to the seg fault issue that
my associative domain/array check-in of the other
day caused.  In Steve's absence, David determined
that it was due to a reference to a global
variable within a field's initializer (and therefore
the default constructor's default argument list),
so I worked around it by moving the global variable
in question (the _ps array) into the classes for
the domain and array.  Wasted space, but it makes
those pesky seg faults go away.

In the meantime, David had some small tests of
the behavior written up and will check those in
as bug futures under the --no-local flag.

Seems like a possible fix might be to move some
of those more complex initializers (or ones that
refer to more global state) into their own
functions as part of the normalization process?
But that's said with only the vaguest grasp on
the issue.

Meanwhile, I'm continuing to debug my complete
rewrite of the associative domain/array class
and will check that in once I get a few more
bugs out.

Note: this check-in does not fix the opaque
example test regression.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13611 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use List;

param _MIN_SIZE = 0;
param _MAX_SIZE = 26;

param _EMPTY   = -1;
param _DELETED = -2;
param _NOPLACE = -3;

record _ind_data_t {
  type itype;
  var  data: itype;
  var  valid: bool;
}

class SingleLocaleAssociativeDomain: BaseDomain {
  param rank: int;
  type idxType;
  var _arrs2: list(BaseArray);    // WAW: unfortunately redundant list

  const _ps : [0..26] int = (23, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289,
                             24593, 49157, 98317, 196613, 393241, 786433,
                             1572869, 3145739, 6291469, 12582917, 25165843,
                             50331653, 100663319, 201326611, 402653189,
                             805306457, 1610612741);

  var num_inds: int;
  var size: int = 0;
  var tableDom = [0.._ps(0)-1];
  var table: [tableDom] int = _EMPTY;
  var indsDom = [0.._ps(0)/2];
  var inds: [indsDom] _ind_data_t(idxType);
  var free_inds: _stack(int) = new _stack(int);

  def getIndices()
    return this; // stopgap measure given old implementation

  def setIndices(b: SingleLocaleAssociativeDomain) {
    oldAssignHelper(this, b);
  }

  def buildEmptyDomain()
    return new SingleLocaleAssociativeDomain(rank=rank,idxType=idxType);

  def these() {
    var i = 0;
    var numFound = 0;
    while numFound < num_inds {
      if (inds(i).valid) {
        numFound += 1;
        yield inds(i).data;
      }
      i += 1;
    }
  }

  // internal routines

  def _half() {
    param debug = false;
    if (size > _MIN_SIZE) {
      var new_len = _ps(size-1);

      var new_tableDom = [0..new_len-1];
      var new_table: [new_tableDom] int = _EMPTY;

      var new_indsDom = [0..new_len/2-1];
      var new_inds: [new_indsDom] _ind_data_t(idxType);
      var old_map: [0..num_inds-1] int;
      var inds_count = 0;
      var inds_pos = 0;
      while inds_count<num_inds {
        var ind = inds(inds_pos);
        if (ind.valid) {
          old_map(inds_count) = table( _map(ind.data));
          new_inds(inds_count) = ind;
          new_table( _map(ind.data, false, new_table, new_inds)) = inds_count;
          inds_count += 1;
        }
        inds_pos += 1;
      }

      for ia in _arrs2 {
        ia._resize( new_len/2, old_map);
      }

      if debug then writeln("Assigning tableDom");
      tableDom = new_tableDom;
      if debug then writeln("Done");
      table = new_table;
      if debug then writeln("Assigning indsDom");
      indsDom = new_indsDom;
      if debug then writeln("Done");
      inds = new_inds;
      size -= 1;
      free_inds.empty();
    }
  }

  def _double() {
    param debug = false;
    if (size < _MAX_SIZE) {
      var new_len = _ps(size+1);

      var new_tableDom = [0..new_len-1];
      var new_table: [new_tableDom] int = _EMPTY;
      var new_indsDom = [0..new_len/2-1];
      var new_inds: [new_indsDom] _ind_data_t(idxType);
      var old_map: [inds.domain] int; // note: avoiding using indsDom here
                                      // because it causes all old_map arrays
                                      // to be reallocated whenever indsDom is
                                      // assigned ... :P  (BLC)
      for inds_pos in indsDom {
        var ind = inds(inds_pos);
        if (ind.valid) {                 // should be dense
          old_map(inds_pos) = table( _map(ind.data));
          new_inds(inds_pos) = ind;
          new_table( _map( ind.data, false, new_table, new_inds)) = inds_pos;
        } else {
          halt( "doubling without being full");
        }
      }

      for ia in _arrs2 {
        ia._resize( new_len/2, old_map);
      }

      if debug then writeln("Assigning tableDom (double)");
      tableDom = new_tableDom;
      if debug then writeln("Done");
      table = new_table;
      if debug then writeln("Assigning indsDom (double)");
      indsDom = new_indsDom;
      if debug then writeln("Done");
      inds = new_inds;
      size += 1;
      free_inds.empty();
    } else {
      halt( "cannot double associative domain");
    }
  }

  def _map(ind: idxType, deletedOK: bool = false,
           itable, iinds): int { // was: itable = table, iinds = inds; see below
    param debug = false;
    if debug then writeln("itable.domain = ", itable.domain);
    const base = _associative_hash(ind);
    for probe in itable.domain {
      var i = ((base + probe**2) % itable.numElements):int(32);
      if debug then writeln("Trying i = ", i);
      var table_i = itable(i);
      if debug then writeln("table_i = ", table_i);
      select (table_i) {
        when _EMPTY {
          if debug then writeln("EMPTY");
          return i;
        }
        when _DELETED {
          if debug then writeln("DELETED");
          if deletedOK then
            return i;
          if debug then writeln("...but not OK");
        }
        otherwise {
          if debug then writeln("iinds(table_i).data = ", iinds(table_i).data);
          if iinds(table_i).data==ind then
            return i;
        }
      }
    }
    return _NOPLACE;
  }

  // This version of _map is redundant with the previous, but
  // put here in order to avoid the array copies that are currently
  // inserted when default arguments of type array are used.
  def _map(ind: idxType, deletedOK: bool = false): int {
    param debug = false;
    if debug then writeln("table.domain = ", table.domain);
    const base = _associative_hash(ind);
    for probe in table.domain {
      var i = ((base + probe**2) % table.numElements):int(32);
      if debug then writeln("Trying i = ", i);
      var table_i = table(i);
      if debug then writeln("table_i = ", table_i);
      select (table_i) {
        when _EMPTY {
          if debug then writeln("EMPTY");
          return i;
        }
        when _DELETED {
          if debug then writeln("DELETED");
          if deletedOK then
            return i;
          if debug then writeln("...but not OK");
        }
        otherwise {
          if debug then writeln("inds(table_i).data = ", inds(table_i).data);
          if inds(table_i).data==ind then
            return i;
        }
      }
    }
    return _NOPLACE;
  }

  def _get_index(ind : idxType) {
    var i = _map(ind);
    if (table(i) < 0) then
      halt("array index out of bounds: ", ind);
    if (i >= 0) then
      return table(i);
    else
      halt( "index not found: ", ind);
    // will never get here, but to avoid compiler warnings:
    return 0;
  }

  def buildArray(type eltType) {
    var ia = new SingleLocaleAssociativeArray(eltType, idxType, dom=this);
    _arrs2.append(ia);
    return ia;
  }

  // public routines

  def clear() {
    // BLC: implemented this for simplicity, not performance
    while (numIndices != 0) do
      removeOne();

    def removeOne() {
      // Since it's not safe to iterate over a hash table while you're
      // removing stuff from it, we only do one iteration at a time
      for i in these() {
        remove(i);
        return;
      }
    }
  }

  def add(ind : idxType) {
    var hash = _map(ind, deletedOK = true);
    var ind_pos = table(hash);
    if (ind_pos==_EMPTY || ind_pos==_DELETED) {

      if (free_inds.length > 0) {      // recycle
        ind_pos = free_inds.pop();
      } else {
        if (num_inds == inds.numElements) {
          _double();
        }
        ind_pos = num_inds;
        hash = _map(ind, deletedOK = true);
      }
      table(hash) = ind_pos;
      inds(ind_pos).data = ind;
      inds(ind_pos).valid = true;
      num_inds += 1;
    } // else, already in domain
  }

  def remove( ind: idxType) {
    var ind_pos = _map(ind);
    if (ind_pos >= 0) {
      var table_i = table(ind_pos);
      if (table_i >= 0) {
        inds(table_i).valid = false;
        num_inds -= 1;
        free_inds.push( table_i);
        table(ind_pos) = _DELETED;

        for ia in _arrs2 do

          ia._purge( table_i);
      } else {
        halt("index not in domain: ", ind);
      }
    }

    if (size > _MIN_SIZE) then
      if (num_inds < _ps(size-1)/2) {  // downsize?
        _half();
      }
  }

  def member( ind: idxType) {
    const mapind = _map(ind);
    if (mapind >= 0) {
      var ind_pos = table(mapind);
      return (ind_pos >= 0);
    }
    return false;
  }

  def numIndices {
    return num_inds;
  }
}

def oldAssignHelper(a: SingleLocaleAssociativeDomain, b: SingleLocaleAssociativeDomain) {
  var indices: list(a.idxType);
  var inds_count = 0;
  var inds_pos = 0;
  var total_inds = a.num_inds;
  while inds_count<total_inds {
    var ind = a.inds( inds_pos);
    if (ind.valid) {
      indices.append(ind.data);
      inds_count += 1;
    }
    inds_pos += 1;
  }

  for ind in indices {
    if (!b.member(ind)) {
      a.remove( ind);
    }
  }

  inds_count = 0;
  inds_pos = 0;
  total_inds = b.num_inds;
  while inds_count<total_inds {
    var ind = b.inds( inds_pos);
    if (ind.valid) {
      if (!a.member(ind.data)) {
        a.add( ind.data);
      }
      inds_count += 1;
    }
    inds_pos += 1;
  }
}

def SingleLocaleAssociativeDomain.writeThis(f: Writer) {
  f.write("[");
  var inds_count = 0;
  var inds_pos = 0;
  while inds_count<num_inds {
    var ind = inds( inds_pos);
    if (ind.valid) {
      f.write(ind.data);
      inds_count += 1;
      if (inds_count<num_inds) then f.write(", ");
    }
    inds_pos += 1;
  }
  f.write("]");
}

class SingleLocaleAssociativeArray: BaseArray {
  type eltType;
  type idxType;
  var dom : SingleLocaleAssociativeDomain(rank=1,idxType=idxType);

  const _ps : [0..26] int = (23, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289,
                             24593, 49157, 98317, 196613, 393241, 786433,
                             1572869, 3145739, 6291469, 12582917, 25165843,
                             50331653, 100663319, 201326611, 402653189,
                             805306457, 1610612741);

  var dataDom = [0.._ps(0)/2];
  var data : [dataDom] eltType;

  // This method is unsatisfactory -- see bradc's commit entries of
  // 01/02/08 around 14:30 for details
  def _purge( ind: int) {
    var d: eltType;
    data( ind) = d;
  }

  def _resize( length: int, old_map) {
    var new_dataDom = [0..length-1];
    var new_data: [new_dataDom] eltType;
    for i in old_map.domain {
      new_data(i) = data(old_map(i));
    }
    dataDom = new_dataDom;
    data = new_data;
  }

  def this(ind : idxType) var : eltType
    return data(dom._get_index(ind));

  def these() var {
    var i = 0;
    var numFound = 0;
    while numFound < dom.num_inds {
      if (dom.inds(i).valid) {
        numFound += 1;
        yield data(i);
      }
      i += 1;
    }
  }

  // WAW: hack due to domain-array interface. The array is not reallocated
  // here. Values of indices not in the new domain are reset. The
  // domain assignment will perform the reallocation of each array that
  // maps to it.
  def reallocate(d: _domain) {
    if (d._value.type == dom.type) {
      var td = d._value;
      var inds_count = 0;
      var inds_pos = 0;
      while (inds_pos < dom.num_inds) && (inds_count < dom.inds.numElements) {
	var ind = dom.inds(inds_count);
	if ind.valid {
	  var i = td._map( ind.data);
	  if (i < 0) then {
	    _purge( dom.table(dom._map( ind.data)));
	  }
	  inds_pos += 1;
	}
	inds_count += 1;
      }
    }
  }

  def numElements {
    return dom.numIndices;
  }

  def tupleInit(b: _tuple) {
    for param i in 1..b.size do
      data(i-1) = b(i);
  }
}

def SingleLocaleAssociativeArray.writeThis(f: Writer) {
  var inds_count = 0;
  var inds_pos = 0;
  while inds_count<dom.num_inds {
    var ind = dom.inds(inds_pos);
    if (ind.valid) {
      f.write(data(inds_pos));
      inds_count += 1;
      if (inds_count < dom.num_inds) then f.write(" ");
    }
    inds_pos += 1;
  }
}

// Thomas Wang's 64b mix function from http://www.concentric.net/~Ttwang/tech/inthash.htm
def _gen_key(i: int(64)): int(64) {
  var key = i;
  key += ~(key << 32);
  key = key ^ (key >> 22);
  key += ~(key << 13);
  key = key ^ (key >> 8);
  key += (key << 3);
  key = key ^ (key >> 15);
  key += ~(key << 27);
  key = key ^ (key >> 31);
  return key & 0x7fffffffffffffff;  // YAH, make non-negative
}

pragma "inline"
def _associative_hash(b: bool): int(64) {
  if (b)
    return 17;
  else
    return 31;
}

pragma "inline"
def _associative_hash(i: int(64)): int(64) {
  return _gen_key(i);
}

pragma "inline"
def _associative_hash(u: uint(64)): int(64) {
  return _gen_key(u:int(64));
}

pragma "inline"
def _associative_hash(f: real): int(64) {
  return _gen_key(__primitive( "real2int", f));
}

pragma "inline"
def _associative_hash(c: complex): int(64) {
  return _gen_key(c.re:int ^ c.im:int);
}

// Use djb2 (Dan Bernstein in comp.lang.c.
pragma "inline"
def _associative_hash(x : string): int(64) {
  var hash: int(64) = 0;
  for c in 1..length(x) {
    hash = ((hash << 5) + hash) ^ ascii(x.substring(c));
  }
  return _gen_key(hash);
}

pragma "inline"
def _associative_hash(o: object): int(64) {
  return _gen_key(__primitive( "object2int", o));
}

use List;

param _MIN_SIZE = 0;
param _MAX_SIZE = 26;

var _ps : [0..26] int = (23, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289,
                         24593, 49157, 98317, 196613, 393241, 786433, 1572869,
                         3145739, 6291469, 12582917, 25165843, 50331653,
                         100663319, 201326611, 402653189, 805306457,
                         1610612741);

param _EMPTY   = -1;
param _DELETED = -2;
param _NOPLACE = -3;

record _ind_data_t {
  type itype;
  var  data: itype;
  var  valid: bool;
}

class SingleLocaleAssociativeDomain: BaseDomain {
  param rank: int;
  type idxType;
  var _arrs2: list(BaseArray);    // WAW: unfortunately redundant list
  var num_inds: int;
  var size: int = 0;
  var tableDom = [0.._ps(0)-1];
  var table: [tableDom] int = _EMPTY;
  var indsDom = [0.._ps(0)/2];
  var inds: [indsDom] _ind_data_t(idxType);
  var free_inds: _stack(int) = new _stack(int);

  def getIndices()
    return this; // stopgap measure given old implementation

  def setIndices(b: SingleLocaleAssociativeDomain) {
    oldAssignHelper(this, b);
  }

  def buildEmptyDomain()
    return new SingleLocaleAssociativeDomain(rank=rank,idxType=idxType);

  def these() {
    var i = 0;
    var numFound = 0;
    while numFound < num_inds {
      if (inds(i).valid) {
        numFound += 1;
        yield inds(i).data;
      }
      i += 1;
    }
  }

  // internal routines

  def _half() {
    param debug = false;
    if (size > _MIN_SIZE) {
      var new_len = _ps(size-1);

      var new_tableDom = [0..new_len-1];
      var new_table: [new_tableDom] int = _EMPTY;

      var new_indsDom = [0..new_len/2-1];
      var new_inds: [new_indsDom] _ind_data_t(idxType);
      var old_map: [0..num_inds-1] int;
      var inds_count = 0;
      var inds_pos = 0;
      while inds_count<num_inds {
        var ind = inds(inds_pos);
        if (ind.valid) {
          old_map(inds_count) = table( _map(ind.data));
          new_inds(inds_count) = ind;
          new_table( _map(ind.data, false, new_table, new_inds)) = inds_count;
          inds_count += 1;
        }
        inds_pos += 1;
      }

      for ia in _arrs2 {
        ia._resize( new_len/2, old_map);
      }

      if debug then writeln("Assigning tableDom");
      tableDom = new_tableDom;
      if debug then writeln("Done");
      table = new_table;
      if debug then writeln("Assigning indsDom");
      indsDom = new_indsDom;
      if debug then writeln("Done");
      inds = new_inds;
      size -= 1;
      free_inds.empty();
    }
  }

  def _double() {
    param debug = false;
    if (size < _MAX_SIZE) {
      var new_len = _ps(size+1);

      var new_tableDom = [0..new_len-1];
      var new_table: [new_tableDom] int = _EMPTY;
      var new_indsDom = [0..new_len/2-1];
      var new_inds: [new_indsDom] _ind_data_t(idxType);
      var old_map: [inds.domain] int; // note: avoiding using indsDom here
                                      // because it causes all old_map arrays
                                      // to be reallocated whenever indsDom is
                                      // assigned ... :P  (BLC)
      for inds_pos in indsDom {
        var ind = inds(inds_pos);
        if (ind.valid) {                 // should be dense
          old_map(inds_pos) = table( _map(ind.data));
          new_inds(inds_pos) = ind;
          new_table( _map( ind.data, false, new_table, new_inds)) = inds_pos;
        } else {
          halt( "doubling without being full");
        }
      }

      for ia in _arrs2 {
        ia._resize( new_len/2, old_map);
      }

      if debug then writeln("Assigning tableDom (double)");
      tableDom = new_tableDom;
      if debug then writeln("Done");
      table = new_table;
      if debug then writeln("Assigning indsDom (double)");
      indsDom = new_indsDom;
      if debug then writeln("Done");
      inds = new_inds;
      size += 1;
      free_inds.empty();
    } else {
      halt( "cannot double associative domain");
    }
  }

  def _map(ind: idxType, deletedOK: bool = false,
           itable, iinds): int { // was: itable = table, iinds = inds; see below
    param debug = false;
    if debug then writeln("itable.domain = ", itable.domain);
    const base = _associative_hash(ind);
    for probe in itable.domain {
      var i = ((base + probe**2) % itable.numElements):int(32);
      if debug then writeln("Trying i = ", i);
      var table_i = itable(i);
      if debug then writeln("table_i = ", table_i);
      select (table_i) {
        when _EMPTY {
          if debug then writeln("EMPTY");
          return i;
        }
        when _DELETED {
          if debug then writeln("DELETED");
          if deletedOK then
            return i;
          if debug then writeln("...but not OK");
        }
        otherwise {
          if debug then writeln("iinds(table_i).data = ", iinds(table_i).data);
          if iinds(table_i).data==ind then
            return i;
        }
      }
    }
    return _NOPLACE;
  }

  // This version of _map is redundant with the previous, but
  // put here in order to avoid the array copies that are currently
  // inserted when default arguments of type array are used.
  def _map(ind: idxType, deletedOK: bool = false): int {
    param debug = false;
    if debug then writeln("table.domain = ", table.domain);
    const base = _associative_hash(ind);
    for probe in table.domain {
      var i = ((base + probe**2) % table.numElements):int(32);
      if debug then writeln("Trying i = ", i);
      var table_i = table(i);
      if debug then writeln("table_i = ", table_i);
      select (table_i) {
        when _EMPTY {
          if debug then writeln("EMPTY");
          return i;
        }
        when _DELETED {
          if debug then writeln("DELETED");
          if deletedOK then
            return i;
          if debug then writeln("...but not OK");
        }
        otherwise {
          if debug then writeln("inds(table_i).data = ", inds(table_i).data);
          if inds(table_i).data==ind then
            return i;
        }
      }
    }
    return _NOPLACE;
  }

  def _get_index(ind : idxType) {
    var i = _map(ind);
    if (table(i) < 0) then
      halt("array index out of bounds: ", ind);
    if (i >= 0) then
      return table(i);
    else
      halt( "index not found: ", ind);
    // will never get here, but to avoid compiler warnings:
    return 0;
  }

  def buildArray(type eltType) {
    var ia = new SingleLocaleAssociativeArray(eltType, idxType, dom=this);
    _arrs2.append(ia);
    return ia;
  }

  // public routines

  def clear() {
    // BLC: implemented this for simplicity, not performance
    while (numIndices != 0) do
      removeOne();

    def removeOne() {
      // Since it's not safe to iterate over a hash table while you're
      // removing stuff from it, we only do one iteration at a time
      for i in these() {
        remove(i);
        return;
      }
    }
  }

  def add(ind : idxType) {
    var hash = _map(ind, deletedOK = true);
    var ind_pos = table(hash);
    if (ind_pos==_EMPTY || ind_pos==_DELETED) {

      if (free_inds.length > 0) {      // recycle
        ind_pos = free_inds.pop();
      } else {
        if (num_inds == inds.numElements) {
          _double();
        }
        ind_pos = num_inds;
        hash = _map(ind, deletedOK = true);
      }
      table(hash) = ind_pos;
      inds(ind_pos).data = ind;
      inds(ind_pos).valid = true;
      num_inds += 1;
    } // else, already in domain
  }

  def remove( ind: idxType) {
    var ind_pos = _map(ind);
    if (ind_pos >= 0) {
      var table_i = table(ind_pos);
      if (table_i >= 0) {
        inds(table_i).valid = false;
        num_inds -= 1;
        free_inds.push( table_i);
        table(ind_pos) = _DELETED;

        for ia in _arrs2 do

          ia._purge( table_i);
      } else {
        halt("index not in domain: ", ind);
      }
    }

    if (size > _MIN_SIZE) then
      if (num_inds < _ps(size-1)/2) {  // downsize?
        _half();
      }
  }

  def member( ind: idxType) {
    const mapind = _map(ind);
    if (mapind >= 0) {
      var ind_pos = table(mapind);
      return (ind_pos >= 0);
    }
    return false;
  }

  def numIndices {
    return num_inds;
  }
}

def oldAssignHelper(a: SingleLocaleAssociativeDomain, b: SingleLocaleAssociativeDomain) {
  var indices: list(a.idxType);
  var inds_count = 0;
  var inds_pos = 0;
  var total_inds = a.num_inds;
  while inds_count<total_inds {
    var ind = a.inds( inds_pos);
    if (ind.valid) {
      indices.append(ind.data);
      inds_count += 1;
    }
    inds_pos += 1;
  }

  for ind in indices {
    if (!b.member(ind)) {
      a.remove( ind);
    }
  }

  inds_count = 0;
  inds_pos = 0;
  total_inds = b.num_inds;
  while inds_count<total_inds {
    var ind = b.inds( inds_pos);
    if (ind.valid) {
      if (!a.member(ind.data)) {
        a.add( ind.data);
      }
      inds_count += 1;
    }
    inds_pos += 1;
  }
}

def SingleLocaleAssociativeDomain.writeThis(f: Writer) {
  f.write("[");
  var inds_count = 0;
  var inds_pos = 0;
  while inds_count<num_inds {
    var ind = inds( inds_pos);
    if (ind.valid) {
      f.write(ind.data);
      inds_count += 1;
      if (inds_count<num_inds) then f.write(", ");
    }
    inds_pos += 1;
  }
  f.write("]");
}

class SingleLocaleAssociativeArray: BaseArray {
  type eltType;
  type idxType;
  var dom : SingleLocaleAssociativeDomain(rank=1,idxType=idxType);
  var dataDom = [0.._ps(0)/2];
  var data : [dataDom] eltType;

  // This method is unsatisfactory -- see bradc's commit entries of
  // 01/02/08 around 14:30 for details
  def _purge( ind: int) {
    var d: eltType;
    data( ind) = d;
  }

  def _resize( length: int, old_map) {
    var new_dataDom = [0..length-1];
    var new_data: [new_dataDom] eltType;
    for i in old_map.domain {
      new_data(i) = data(old_map(i));
    }
    dataDom = new_dataDom;
    data = new_data;
  }

  def this(ind : idxType) var : eltType
    return data(dom._get_index(ind));

  def these() var {
    var i = 0;
    var numFound = 0;
    while numFound < dom.num_inds {
      if (dom.inds(i).valid) {
        numFound += 1;
        yield data(i);
      }
      i += 1;
    }
  }

  // WAW: hack due to domain-array interface. The array is not reallocated
  // here. Values of indices not in the new domain are reset. The
  // domain assignment will perform the reallocation of each array that
  // maps to it.
  def reallocate(d: _domain) {
    if (d._value.type == dom.type) {
      var td = d._value;
      var inds_count = 0;
      var inds_pos = 0;
      while (inds_pos < dom.num_inds) && (inds_count < dom.inds.numElements) {
	var ind = dom.inds(inds_count);
	if ind.valid {
	  var i = td._map( ind.data);
	  if (i < 0) then {
	    _purge( dom.table(dom._map( ind.data)));
	  }
	  inds_pos += 1;
	}
	inds_count += 1;
      }
    }
  }

  def numElements {
    return dom.numIndices;
  }

  def tupleInit(b: _tuple) {
    for param i in 1..b.size do
      data(i-1) = b(i);
  }
}

def SingleLocaleAssociativeArray.writeThis(f: Writer) {
  var inds_count = 0;
  var inds_pos = 0;
  while inds_count<dom.num_inds {
    var ind = dom.inds(inds_pos);
    if (ind.valid) {
      f.write(data(inds_pos));
      inds_count += 1;
      if (inds_count < dom.num_inds) then f.write(" ");
    }
    inds_pos += 1;
  }
}

// Thomas Wang's 64b mix function from http://www.concentric.net/~Ttwang/tech/inthash.htm
def _gen_key(i: int(64)): int(64) {
  var key = i;
  key += ~(key << 32);
  key = key ^ (key >> 22);
  key += ~(key << 13);
  key = key ^ (key >> 8);
  key += (key << 3);
  key = key ^ (key >> 15);
  key += ~(key << 27);
  key = key ^ (key >> 31);
  return key & 0x7fffffffffffffff;  // YAH, make non-negative
}

pragma "inline"
def _associative_hash(b: bool): int(64) {
  if (b)
    return 17;
  else
    return 31;
}

pragma "inline"
def _associative_hash(i: int(64)): int(64) {
  return _gen_key(i);
}

pragma "inline"
def _associative_hash(u: uint(64)): int(64) {
  return _gen_key(u:int(64));
}

pragma "inline"
def _associative_hash(f: real): int(64) {
  return _gen_key(__primitive( "real2int", f));
}

pragma "inline"
def _associative_hash(c: complex): int(64) {
  return _gen_key(c.re:int ^ c.im:int);
}

// Use djb2 (Dan Bernstein in comp.lang.c.
pragma "inline"
def _associative_hash(x : string): int(64) {
  var hash: int(64) = 0;
  for c in 1..length(x) {
    hash = ((hash << 5) + hash) ^ ascii(x.substring(c));
  }
  return _gen_key(hash);
}

pragma "inline"
def _associative_hash(o: object): int(64) {
  return _gen_key(__primitive( "object2int", o));
}


Performance bump...

Performance bump...

use Regexp;
use Histogram;
use Time;

// Use the test/twopt code
config const isTest=false;

config const fn1 = "test.dat";
config const fn2 = "test.dat";
config const pairfn = "test-DD.dat";

param NDIM  = 3;
const Ddim = {0.. #NDIM};

config const spaces=compile("\\s+");
config const bufsize=10000;
config const minpart=500;

config const smax=200.0;
config const smax2=smax**2;
config const nmubins=5;
config const nsbins=5;

// Global variables
var gtime1 : Timer;

proc main() {
  if isTest then testPairs(); else doPairs();
}

record WeightedParticle3D {
  var x : [Ddim] real;
  var w : real;
}

proc countLines(fn : string) : int {
  var ff = open(fn, iomode.r);
  var ipart = 0;
  for iff in ff.lines() do ipart +=1;
  ff.close();
  return ipart;
}

proc readFile(fn : string, pp : []WeightedParticle3D)  {
  const maxcols=25;

  var ff = open(fn, iomode.r);
  var cols : [0.. #maxcols] real;
  var icol=0;
  var ipart = 0;
  for iff in ff.lines() {
   icol = 0;
   for col1 in iff.split(spaces) {
     if (col1.length==0) then continue;
     cols[icol] = col1 : real;
     icol += 1;
   }
   if (icol < 4) then assert(false,"malformed line...");
   pp[ipart].x = cols[Ddim];
   pp[ipart].w = cols[3];
   ipart += 1;
  }
}

class SOA_WeightedParticle3D {
  var npart : int;
  var Dpart : domain(1);
  var x,y,z,w,r2 : [Dpart] real;

  proc SOA_WeightedParticle3D(pp : []WeightedParticle3D) {
    npart = pp.size;
    Dpart = 0.. #npart;

    forall ii in Dpart {
      x[ii] = pp[ii].x[0];
      y[ii] = pp[ii].x[1];
      z[ii] = pp[ii].x[2];
      w[ii] = pp[ii].w;
      r2[ii] = + reduce (pp[ii].x**2);
    }
  }
}

proc smuAccumulate(hh : UniformBins, p1,p2 : SOA_WeightedParticle3D, d1,d2 : domain(1), scale : real) {
  forall ii in d1 { // Loop over first set of particles

    var x1,y1,z1,w1,r2 : real;
    var sl, s2, l1, s1, l2, mu, wprod : real;
    x1 = p1.x[ii]; y1 = p1.y[ii]; z1 = p1.z[ii]; w1 = p1.w[ii]; r2 = p1.r2[ii];

    for jj in d2 { // Second set of particles
      mu=2*(p2.x[jj]*x1 + p2.y[jj]*y1 + p2.z[jj]*z1);
      sl = r2 - p2.r2[jj];
      l1 = r2 + p2.r2[jj];
      s2 = l1 - mu;
      l2 = l1 + mu;
      if ((s2 >= smax2) || (s2 < 1.0e-20)) then continue;
      wprod = scale * w1 * p2.w[jj];
      s1 = sqrt(s2);
      mu = sl/(s1*sqrt(l2));
      if (mu < 0) then mu = -mu;

      hh.add((s1,mu),wprod);
    }
  }
}

proc writeHist(fn : string, hh : UniformBins) {
  var ff = openwriter(fn);
  // Dump out values
  for xx in hh.bins(1) do ff.writef("%12.4dr",xx);
  ff.writeln();
  for xx in hh.bins(2) do ff.writef("%12.4dr",xx);
  ff.writeln("\n##");
  for ii in hh.Dhist.dim(1) {
    for jj in hh.Dhist.dim(2) {
      ff.writef("%20.14er ",hh[(ii,jj)]);
    }
    ff.writeln();
  }
  ff.close();
}

proc splitOn(pp : []WeightedParticle3D, scr : []WeightedParticle3D, splitDim : int, rr : []real) : int {
  var npart, lnpart : int;
  var lo = pp.domain.low;
  var hi = pp.domain.high;
  npart = pp.domain.size;
  lnpart = 0;
  for ipp in pp {
    if (ipp.x[splitDim] < rr[splitDim]) then lnpart+=1;
  }
//  var part : [pp.domain] WeightedParticle3D;
  var li, ri : int;
  li = lo; ri = (lo+lnpart);
  for ipp in pp {
    if (ipp.x[splitDim] < rr[splitDim]) {
      scr[li] = ipp;
      li+=1;
    } else {
      scr[ri] = ipp;
      ri+=1;
    }
  }
  assert(li==(lo+lnpart), "splitOn : li != lo+npart");
  assert(ri==(hi+1), "splitOn : ri != hi+1");
  pp = scr;

  return lnpart;
}

class KDNode {
  var lo, hi,npart,id : int;
  var dom : domain(1);
  var xcen : [Ddim]real;
  var rcell : real;
  var left, right : KDNode;

  proc isLeaf() : bool {
    return (left==nil) && (right==nil);
  }

  proc ~KDNode() {
    if left then delete left;
    if right then delete right;
  }

}

proc BuildTree(pp : []WeightedParticle3D, scr : []WeightedParticle3D, id : int) : KDNode{
  gtime1.start();
  var me = new KDNode();
  var dom = pp.domain;
  me.lo = dom.low;
  me.hi = dom.high;
  me.dom = {me.lo..me.hi};
  me.id = id;
  me.npart = (me.hi-me.lo)+1;

  //  work out xcen and vantage point radius
  var pmin = pp[me.lo].x;
  var pmax = pp[me.lo].x;
  for ipp in pp {
    for idim in Ddim {
      if (ipp.x[idim] < pmin[idim]) then pmin[idim] = ipp.x[idim];
      if (ipp.x[idim] > pmax[idim]) then pmax[idim] = ipp.x[idim];
    }
  }
  me.xcen = (pmax+pmin)/2.0;
  var dx : [Ddim]real;
  me.rcell = 0.0;
  var r1 : real;
  for ipp in pp {
    dx = (ipp.x - me.xcen);
    r1 = sqrt(+ reduce dx**2);
    if (r1 > me.rcell) then me.rcell = r1;
  }

  // Continue to split
  gtime1.stop();
  if (me.npart <= minpart) then return me; // Don't split further.
  gtime1.start();

  // Find dimension to split on
  dx = pmax - pmin;
  var splitDim = 0;
  for idim in Ddim {
    if (dx[idim] > dx[splitDim]) then splitDim=idim;
  }

  // Split
  var lnpart = splitOn(pp, scr,splitDim, me.xcen);
  var ldom = {me.lo..(me.lo+lnpart-1)};
  var rdom = {(me.lo+lnpart)..me.hi};
  gtime1.stop();
  me.left = BuildTree(pp[ldom],scr[ldom],2*id+1);
  me.right = BuildTree(pp[rdom],scr[rdom],2*id+2);
  return me;
}

proc TreeAccumulate(hh : UniformBins, p1, p2 : SOA_WeightedParticle3D, node1, node2 : KDNode) {
  // Compute the distance between node1 and node2
  var rr = sqrt (+ reduce(node1.xcen - node2.xcen)**2);
  var rmin = rr - (node1.rcell+node2.rcell);

  // If distance is greater than all cases
  if (rmin > smax) then return;

  // If both nodes are leaves
  if (node1.isLeaf() & node2.isLeaf()) {
    begin smuAccumulate(hh, p1, p2, node1.dom, node2.dom,1);
    return;
  }

  // If one node is a leaf
  if (node1.isLeaf()) {
    begin TreeAccumulate(hh, p1, p2, node1, node2.left);
    begin TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }
  if (node2.isLeaf()) {
    begin TreeAccumulate(hh, p1, p2, node1.left, node2);
    begin TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  }

  // Split the larger case;
  if (node1.npart > node2.npart) {
    begin TreeAccumulate(hh, p1, p2, node1.left, node2);
    begin TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  } else {
    begin TreeAccumulate(hh, p1, p2, node1, node2.left);
    begin TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }

}

// A simple testHarness
proc testPairs() {

  // Read in the file
  var nlines = countLines(fn1);
  var pp1 : [0.. #nlines] WeightedParticle3D;
  readFile("test.dat",pp1);

  // Build the tree
  var scr : [0.. #nlines] WeightedParticle3D;
  var root1 = BuildTree(pp1,scr,0);

  // Set up the histogram
  var hh = new UniformBins(2,(nsbins,nmubins), ((0.0,smax),(0.0,1.0+1.e-10)));

  // AOS -> SOA
  var soa1 = new SOA_WeightedParticle3D(pp1);

  // Do the paircounts with a tree
  sync TreeAccumulate(hh, soa1, soa1, root1, root1);
  for xx in hh.bins(1) do writef("%12.4dr",xx);
  writeln();
  for xx in hh.bins(2) do writef("%12.4dr",xx);
  writeln("\n##");
  for ii in hh.Dhist.dim(1) {
    for jj in hh.Dhist.dim(2) {
      writef("%20.5er ",hh[(ii,jj)]);
    }
    writeln();
  }
}

// The main code
proc doPairs() {
  var tt : Timer;

  // Read in the file
  tt.clear(); tt.start();
  var nlines1 = countLines(fn1);
  var pp1 : [0.. #nlines1] WeightedParticle3D;
  readFile(fn1,pp1);
  var nlines2 = countLines(fn2);
  var pp2 : [0.. #nlines2] WeightedParticle3D;
  readFile(fn1,pp2);
  tt.stop();
  writef("Read in %i lines from file %s \n", pp1.size, fn1);
  writef("Read in %i lines from file %s \n", pp2.size, fn2);
  writef("Time to read : %r \n", tt.elapsed());

  // Build the tree
  tt.clear(); tt.start(); gtime1.clear();
  var scr1 : [0.. #nlines1] WeightedParticle3D;
  var root1 = BuildTree(pp1,scr1,0);
  var scr2 : [0.. #nlines2] WeightedParticle3D;
  var root2 = BuildTree(pp2,scr2,0);
  tt.stop();
  writef("Time to build trees : %r \n", tt.elapsed());
  writef("Time in splitOn : %r \n", gtime1.elapsed());

  // Set up the histogram
  var hh = new UniformBins(2,(nsbins,nmubins), ((0.0,smax),(0.0,1.0+1.e-10)));

  // AOS -> SOA
  tt.clear(); tt.start();
  var soa1 = new SOA_WeightedParticle3D(pp1);
  var soa2 = new SOA_WeightedParticle3D(pp2);
  tt.stop();
  writef("Time to SOA : %r \n", tt.elapsed());

  // Brute force paircounts
  tt.clear(); tt.start();
  smuAccumulate(hh, soa1,soa2, soa1.Dpart, soa2.Dpart, 1.0);
  tt.stop();
  writef("Time to brute force paircount : %r \n", tt.elapsed());
  writeHist("%s.brute".format(pairfn),hh);

   // Do the paircounts with a tree
  hh.reset();
  tt.clear(); tt.start();
  sync TreeAccumulate(hh, soa1, soa2, root1, root2);
  tt.stop();
  writef("Time to tree paircount : %r \n", tt.elapsed());
  writeHist("%s.tree".format(pairfn),hh);
}

module Histogram {

  class UniformBins {
    param dim : int;
    var nbins : dim*int;
    var Dhist : domain(dim);
    var lo, hi,dx, invdx : [1..dim] real;
    var arr : [Dhist] atomic real;

    proc UniformBins(param dim : int, nbins : dim*int, limits : dim*(real,real)) {
      var dd : dim*range;
      this.nbins = nbins;
      for param ii in 1..dim {
        lo[ii] = limits(ii)(1);
        hi[ii] = limits(ii)(2);
        dx[ii] = (hi[ii]-lo[ii])/nbins(ii);
        dd(ii) = 0.. #nbins(ii);
      }
      invdx = 1.0/dx;
      Dhist = {(...dd)};
      [x in arr] x.write(0.0);
    }

    proc reset() {
      [x in arr] x.write(0.0);
    }

    proc bins(idim : int) {
      var bounds : [0.. #nbins(idim)+1] real;
      for i in {0.. #(nbins(idim)+1)} do bounds[i] = lo[idim] + dx[idim]*i;
      return bounds;
    }

    proc add(x : dim*real, w : real) {
      for param ii in 1..dim do
        if ((x(ii) < lo[ii]) | (x(ii) >= hi[ii])) then return;

      var pos : dim*int;
      for param ii in 1..dim do pos(ii) = ((x(ii)-lo(ii))*invdx[ii]) : int;
      arr[pos].add(w);
    }

    proc this(ndx) : real {
      return arr[ndx].read();
    }

  } // UniformBins

} // End module Histogram

use Regexp;
use Histogram;
use Time;

// Use the test/twopt code
config const isTest=false;

config const fn1 = "test.dat";
config const fn2 = "test.dat";
config const pairfn = "test-DD.dat";

param NDIM  = 3;
const Ddim = {0.. #NDIM};

config const spaces=compile("\\s+");
config const bufsize=10000;
config const minpart=500;

config const smax=200.0;
config const smax2=smax**2;
config const nmubins=5;
config const nsbins=5;

// Global variables
var gtime1 : Timer;

proc main() {
  if isTest then testPairs(); else doPairs();
}

record WeightedParticle3D {
  var x : [Ddim] real;
  var w : real;
}

proc countLines(fn : string) : int {
  var ff = open(fn, iomode.r);
  var ipart = 0;
  for iff in ff.lines() do ipart +=1;
  ff.close();
  return ipart;
}

proc readFile(fn : string, pp : []WeightedParticle3D)  {
  const maxcols=25;

  var ff = open(fn, iomode.r);
  var cols : [0.. #maxcols] real;
  var icol=0;
  var ipart = 0;
  for iff in ff.lines() {
   icol = 0;
   for col1 in iff.split(spaces) {
     if (col1.length==0) then continue;
     cols[icol] = col1 : real;
     icol += 1;
   }
   if (icol < 4) then assert(false,"malformed line...");
   pp[ipart].x = cols[Ddim];
   pp[ipart].w = cols[3];
   ipart += 1;
  }
}

class SOA_WeightedParticle3D {
  var npart : int;
  var Dpart : domain(1);
  var x,y,z,w,r2 : [Dpart] real;

  proc SOA_WeightedParticle3D(pp : []WeightedParticle3D) {
    npart = pp.size;
    Dpart = 0.. #npart;

    forall ii in Dpart {
      x[ii] = pp[ii].x[0];
      y[ii] = pp[ii].x[1];
      z[ii] = pp[ii].x[2];
      w[ii] = pp[ii].w;
      r2[ii] = + reduce (pp[ii].x**2);
    }
  }
}

proc smuAccumulate(hh : UniformBins, p1,p2 : SOA_WeightedParticle3D, d1,d2 : domain(1), scale : real) {
  forall ii in d1 { // Loop over first set of particles

    var x1,y1,z1,w1,r2 : real;
    var sl, s2, l1, s1, l2, mu, wprod : real;
    x1 = p1.x[ii]; y1 = p1.y[ii]; z1 = p1.z[ii]; w1 = p1.w[ii]; r2 = p1.r2[ii];

    for jj in d2 { // Second set of particles
      mu=2*(p2.x[jj]*x1 + p2.y[jj]*y1 + p2.z[jj]*z1);
      sl = r2 - p2.r2[jj];
      l1 = r2 + p2.r2[jj];
      s2 = l1 - mu;
      l2 = l1 + mu;
      if ((s2 >= smax2) || (s2 < 1.0e-20)) then continue;
      wprod = scale * w1 * p2.w[jj];
      s1 = sqrt(s2);
      mu = sl/(s1*sqrt(l2));
      if (mu < 0) then mu = -mu;

      hh.add((s1,mu),wprod);
    }
  }
}

proc writeHist(fn : string, hh : UniformBins) {
  var ff = openwriter(fn);
  // Dump out values
  for xx in hh.bins(1) do ff.writef("%12.4dr",xx);
  ff.writeln();
  for xx in hh.bins(2) do ff.writef("%12.4dr",xx);
  ff.writeln("\n##");
  for ii in hh.Dhist.dim(1) {
    for jj in hh.Dhist.dim(2) {
      ff.writef("%20.14er ",hh[(ii,jj)]);
    }
    ff.writeln();
  }
  ff.close();
}

proc splitOn(pp : []WeightedParticle3D, scr : []WeightedParticle3D, splitDim : int, rr : []real) : int {
  var npart, lnpart : int;
  var lo = pp.domain.low;
  var hi = pp.domain.high;
  npart = pp.domain.size;
  lnpart = 0;
  for ipp in pp {
    if (ipp.x[splitDim] < rr[splitDim]) then lnpart+=1;
  }
//  var part : [pp.domain] WeightedParticle3D;
  var li, ri : int;
  li = lo; ri = (lo+lnpart);
  for ipp in pp {
    if (ipp.x[splitDim] < rr[splitDim]) {
      scr[li] = ipp;
      li+=1;
    } else {
      scr[ri] = ipp;
      ri+=1;
    }
  }
  assert(li==(lo+lnpart), "splitOn : li != lo+npart");
  assert(ri==(hi+1), "splitOn : ri != hi+1");
  pp = scr;

  return lnpart;
}

class KDNode {
  var lo, hi,npart,id : int;
  var dom : domain(1);
  var xcen : [Ddim]real;
  var rcell : real;
  var left, right : KDNode;

  proc isLeaf() : bool {
    return (left==nil) && (right==nil);
  }

  proc ~KDNode() {
    if left then delete left;
    if right then delete right;
  }

}

proc BuildTree(pp : []WeightedParticle3D, scr : []WeightedParticle3D, id : int) : KDNode{
  gtime1.start();
  var me = new KDNode();
  var dom = pp.domain;
  me.lo = dom.low;
  me.hi = dom.high;
  me.dom = {me.lo..me.hi};
  me.id = id;
  me.npart = (me.hi-me.lo)+1;

  //  work out xcen and vantage point radius
  var pmin = pp[me.lo].x;
  var pmax = pp[me.lo].x;
  for ipp in pp {
    for idim in Ddim {
      if (ipp.x[idim] < pmin[idim]) then pmin[idim] = ipp.x[idim];
      if (ipp.x[idim] > pmax[idim]) then pmax[idim] = ipp.x[idim];
    }
  }
  me.xcen = (pmax+pmin)/2.0;
  var dx : [Ddim]real;
  me.rcell = 0.0;
  var r1 : real;
  for ipp in pp {
    dx = (ipp.x - me.xcen);
    r1 = sqrt(+ reduce dx**2);
    if (r1 > me.rcell) then me.rcell = r1;
  }

  // Continue to split
  gtime1.stop();
  if (me.npart <= minpart) then return me; // Don't split further.
  gtime1.start();

  // Find dimension to split on
  dx = pmax - pmin;
  var splitDim = 0;
  for idim in Ddim {
    if (dx[idim] > dx[splitDim]) then splitDim=idim;
  }

  // Split
  var lnpart = splitOn(pp, scr,splitDim, me.xcen);
  var ldom = {me.lo..(me.lo+lnpart-1)};
  var rdom = {(me.lo+lnpart)..me.hi};
  gtime1.stop();
  me.left = BuildTree(pp[ldom],scr[ldom],2*id+1);
  me.right = BuildTree(pp[rdom],scr[rdom],2*id+2);
  return me;
}

proc TreeAccumulate(hh : UniformBins, p1, p2 : SOA_WeightedParticle3D, node1, node2 : KDNode) {
  // Compute the distance between node1 and node2
  var rr = sqrt (+ reduce(node1.xcen - node2.xcen)**2);
  var rmin = rr - (node1.rcell+node2.rcell);

  // If distance is greater than all cases
  if (rmin > smax) then return;

  // If both nodes are leaves
  if (node1.isLeaf() & node2.isLeaf()) {
    begin smuAccumulate(hh, p1, p2, node1.dom, node2.dom,1);
    return;
  }

  // If one node is a leaf
  if (node1.isLeaf()) {
    begin TreeAccumulate(hh, p1, p2, node1, node2.left);
    begin TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }
  if (node2.isLeaf()) {
    begin TreeAccumulate(hh, p1, p2, node1.left, node2);
    begin TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  }

  // Split the larger case;
  if (node1.npart > node2.npart) {
    begin TreeAccumulate(hh, p1, p2, node1.left, node2);
    begin TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  } else {
    begin TreeAccumulate(hh, p1, p2, node1, node2.left);
    begin TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }

}

// A simple testHarness
proc testPairs() {

  // Read in the file
  var nlines = countLines(fn1);
  var pp1 : [0.. #nlines] WeightedParticle3D;
  readFile("test.dat",pp1);

  // Build the tree
  var Dscr : domain(1);
  var scr : [Dscr] WeightedParticle3D;
  Dscr = pp1.domain;
  var root1 = BuildTree(pp1,scr,0);

  // Set up the histogram
  var hh = new UniformBins(2,(nsbins,nmubins), ((0.0,smax),(0.0,1.0+1.e-10)));

  // AOS -> SOA
  var soa1 = new SOA_WeightedParticle3D(pp1);

  // Do the paircounts with a tree
  sync TreeAccumulate(hh, soa1, soa1, root1, root1);
  for xx in hh.bins(1) do writef("%12.4dr",xx);
  writeln();
  for xx in hh.bins(2) do writef("%12.4dr",xx);
  writeln("\n##");
  for ii in hh.Dhist.dim(1) {
    for jj in hh.Dhist.dim(2) {
      writef("%20.5er ",hh[(ii,jj)]);
    }
    writeln();
  }
}

// The main code
proc doPairs() {
  var tt : Timer;

  // Read in the file
  tt.clear(); tt.start();
  var nlines = countLines(fn1);
  var pp1 : [0.. #nlines] WeightedParticle3D;
  readFile(fn1,pp1);
  nlines = countLines(fn2);
  var pp2 : [0.. #nlines] WeightedParticle3D;
  readFile(fn1,pp2);
  tt.stop();
  writef("Read in %i lines from file %s \n", pp1.size, fn1);
  writef("Read in %i lines from file %s \n", pp2.size, fn2);
  writef("Time to read : %r \n", tt.elapsed());

  // Build the tree
  var Dscr : domain(1);
  var scr : [Dscr] WeightedParticle3D;
  tt.clear(); tt.start(); gtime1.clear();
  Dscr = pp1.domain;
  var root1 = BuildTree(pp1,scr,0);
  Dscr = pp2.domain;
  var root2 = BuildTree(pp2,scr,0);
  tt.stop();
  writef("Time to build trees : %r \n", tt.elapsed());
  writef("Time in splitOn : %r \n", gtime1.elapsed());

  // Set up the histogram
  var hh = new UniformBins(2,(nsbins,nmubins), ((0.0,smax),(0.0,1.0+1.e-10)));

  // AOS -> SOA
  tt.clear(); tt.start();
  var soa1 = new SOA_WeightedParticle3D(pp1);
  var soa2 = new SOA_WeightedParticle3D(pp2);
  tt.stop();
  writef("Time to SOA : %r \n", tt.elapsed());

  // Brute force paircounts
  tt.clear(); tt.start();
  smuAccumulate(hh, soa1,soa2, soa1.Dpart, soa2.Dpart, 1.0);
  tt.stop();
  writef("Time to brute force paircount : %r \n", tt.elapsed());
  writeHist("%s.brute".format(pairfn),hh);

   // Do the paircounts with a tree
  hh.reset();
  tt.clear(); tt.start();
  sync TreeAccumulate(hh, soa1, soa2, root1, root2);
  tt.stop();
  writef("Time to tree paircount : %r \n", tt.elapsed());
  writeHist("%s.tree".format(pairfn),hh);
}

module Histogram {

  class UniformBins {
    param dim : int;
    var nbins : dim*int;
    var Dhist : domain(dim);
    var lo, hi,dx, invdx : [1..dim] real;
    var arr : [Dhist] atomic real;

    proc UniformBins(param dim : int, nbins : dim*int, limits : dim*(real,real)) {
      var dd : dim*range;
      this.nbins = nbins;
      for param ii in 1..dim {
        lo[ii] = limits(ii)(1);
        hi[ii] = limits(ii)(2);
        dx[ii] = (hi[ii]-lo[ii])/nbins(ii);
        dd(ii) = 0.. #nbins(ii);
      }
      invdx = 1.0/dx;
      Dhist = {(...dd)};
      [x in arr] x.write(0.0);
    }

    proc reset() {
      [x in arr] x.write(0.0);
    }

    proc bins(idim : int) {
      var bounds : [0.. #nbins(idim)+1] real;
      for i in {0.. #(nbins(idim)+1)} do bounds[i] = lo[idim] + dx[idim]*i;
      return bounds;
    }

    proc add(x : dim*real, w : real) {
      for param ii in 1..dim do
        if ((x(ii) < lo[ii]) | (x(ii) >= hi[ii])) then return;

      var pos : dim*int;
      for param ii in 1..dim do pos(ii) = ((x(ii)-lo(ii))*invdx[ii]) : int;
      arr[pos].add(w);
    }

    proc this(ndx) : real {
      return arr[ndx].read();
    }

  } // UniformBins

} // End module Histogram