More cleanup to HPL

More cleanup to HPL

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14881 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of matrices and the element type of those matrices
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2): indexType,
             blkSize = 5;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic distribution.
  //
  const MatVectSpace: domain(2, indexType) = [1..n, 1..n+1],
        MatrixSpace = MatVectSpace[.., ..n];

  var Ab : [MatVectSpace] elemType,  // the matrix A and vector b
      piv: [1..n] indexType,         // a vector of pivot values
      x  : [1..n] elemType;          // the solution vector, x

  var A => Ab[MatrixSpace],          // an alias for the Matrix part of Ab
      b => Ab[.., n+1];              // an alias for the last column of Ab

  initAB(Ab);

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, Ab, piv);                 // compute the LU factorization

  x = backwardSub(n, A, b);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(Ab, MatrixSpace, x);
  printResults(validAnswer, execTime);
}

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
def LUFactorize(n: indexType, Ab: [1..n, 1..n+1] elemType,
                piv: [1..n] indexType) {
  const AbD = Ab.domain;    // alias Ab.domain to save typing

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(Ab, l, piv);
    if (tr.numIndices > 0) then
      updateBlockRow(Ab, tl, tr);

    //
    // update trailing submatrix (if any)
    //
    if (br.numIndices > 0) then
      schurComplement(Ab, blk);
  }
}

// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The vertex labeled [1] is location
//     |aaaaa|.....|.....|.....|  (ptOp, ptOp) in the code below.
//     +-----+-----+-----+-----+
//     |aaaaa|.....|.....|.....|  The vertex labeled [2] is location
//     |aaaaa|.....|.....|.....|  (ptSol, ptSol)
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
def schurComplement(Ab: [1..n, 1..n+1] elemType, ptOp: indexType) {
  const AbD = Ab.domain;

  //
  // Calculate location of ptSol (see diagram above)
  //
  const ptSol = ptOp+blkSize;

  //
  // Copy data into replicated array so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.  These
  // replicated distributions aren't implemented yet, but imagine that
  // they look something like the following:
  //
  //var replAbD: domain(2)
  //            distributed new Dimensional(BlkCyc(blkSize), Replicated))
  //          = AbD[ptSol.., 1..#blkSize];
  //
  const replAD: domain(2) = AbD[ptSol.., ptOp..#blkSize],
        replBD: domain(2) = AbD[ptOp..#blkSize, ptSol..];

  const replA : [replAD] elemType = Ab[ptSol.., ptOp..#blkSize],
        replB : [replBD] elemType = Ab[ptOp..#blkSize, ptSol..];

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in AbD[ptSol.., ptSol..] by (blkSize, blkSize) {
    //
    // At this point, the dgemms should all be local, so assert that
    // fact
    //
    local {
      const aBlkD = replAD[row..#blkSize, ptOp..#blkSize],
            bBlkD = replBD[ptOp..#blkSize, col..#blkSize],
            cBlkD = AbD[row..#blkSize, col..#blkSize];

      dgemm(aBlkD.dim(1).length,
            aBlkD.dim(2).length,
            bBlkD.dim(2).length,
            replA(aBlkD),
            replB(bBlkD),
            Ab(cBlkD));
    }
  }
}

//
// calculate C = C - A * B.
//
def dgemm(p: indexType,       // number of rows in A
          q: indexType,       // number of cols in A, number of rows in B
          r: indexType,       // number of cols in B
          A: [1..p, 1..q] ?t,
          B: [1..q, 1..r] t,
          C: [1..p, 1..r] t) {
  // Calculate (i,j) using a dot product of a row of A and a column of B.
  for i in 1..p do
    for j in 1..r do
      for k in 1..q do
        C[i,j] -= A[i, k] * B[k, j];
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
def panelSolve(Ab: [] ?t,
               panel: domain(2, indexType),
               piv: [] indexType) {
  const pnlRows = panel.dim(1),
        pnlCols = panel.dim(2);

  //
  // Ideally some type of assertion to ensure panel is embedded in Ab's
  // domain
  //
  assert(piv.domain.dim(1) == Ab.domain.dim(1));

  if (pnlCols.length == 0) then return;

  for k in pnlCols {             // iterate through the columns
    var col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.dim(1).length == 0 then return;

    // The pivot is the element with the largest absolute value.  I need to
    // do this in two steps since if I assign it like (pivot, pivotRow) =
    // maxloc(...) pivot will be assigned to the absolute value, not the
    // actual value.
    const (_, pivotRow) = maxloc reduce(abs(Ab(col)), col.dim(1)),
          pivot = Ab[pivotRow, k];

    // Swap the current row with the pivot row
    piv[k] <=> piv[pivotRow];

    Ab[k, ..] <=> Ab[pivotRow, ..];

    if (pivot == 0) then
      halt("Matrix can not be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot
    if k+1 <= pnlRows.high then
      Ab(col)[k+1.., k..k] /= pivot;

    // update all other values below the pivot
    if k+1 <= pnlRows.high && k+1 <= pnlCols.high then
      forall (i,j) in panel[k+1.., k+1..] do
        Ab[i,j] -= Ab[i,k] * Ab[k,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
def updateBlockRow(Ab: [] ?t, tl: domain(2), tr: domain(2)) {
  const tlRows = tl.dim(1),
        tlCols = tl.dim(2),
        trRows = tr.dim(1),
        trCols = tr.dim(2);

  assert(tlCols == trRows);

  //
  // Ultimately, we will probably want to do some replication of the
  // tl block in order to make this operation completely localized as
  // in the dgemm.  We have not yet undertaken that optimization.
  //
  for i in trRows do
    forall j in trCols do
      for k in tlRows.low..i-1 do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
}

//
// compute the backwards substitution
//
def backwardSub(n: int,
                A: [1..n, 1..n] elemType,
                b: [1..n] elemType) {
  var x: [b.domain] elemType;

  for i in [b.domain by -1] {
    x[i] = b[i];

    for j in [i+1..b.domain.high] do
      x[i] -= A[i,j] * x[j];

    x[i] /= A[i,i];
  }

  return x;
}

//
// print out the problem size and block size if requested
//
def printConfiguration() {
  if (printParams) {
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
def initAB(Ab: [] elemType) {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
def verifyResults(Ab, MatrixSpace, x) {
  var A => Ab[MatrixSpace],
      b => Ab[.., n+1];

  initAB(Ab);

  const axmbNorm = norm(gaxpyMinus(n, n, A, x, b), normType.normInf);

  const a1norm   = norm(A, normType.norm1),
        aInfNorm = norm(A, normType.normInf),
        x1Norm   = norm(x, normType.norm1),
        xInfNorm = norm(x, normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
def gaxpyMinus(n: indexType,
               m: indexType,
               A: [1..n, 1..m],
               x: [1..m],
               y: [1..n]) {
  var res: [1..n] elemType;

  for i in 1..n do
    for j in 1..m do
      res[i] += A[i,j]*x[j];

  for i in 1..n do
    res[i] -= y[i];

  return res;
}

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of matrices and the element type of those matrices
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2): indexType,
             blkSize = 5;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic distribution.
  //
  const MatVectSpace: domain(2, indexType) = [1..n, 1..n+1],
        MatrixSpace = MatVectSpace[.., ..n];

  var A  : [MatVectSpace] elemType,  // the matrix A (and vector b)
      piv: [1..n] indexType,         // a vector of pivot values
      x  : [1..n] elemType;          // the solution vector, x

  var b => A[.., n+1];               // an alias for the last column of A

  //
  // construct an n by n+1 matrix filled with random values and scale
  // it to be in the interval -1.0..1.0
  //
  fillRandom(A, seed);
  A = A * 2.0 - 1.0;

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, A, piv);                 // compute the LU factorization

  x = backwardSub(n, A[MatrixSpace], b);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(A, MatrixSpace, x, b);
  printResults(validAnswer, execTime);
}

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
def LUFactorize(n: indexType, A: [1..n, 1..n+1] elemType,
                piv: [1..n] indexType) {
  const AD = A.domain;    // alias A.domain to save typing

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    const tl = AD[blk..#blkSize, blk..#blkSize],
          tr = AD[blk..#blkSize, blk+blkSize..],
          bl = AD[blk+blkSize.., blk..#blkSize],
          br = AD[blk+blkSize.., blk+blkSize..],
          l  = AD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced A properly, do the blocked-LU
    // computation:
    //
    panelSolve(A, l, piv);
    if (tr.numIndices > 0) then
      updateBlockRow(A, tl, tr);

    //
    // update trailing submatrix (if any)
    //
    if (br.numIndices > 0) then
      schurComplement(A, blk);
  }
}

// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The vertex labeled [1] is location
//     |aaaaa|.....|.....|.....|  (ptOp, ptOp) in the code below.
//     +-----+-----+-----+-----+
//     |aaaaa|.....|.....|.....|  The vertex labeled [2] is location
//     |aaaaa|.....|.....|.....|  (ptSol, ptSol)
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
def schurComplement(A: [1..n, 1..n+1] elemType, ptOp: indexType) {
  const AD = A.domain;

  //
  // Calculate location of ptSol (see diagram above)
  //
  const ptSol = ptOp+blkSize;

  //
  // Copy data into replicated array so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.  These
  // replicated distributions aren't implemented yet, but imagine that
  // they look something like the following:
  //
  //var replAD: domain(2)
  //            distributed new Dimensional(BlkCyc(blkSize), Replicated))
  //          = AD[ptSol.., 1..#blkSize];
  //
  const replAD: domain(2) = AD[ptSol.., ptOp..#blkSize],
        replBD: domain(2) = AD[ptOp..#blkSize, ptSol..];

  const replA : [replAD] elemType = A[ptSol.., ptOp..#blkSize],
        replB : [replBD] elemType = A[ptOp..#blkSize, ptSol..];

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in AD[ptSol.., ptSol..] by (blkSize, blkSize) {
    //
    // At this point, the dgemms should all be local, so assert that
    // fact
    //
    local {
      const aBlkD = replAD[row..#blkSize, ptOp..#blkSize],
            bBlkD = replBD[ptOp..#blkSize, col..#blkSize],
            cBlkD = AD[row..#blkSize, col..#blkSize];

      dgemm(aBlkD.dim(1).length,
            aBlkD.dim(2).length,
            bBlkD.dim(2).length,
            replA(aBlkD),
            replB(bBlkD),
            A(cBlkD));
    }
  }
}

//
// calculate C = C - A * B.
//
def dgemm(p: indexType,       // number of rows in A
          q: indexType,       // number of cols in A, number of rows in B
          r: indexType,       // number of cols in B
          A: [1..p, 1..q] ?t,
          B: [1..q, 1..r] t,
          C: [1..p, 1..r] t) {
  // Calculate (i,j) using a dot product of a row of A and a column of B.
  for i in 1..p do
    for j in 1..r do
      for k in 1..q do
        C[i,j] -= A[i, k] * B[k, j];
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
def panelSolve(A: [] ?t,
               panel: domain(2, indexType),
               piv: [] indexType) {
  const pnlRows = panel.dim(1),
        pnlCols = panel.dim(2);

  //
  // Ideally some type of assertion to ensure panel is embedded in A's
  // domain
  //
  assert(piv.domain.dim(1) == A.domain.dim(1));

  if (pnlCols.length == 0) then return;

  for k in pnlCols {             // iterate through the columns
    var col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.dim(1).length == 0 then return;

    // The pivot is the element with the largest absolute value.  I need to
    // do this in two steps since if I assign it like (pivot, pivotRow) =
    // maxloc(...) pivot will be assigned to the absolute value, not the
    // actual value.
    const (_, pivotRow) = maxloc reduce(abs(A(col)), col.dim(1)),
          pivot = A[pivotRow, k];

    // Swap the current row with the pivot row
    piv[k] <=> piv[pivotRow];

    A[k, ..] <=> A[pivotRow, ..];

    if (pivot == 0) then
      halt("Matrix can not be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot
    if k+1 <= pnlRows.high then
      A(col)[k+1.., k..k] /= pivot;

    // update all other values below the pivot
    if k+1 <= pnlRows.high && k+1 <= pnlCols.high then
      forall (i,j) in panel[k+1.., k+1..] do
        A[i,j] -= A[i,k] * A[k,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
def updateBlockRow(A: [] ?t, tl: domain(2), tr: domain(2)) {
  const tlRows = tl.dim(1),
        tlCols = tl.dim(2),
        trRows = tr.dim(1),
        trCols = tr.dim(2);

  assert(tlCols == trRows);

  //
  // Ultimately, we will probably want to do some replication of the
  // tl block in order to make this operation completely localized as
  // in the dgemm.  We have not yet undertaken that optimization.
  //
  for i in trRows do
    forall j in trCols do
      for k in tlRows.low..i-1 do
        A[i, j] -= A[i, k] * A[k,j];
}

//
// compute the backwards substitution
//
def backwardSub(n: int,
                A: [1..n, 1..n] elemType,
                b: [1..n] elemType) {
  var x: [b.domain] elemType;

  for i in [b.domain by -1] {
    x[i] = b[i];

    for j in [i+1..b.domain.high] do
      x[i] -= A[i,j] * x[j];

    x[i] /= A[i,i];
  }

  return x;
}

//
// print out the problem size and block size if requested
//
def printConfiguration() {
  if (printParams) {
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// verify the results
//
def verifyResults(A, MatrixSpace, x, b) {
  // calculate and report verification residuals
  fillRandom(A, seed);
  A = A * 2.0 - 1.0;

  const axmbNorm = norm(gaxpyMinus(n, n, A[MatrixSpace], x, b),
                        normType.normInf);

  const a1norm   = norm(A[MatrixSpace], normType.norm1),
        aInfNorm = norm(A[MatrixSpace], normType.normInf),
        x1Norm   = norm(x, normType.norm1),
        xInfNorm = norm(x, normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
def gaxpyMinus(n: indexType,
               m: indexType,
               A: [1..n, 1..m],
               x: [1..m],
               y: [1..n]) {
  var res: [1..n] elemType;

  for i in 1..n do
    for j in 1..m do
      res[i] += A[i,j]*x[j];

  for i in 1..n do
    res[i] -= y[i];

  return res;
}


Grab CHPL_HOME by calling chpl --print-chpl-home.

use Exec;
use FileSystem;

config const testDir = "/Users/tvandoren/src/chapel/test/release/examples/programs";

config const testverbose = false;
config const printEnv = testverbose;

proc main() {
  const chplHome = getChplHome();

  if printEnv {
    writeln("### Chapel Environment ###");
    const chplEnv = run("$CHPL_HOME/util/printchplenv");
    write(chplEnv);
    writeln("##########################");
    writeln();
  }

  forall test in findTests(testDir) {
    runTest(test);
  }
}

proc runTest(test) {
  writeln("testing: ", test);
}

iter findTests(testDir) {
  for filename in findfiles(testDir, recursive=true) {
    if isChapelFile(filename) {
      yield filename;
    }
  }
}

iter findTests(testDir, param tag: iterKind)
  where tag == iterKind.standalone
{
  forall filename in findfiles(testDir, recursive=true) {
    if isChapelFile(filename) {
      yield filename;
    }
  }
}

proc getChplHome() {
  const output = run("chpl --print-chpl-home");
  var chplHome: string;

  for i in 1..output.length {
    if output.substring(i) == "\t" {
      chplHome = output.substring(1..i);
      break;
    }
  }

  if testverbose then
    writeln("CHPL_HOME: ", chplHome);

  return chplHome;
}

proc isChapelFile(filename) {
  return ".chpl" == filename.substring(filename.length-4..);
}

use Exec;
use FileSystem;

config const testDir = "/Users/tvandoren/src/chapel/test/release/examples/programs";

config const printEnv = false;

proc main() {
  if printEnv {
    writeln("### Chapel Environment ###");
    const chplEnv = run("$CHPL_HOME/util/printchplenv");
    write(chplEnv);
    writeln("##########################");
    writeln();
  }

  forall test in findTests(testDir) {
    runTest(test);
  }
}

proc runTest(test) {
  writeln("testing: ", test);
}

iter findTests(testDir) {
  for filename in findfiles(testDir, recursive=true) {
    if isChapelFile(filename) {
      yield filename;
    }
  }
}

iter findTests(testDir, param tag: iterKind)
  where tag == iterKind.standalone
{
  forall filename in findfiles(testDir, recursive=true) {
    if isChapelFile(filename) {
      yield filename;
    }
  }
}

proc isChapelFile(filename) {
  return ".chpl" == filename.substring(filename.length-4..);
}


cleaning

cleaning

use LinearAlgebra,
    LinearAlgebra.Sparse,
    NumSuch,
    Random;

config param batchsize = 10000;

/*
 A matrix of data with named rows and columns.
 */
class NamedMatrix {
  var D: domain(2),
      SD = CSRDomain(D),
      X: [SD] real,  // the actual data
      rows: BiMap = new BiMap(),
      cols: BiMap = new BiMap();

   proc init() {
     this.complete(); // NAMEDMATRIX DOESNT EXPLICITLY INHERIT ANYTHING SO IS THIS NECESSARY??
   }

   proc init(X) {
     this.D = {X.domain.dim(1), X.domain.dim(2)};
     this.complete();
     this.loadX(X);
   }

   proc init(X, names: [] string) {
     this.init(X);
     try! this.setRowNames(names);
     try! this.setColNames(names);
   }

   proc init(X, rownames: [] string, colnames: [] string) {
     this.init(X);
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(rownames: [] string, colnames: [] string) {
     this.D = {1..rownames.size, 1..colnames.size};
     this.complete();
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(N: NamedMatrix) {
     this.init(N.X);
     this.rows = N.rows;
     this.cols = N.cols;
   }

   proc init(X, rows: BiMap, cols: BiMap) {
     this.init(X);
     this.rows = rows;
     this.cols = cols;
   }
}

/*
Returns the number of rows in the matrix frame
 */
proc NamedMatrix.nrows() {
  return X.domain.dim(1).size;
}

/*
Returns the number of columns in the matrix frame
 */
proc NamedMatrix.ncols() {
  return X.domain.dim(2).size;
}

/*
 Finds the max along the row indicated by row number
 */

proc NamedMatrix.rowMax(i: int) {
  return aMax(this.X, 1)[i];
}
proc NamedMatrix.rowMax(s: string) {
  return this.rowMax(this.rows.get(s));
}

proc NamedMatrix.rowMax() {
  return aMax(this.X, 1);
}

proc NamedMatrix.colMax(i: int) {
  return aMax(this.X, 2)[i];
}
proc NamedMatrix.colMax(s: string) {
  return this.colMax(this.cols.get(s));
}

proc NamedMatrix.colMax() {
  return aMax(this.X, 2);
}

proc NamedMatrix.rowArgMax(i: int) {
  return argMax(this.X, 1)[i];
}
proc NamedMatrix.rowArgMax(s: string) {
  return this.rowArgMax(this.rows.get(s));
}

proc NamedMatrix.rowArgMax() {
  return argMax(this.X, 1);
}

proc NamedMatrix.colArgMax(i: int) {
  return argMax(this.X, 2)[i];
}
proc NamedMatrix.colArgMax(s: string) {
  return this.colArgMax(this.cols.get(s));
}

proc NamedMatrix.colArgMax() {
  return argMax(this.X, 2);
}

proc NamedMatrix.rowMin(i: int) {
  return aMin(this.X, 1)[i];
}
proc NamedMatrix.rowMin(s: string) {
  return this.rowMin(this.rows.get(s));
}

proc NamedMatrix.rowMin() {
  return aMin(this.X, 1);
}

proc NamedMatrix.colMin(i: int) {
  return aMin(this.X, 2)[i];
}
proc NamedMatrix.colMin(s: string) {
  return this.colMin(this.cols.get(s));
}

proc NamedMatrix.colMin() {
  return aMin(this.X, 2);
}

proc NamedMatrix.rowArgMin(i:int) {
  return argMin(this.X, 1)[i];
}
proc NamedMatrix.rowArgMin(s: string) {
  return this.rowArgMin(this.rows.get(s));
}

proc NamedMatrix.rowArgMin() {
  return argMin(this.X, 1);
}

proc NamedMatrix.colArgMin(i) {
  return argMin(this.X, 2)[i];
}
proc NamedMatrix.colArgMin(s: string) {
  return this.colArgMin(this.cols.get(s));
}

proc NamedMatrix.colArgMin() {
  return argMin(this.X, 2);
}

/* We don't have a good matPlus for Q + E yet, so doing this element
  by element. kinda sucks.
 */
proc NamedMatrix.matPlus(Y: NamedMatrix) {
  forall ij in Y.SD {
    this.X[ij] += Y.X[ij];
  }
}

/*
 Multiplies the underlying matrix in place.
 */
proc NamedMatrix.eTimes(a: real) {
  this.X = a * this.X;
}

/*
 Prints the matrix with row / column headers, not recommended for large matrices.
 Also width of entries should not exceed about 8 places.
 */
proc NamedMatrix.pprint() {
  write("     ");
  for c in this.cols.sorted() {
    write("       %5s  |".format(c[1]));
  }
  write("\n");
  for r in this.rows.sorted() {
    write(" %3s ".format(r[1]));
    for c in this.cols.sorted() {
      write(" %13.3dr ".format(this.get(r[2], c[2])));
    }
    write("\n");
  }

}

//proc NamedMatrix.argRowMax()

/*
Loads the data from X into the internal array, also called X.  We call them all X to keep it clear.

:arg real[]: Array representing the matrix
 */
proc NamedMatrix.loadX(X:[], shape: 2*int =(-1,-1)) {
  if shape(1) > 0 && shape(2) > 0 {
    this.D = {1..shape(1), 1..shape(2)};
  }
  for (i,j) in X.domain {
    this.SD += (i,j);
    this.X(i,j) = X(i,j);
  }
}

/*
Sets the row names for the matrix X
 */
proc NamedMatrix.setRowNames(rn:[]): string throws {
  //if (X.domain.dim(1).size > 0) && (rn.size != X.domain.dim(1).size) {
  //if (1 > 0) && (rn.size != X.domain.dim(1).size) {
  if rn.size != X.domain.dim(1).size {
    const err = new DimensionMatchError(expected = X.domain.dim(1).size, actual=rn.size);
    throw err;
    return "";
  } else {
    for i in 1..rn.size {
      this.rows.add(rn[i]);
    }
    return this.rows;
  }
}

/*
Sets the column names for the matrix X
*/
proc NamedMatrix.setColNames(cn:[]): string throws {
  if cn.size != X.domain.dim(2).size {
    const err = new DimensionMatchError(expected = X.domain.dim(2).size, actual=cn.size);
    throw err;
    return "";
  } else {
    for i in 1..cn.size {
      this.cols.add(cn[i]);
    }
  }
  return this.cols;
}

/*
 Gets the value of the (i,j) entry of the matrix X in the NamedMatrix
 */
proc NamedMatrix.get(i: int, j: int) {
   return this.X(i,j);
 }

 /*
 Get the values using the named rows and columns
  */
proc NamedMatrix.get(f: string, t: string) {
    return this.X(rows.get(f), cols.get(t));
 }

/*
 Set the value of this.X(i,j) by index row
 */
proc NamedMatrix.set(i: int, j: int, w: real) {
    if this.SD.member((i,j)) {
      this.X(i,j) = w;
    } else {
      this.SD += (i,j);
      this.X(i,j) = w;
    }
    return w;
}

/*
Set the values using the row and column names
*/
proc NamedMatrix.set(f: string, t: string, w: real) {
  return this.set(rows.get(f), cols.get(t), w);
}

/*
Update the value in X(i,j) with `w`
 */
proc NamedMatrix.update(i: int, j: int, w: real) {
  const x = this.get(i,j);
  return this.set(i,j, x + w);
}

/*
Update the value in X(i,j) with `w` using the row column names. This adds
`w` to the current value. To replace the current value, use `.set()`
 */
proc NamedMatrix.update(f: string, t: string, w: real) {
  const x = this.get(rows.get(f),cols.get(t));
  return this.set(rows.get(f),cols.get(t), x + w);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column ids
 */
proc NamedMatrix.remove(i: int, j:int) {
  this.SD -= (i,j);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column names
 */
proc NamedMatrix.remove(i: string, j:string) {
  this.remove(this.rows.get(i), this.cols.get(j));
}

/*
Find the number of non-zeroes in the matrix
 */
proc NamedMatrix.nnz() {
  var i: int = 0;
  for ij in this.X.domain {
    i += 1;
  }
  return i;
}

/*
Calculates the sparsity of the matrix: Number of entries / frame size
 */
proc NamedMatrix.sparsity() {
  const d = this.X.shape[1]:real * this.X.shape[2]: real;
  return this.nnz():real / d;
}

/*
 Takes a index (i,j) tuple and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(ij: 2*int) {
  return this.ncols() * ij[1] + ij[2] - this.ncols();
}
/*
 Takes a index values `i,j` and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(i: int, j: int) {
  return this.grid2seq((i,j));
}

/*
 Takes a sequential value and gives the row / column indices of that entry
 */
proc NamedMatrix.seq2grid(k: int) {
  return (divfloor(k, this.ncols())+1, k % this.ncols());
}

proc NamedMatrix.ntropic(N: NamedMatrix) {
  var T: NamedMatrix = new NamedMatrix(X = tropic(this.X, N.X), this.rows, N.cols);
  return T;
}

/*
 Multiplies the current NamedMatrix `X` against the argument `Y`, but first it aligns
 the names of `X.cols` with `Y.rows`.  Returns an appropriately named NamedMatrix
 :arg NamedMatrix Y:
 */
proc NamedMatrix.ndot(N: NamedMatrix) {
  var C: NamedMatrix = new NamedMatrix(X = dot(this.X,N.X), this.rows, N.cols);
  return C;
}

/*
 Routine to take two NamedMatrices and multiply them along just the row/column
 intersections.  If X has cols [red yellow blue green] and Y has rows [red blue green]
 this function produces a NamedMatrix with rows [red blue green]
 */
proc NamedMatrix.alignAndMultiply(Y: NamedMatrix) {
    var rcOverlap: domain(string) = this.cols.keys & Y.rows.keys;

    var xSD: sparse subdomain(this.D) dmapped CS(),
        ySD: sparse subdomain(Y.D) dmapped CS(),
        xX: [xSD] real,  // the actual data
        yX: [ySD] real,  // the actual data
        xrows: BiMap = new BiMap(),
        yrows: BiMap = new BiMap(),
        xcols: BiMap = new BiMap(),
        ycols: BiMap = new BiMap();

    for (left_row, right_col) in zip(this.D.dim(2), Y.D.dim(1)) {
      for rc in rcOverlap {
        const j = this.cols.get(rc);
        if this.SD.member(left_row, j) {
          xSD += (left_row, j);
          xX[left_row, j] = this.X[left_row, j];
        }
        if Y.SD.member(j, right_col) {
          ySD += (j, right_col);
          yX[j, right_col] = Y.X[j, right_col];
        }
      }
    }
    const z = xX.dot(yX);
    var n = new NamedMatrix(X=z);
    n.rows = this.rows;
    n.cols = Y.cols;
    return n;
}

/*
 Designed to be similar to SciPy's `amax() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amax.html>`_
 Useful on sparse matrices

 We'll get around to documenting here it eventually
 */
proc aMax(X:[], axis = 0) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    maxima[1] = (max reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  }
  return maxima;
}

/*
 Designed to be similar to SciPy's `amin() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amin.html>`_
 Useful on sparse matrices.

 We'll get around to documenting here it eventually
 */
proc aMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    minima[1] = (min reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  }
  return minima;
}

/*
Provides the "where" of the min along an axis in a sparse matrix
 */
proc argMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    minima[1] = miniLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = miniLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = miniLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  }
  return minima;
}

/*
 Provides the "where" of a max along a given axis in a sparse matrix
 */
proc argMax(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    maxima[1] = maxiLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = maxiLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = maxiLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  }
  return maxima;
}

// HELPER FUNCTIONS SHOULD NOT BE EXPOSED TO USER
proc miniLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      minimum: real = 1000000;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) < minimum {
        minimum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) < minimum {
        minimum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) < minimum {
        minimum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc maxiLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      maximum: real;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) > maximum {
        maximum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) > maximum {
        maximum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) > maximum {
        maximum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc sparseEquals(A:[], B:[]) {}

proc tropic(A:[],B:[]) { // UNDER CONSTRUCTION
  var dom: domain(2) = {A.domain.dim(1),B.domain.dim(2)};
  var sps = CSRDomain(dom);
  var BT = transpose(B);
  var T: [sps] real;
  for (i,j) in dom {
    var mini = 100000000: real;
    var wdom: domain(1);
    var wids: [wdom] int;
    for w in A.domain.dimIter(2,i) {
      wids.push_back(w);
    }
    for w2 in BT.domain.dimIter(2,j) {
      wids.push_back(w2);
    }
    var wDom: domain(1);
    var witness: [wDom] real;
    for w in wids {
      if A(i,w) > 0 && BT(j,w) > 0 {
        witness.push_back(A(i,w) + BT(j,w));
      }
    }
    if A.domain.member((i,j)) {
      witness.push_back(A(i,j));
    }
    if B.domain.member((i,j)) {
      witness.push_back(B(i,j));
    }
    mini = min reduce witness;
    if mini < 1000000000 {
      sps += (i,j);
      T(i,j) = mini;
    }
  }
  return T;
}

proc sparseEq(A:[], B:[]) {
  var s: bool = true;
  forall (i,j) in A.domain with (&& reduce s) {
    s &&= A(i,j) == B(i,j);
  }
  return s;
}

proc tropicLimit(A:[] real,B:[] real): A.type {
 var R = tropic(A,B);
 if A.domain == R.domain {
   if sparseEq(R,A) {
     return R;
   } else {
     return tropicLimit(R,B);
   }
 } else {
   return tropicLimit(R,B);
 }
}

/*
 Build a random sparse matrix.  Good for testing;
 */
proc generateRandomSparseMatrix(size: int, sparsity: real) {
  const D: domain(2) = {1..size, 1..size};
  var SD: sparse subdomain(D) dmapped CS();
  var R: [SD] real;
  var da: domain(1) = {1..size};
  var array: [da] int = 1..size;
  var dom: domain(1) = {1..0};
  var indices: [dom] 2*int;
  var N = floor(size*(1-sparsity)): int;
  forall i in array {
    forall j in array {
      indices.push_back((i,j));
    }
  }
  shuffle(indices);
  var sparseids = indices[1..N];
  SD.bulkAdd(sparseids);
  forall (i,j) in sparseids {
    R(i,j) = 1;
  }
  //ndices = zip(array1, array2)
  return R;
}

/*
 General sparsity function for any `real` matrix.
 */
proc sparsity(X) {
  const d = X.shape[1]:real * X.shape[2]: real;
  var i: real = 0.0;
  for ij in X.domain {
    i += 1.0;
  }
  return i / d;
}

/*
 Error classes
 */
class NumSuchError : Error {
  proc init() {
    super.init();
    this.complete();
  }
  proc message() {
    return "Generic NumSuch Error";
  }
}

/*
 Used to indidcate dimension mismatches on NamedMatrices and vectors
 */
class DimensionMatchError : NumSuchError {
  var expected: int,
      actual: int;

  proc init(expected: int, actual:int) {
    super.init();
    this.complete();
    this.expected = expected;
    this.actual = actual;
  }

  proc message() {
    return "Error matching dimensions.  Expected: " + this.expected + " Actual: " + this.actual;
  }
}

use LinearAlgebra,
    LinearAlgebra.Sparse,
    NumSuch,
    Random;

config param batchsize = 10000;

/*
 A matrix of data with named rows and columns.
 */
class NamedMatrix {
  var D: domain(2),
      SD = CSRDomain(D),
      X: [SD] real,  // the actual data
      rows: BiMap = new BiMap(),
      cols: BiMap = new BiMap();

   proc init() {
     this.complete(); // NAMEDMATRIX DOESNT EXPLICITLY INHERIT ANYTHING SO IS THIS NECESSARY??
   }

   proc init(X) {
     this.D = {X.domain.dim(1), X.domain.dim(2)};
     this.complete();
     this.loadX(X);
   }

   proc init(X, names: [] string) {
     this.init(X);
     try! this.setRowNames(names);
     try! this.setColNames(names);
   }

   proc init(X, rownames: [] string, colnames: [] string) {
     this.init(X);
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(rownames: [] string, colnames: [] string) {
     this.D = {1..rownames.size, 1..colnames.size};
     this.complete();
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(N: NamedMatrix) {
     this.init(N.X);
     this.rows = N.rows;
     this.cols = N.cols;
   }

   proc init(X, rows: BiMap, cols: BiMap) {
     this.init(X);
     this.rows = rows;
     this.cols = cols;
   }
}

/*
Returns the number of rows in the matrix frame
 */
proc NamedMatrix.nrows() {
  return X.domain.dim(1).size;
}

/*
Returns the number of columns in the matrix frame
 */
proc NamedMatrix.ncols() {
  return X.domain.dim(2).size;
}

/*
 Finds the max along the row indicated by row number
 */

proc NamedMatrix.rowMax(i: int) {
  return aMax(this.X, 1)[i];
}
proc NamedMatrix.rowMax(s: string) {
  return this.rowMax(this.rows.get(s));
}

proc NamedMatrix.rowMax() {
  return aMax(this.X, 1);
}

proc NamedMatrix.colMax(i: int) {
  return aMax(this.X, 2)[i];
}
proc NamedMatrix.colMax(s: string) {
  return this.colMax(this.cols.get(s));
}

proc NamedMatrix.colMax() {
  return aMax(this.X, 2);
}

proc NamedMatrix.rowArgMax(i: int) {
  return argMax(this.X, 1)[i];
}
proc NamedMatrix.rowArgMax(s: string) {
  return this.rowArgMax(this.rows.get(s));
}

proc NamedMatrix.rowArgMax() {
  return argMax(this.X, 1);
}

proc NamedMatrix.colArgMax(i: int) {
  return argMax(this.X, 2)[i];
}
proc NamedMatrix.colArgMax(s: string) {
  return this.colArgMax(this.cols.get(s));
}

proc NamedMatrix.colArgMax() {
  return argMax(this.X, 2);
}

proc NamedMatrix.rowMin(i: int) {
  return aMin(this.X, 1)[i];
}
proc NamedMatrix.rowMin(s: string) {
  return this.rowMin(this.rows.get(s));
}

proc NamedMatrix.rowMin() {
  return aMin(this.X, 1);
}

proc NamedMatrix.colMin(i: int) {
  return aMin(this.X, 2)[i];
}
proc NamedMatrix.colMin(s: string) {
  return this.colMin(this.cols.get(s));
}

proc NamedMatrix.colMin() {
  return aMin(this.X, 2);
}

proc NamedMatrix.rowArgMin(i:int) {
  return argMin(this.X, 1)[i];
}
proc NamedMatrix.rowArgMin(s: string) {
  return this.rowArgMin(this.rows.get(s));
}

proc NamedMatrix.rowArgMin() {
  return argMin(this.X, 1);
}

proc NamedMatrix.colArgMin(i) {
  return argMin(this.X, 2)[i];
}
proc NamedMatrix.colArgMin(s: string) {
  return this.colArgMin(this.cols.get(s));
}

proc NamedMatrix.colArgMin() {
  return argMin(this.X, 2);
}

/* We don't have a good matPlus for Q + E yet, so doing this element
  by element. kinda sucks.
 */
proc NamedMatrix.matPlus(Y: NamedMatrix) {
  forall ij in Y.SD {
    this.X[ij] += Y.X[ij];
  }
}

/*
 Multiplies the underlying matrix in place.
 */
proc NamedMatrix.eTimes(a: real) {
  this.X = a * this.X;
}

/*
 Prints the matrix with row / column headers, not recommended for large matrices.
 Also width of entries should not exceed about 8 places.
 */
proc NamedMatrix.pprint() {
  write("     ");
  for c in this.cols.sorted() {
    write("       %5s  |".format(c[1]));
  }
  write("\n");
  for r in this.rows.sorted() {
    write(" %3s ".format(r[1]));
    for c in this.cols.sorted() {
      write(" %13.3dr ".format(this.get(r[2], c[2])));
    }
    write("\n");
  }

}

//proc NamedMatrix.argRowMax()

/*
Loads the data from X into the internal array, also called X.  We call them all X to keep it clear.

:arg real[]: Array representing the matrix
 */
proc NamedMatrix.loadX(X:[], shape: 2*int =(-1,-1)) {
  if shape(1) > 0 && shape(2) > 0 {
    this.D = {1..shape(1), 1..shape(2)};
  }
  for (i,j) in X.domain {
    this.SD += (i,j);
    this.X(i,j) = X(i,j);
  }
}

/*
Sets the row names for the matrix X
 */
proc NamedMatrix.setRowNames(rn:[]): string throws {
  //if (X.domain.dim(1).size > 0) && (rn.size != X.domain.dim(1).size) {
  //if (1 > 0) && (rn.size != X.domain.dim(1).size) {
  if rn.size != X.domain.dim(1).size {
    const err = new DimensionMatchError(expected = X.domain.dim(1).size, actual=rn.size);
    throw err;
    return "";
  } else {
    for i in 1..rn.size {
      this.rows.add(rn[i]);
    }
    return this.rows;
  }
}

/*
Sets the column names for the matrix X
*/
proc NamedMatrix.setColNames(cn:[]): string throws {
  if cn.size != X.domain.dim(2).size {
    const err = new DimensionMatchError(expected = X.domain.dim(2).size, actual=cn.size);
    throw err;
    return "";
  } else {
    for i in 1..cn.size {
      this.cols.add(cn[i]);
    }
  }
  return this.cols;
}

/*
 Gets the value of the (i,j) entry of the matrix X in the NamedMatrix
 */
proc NamedMatrix.get(i: int, j: int) {
   return this.X(i,j);
 }

 /*
 Get the values using the named rows and columns
  */
proc NamedMatrix.get(f: string, t: string) {
    return this.X(rows.get(f), cols.get(t));
 }

/*
 Set the value of this.X(i,j) by index row
 */
proc NamedMatrix.set(i: int, j: int, w: real) {
    if this.SD.member((i,j)) {
      this.X(i,j) = w;
    } else {
      this.SD += (i,j);
      this.X(i,j) = w;
    }
    return w;
}

/*
Set the values using the row and column names
*/
proc NamedMatrix.set(f: string, t: string, w: real) {
  return this.set(rows.get(f), cols.get(t), w);
}

/*
Update the value in X(i,j) with `w`
 */
proc NamedMatrix.update(i: int, j: int, w: real) {
  const x = this.get(i,j);
  return this.set(i,j, x + w);
}

/*
Update the value in X(i,j) with `w` using the row column names. This adds
`w` to the current value. To replace the current value, use `.set()`
 */
proc NamedMatrix.update(f: string, t: string, w: real) {
  const x = this.get(rows.get(f),cols.get(t));
  return this.set(rows.get(f),cols.get(t), x + w);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column ids
 */
proc NamedMatrix.remove(i: int, j:int) {
  this.SD -= (i,j);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column names
 */
proc NamedMatrix.remove(i: string, j:string) {
  this.remove(this.rows.get(i), this.cols.get(j));
}

/*
Find the number of non-zeroes in the matrix
 */
proc NamedMatrix.nnz() {
  var i: int = 0;
  for ij in this.X.domain {
    i += 1;
  }
  return i;
}

/*
Calculates the sparsity of the matrix: Number of entries / frame size
 */
proc NamedMatrix.sparsity() {
  const d = this.X.shape[1]:real * this.X.shape[2]: real;
  return this.nnz():real / d;
}

/*
 Takes a index (i,j) tuple and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(ij: 2*int) {
  return this.ncols() * ij[1] + ij[2] - this.ncols();
}
/*
 Takes a index values `i,j` and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(i: int, j: int) {
  return this.grid2seq((i,j));
}

/*
 Takes a sequential value and gives the row / column indices of that entry
 */
proc NamedMatrix.seq2grid(k: int) {
  return (divfloor(k, this.ncols())+1, k % this.ncols());
}

proc NamedMatrix.ntropic(N: NamedMatrix) {
  var T: NamedMatrix = new NamedMatrix(X = tropic(this.X, N.X), this.rows, N.cols);
  return T;
}

/*
 Multiplies the current NamedMatrix `X` against the argument `Y`, but first it aligns
 the names of `X.cols` with `Y.rows`.  Returns an appropriately named NamedMatrix
 :arg NamedMatrix Y:
 */
proc NamedMatrix.ndot(N: NamedMatrix) {
  var C: NamedMatrix = new NamedMatrix(X = dot(this.X,N.X), this.rows, N.cols);
  return C;
}

/*
 Routine to take two NamedMatrices and multiply them along just the row/column
 intersections.  If X has cols [red yellow blue green] and Y has rows [red blue green]
 this function produces a NamedMatrix with rows [red blue green]
 */
proc NamedMatrix.alignAndMultiply(Y: NamedMatrix) {
    var rcOverlap: domain(string) = this.cols.keys & Y.rows.keys;

    var xSD: sparse subdomain(this.D) dmapped CS(),
        ySD: sparse subdomain(Y.D) dmapped CS(),
        xX: [xSD] real,  // the actual data
        yX: [ySD] real,  // the actual data
        xrows: BiMap = new BiMap(),
        yrows: BiMap = new BiMap(),
        xcols: BiMap = new BiMap(),
        ycols: BiMap = new BiMap();

    for (left_row, right_col) in zip(this.D.dim(2), Y.D.dim(1)) {
      for rc in rcOverlap {
        const j = this.cols.get(rc);
        if this.SD.member(left_row, j) {
          xSD += (left_row, j);
          xX[left_row, j] = this.X[left_row, j];
        }
        if Y.SD.member(j, right_col) {
          ySD += (j, right_col);
          yX[j, right_col] = Y.X[j, right_col];
        }
      }
    }
    const z = xX.dot(yX);
    var n = new NamedMatrix(X=z);
    n.rows = this.rows;
    n.cols = Y.cols;
    return n;
}

/*
 Designed to be similar to SciPy's `amax() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amax.html>`_
 Useful on sparse matrices

 We'll get around to documenting here it eventually
 */
proc aMax(X:[], axis = 0) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    maxima[1] = (max reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  }
  return maxima;
}

/*
 Designed to be similar to SciPy's `amin() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amin.html>`_
 Useful on sparse matrices.

 We'll get around to documenting here it eventually
 */
proc aMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    minima[1] = (min reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  }
  return minima;
}

/*
Provides the "where" of the min along an axis in a sparse matrix
 */
proc argMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    minima[1] = miniLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = miniLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = miniLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  }
  return minima;
}

/*
 Provides the "where" of a max along a given axis in a sparse matrix
 */
proc argMax(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    maxima[1] = maxiLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = maxiLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = maxiLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  }
  return maxima;
}

// HELPER FUNCTIONS SHOULD NOT BE EXPOSED TO USER
proc miniLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      minimum: real = 1000000;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) < minimum {
        minimum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) < minimum {
        minimum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) < minimum {
        minimum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc maxiLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      maximum: real;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) > maximum {
        maximum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) > maximum {
        maximum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) > maximum {
        maximum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc sparseEquals(A:[], B:[]) {}

proc tropic(A:[],B:[]) { // UNDER CONSTRUCTION
  var dom: domain(2) = {A.domain.dim(1),B.domain.dim(2)};
  var sps = CSRDomain(dom);
  var BT = transpose(B);
  var T: [sps] real;
  for (i,j) in dom {
    var mini = 100000000: real;
    var wdom: domain(1);
    var wids: [wdom] int;
    for w in A.domain.dimIter(2,i) {
      wids.push_back(w);
    }
    for w2 in BT.domain.dimIter(2,j) {
      wids.push_back(w2);
    }
    var wDom: domain(1);
    var witness: [wDom] real;
    for w in wids {
      if A(i,w) > 0 && BT(j,w) > 0 {
        witness.push_back(A(i,w) + BT(j,w));
      }
    }
    if A.domain.member((i,j)) {
      witness.push_back(A(i,j));
    }
    if B.domain.member((i,j)) {
      witness.push_back(B(i,j));
    }
    mini = min reduce witness;
    if mini < 1000000000 {
      sps += (i,j);
      T(i,j) = mini;
    }
  }
  return T;
}

proc sparseEq(A:[], B:[]) {
  var s: bool = true;
  forall (i,j) in A.domain with (&& reduce s) {
    s &&= A(i,j) == B(i,j);
  }
  return s;
}

proc tropicLimit(A:[] real,B:[] real): A.type {
 var R = tropic(A,B);
 if A.domain == R.domain {
   writeln("Domains are equal!");
   if sparseEq(R,A) {
     writeln("This is the Limit");
     return R;
   } else {
     writeln(R);
     writeln("Not yet Limiting");
     return tropicLimit(R,B);
   }
 } else {
   writeln(R);
   writeln("Domains weren't equal...");
   return tropicLimit(R,B);
 }
}

/*
 Build a random sparse matrix.  Good for testing;
 */
proc generateRandomSparseMatrix(size: int, sparsity: real) {
  const D: domain(2) = {1..size, 1..size};
  var SD: sparse subdomain(D) dmapped CS();
  var R: [SD] real;
  var da: domain(1) = {1..size};
  var array: [da] int = 1..size;
  var dom: domain(1) = {1..0};
  var indices: [dom] 2*int;
  var N = floor(size*(1-sparsity)): int;
  forall i in array {
    forall j in array {
      indices.push_back((i,j));
    }
  }
  shuffle(indices);
  var sparseids = indices[1..N];
  SD.bulkAdd(sparseids);
  forall (i,j) in sparseids {
    R(i,j) = 1;
  }
  //ndices = zip(array1, array2)
  return R;
}

/*
 General sparsity function for any `real` matrix.
 */
proc sparsity(X) {
  const d = X.shape[1]:real * X.shape[2]: real;
  var i: real = 0.0;
  for ij in X.domain {
    i += 1.0;
  }
  return i / d;
}

/*
 Error classes
 */
class NumSuchError : Error {
  proc init() {
    super.init();
    this.complete();
  }
  proc message() {
    return "Generic NumSuch Error";
  }
}

/*
 Used to indidcate dimension mismatches on NamedMatrices and vectors
 */
class DimensionMatchError : NumSuchError {
  var expected: int,
      actual: int;

  proc init(expected: int, actual:int) {
    super.init();
    this.complete();
    this.expected = expected;
    this.actual = actual;
  }

  proc message() {
    return "Error matching dimensions.  Expected: " + this.expected + " Actual: " + this.actual;
  }
}


removing a space

removing a space

use LinearAlgebra,
    LinearAlgebra.Sparse,
    NumSuch,
    Random;

config param batchsize = 10000;

/*
 A matrix of data with named rows and columns.
 */
class NamedMatrix {
  var D: domain(2),
      SD = CSRDomain(D),
      X: [SD] real,  // the actual data
      rows: BiMap = new BiMap(),
      cols: BiMap = new BiMap();

   proc init() {
     this.initDone(); // NAMEDMATRIX DOESNT EXPLICITLY INHERIT ANYTHING SO IS THIS NECESSARY??
   }

   proc init(X) {
     this.D = {X.domain.dim(1), X.domain.dim(2)};
     this.initDone();
     this.loadX(X);
   }

   proc init(X, names: [] string) {
     this.init(X);
     try! this.setRowNames(names);
     try! this.setColNames(names);
   }

   proc init(X, rownames: [] string, colnames: [] string) {
     this.init(X);
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(rownames: [] string, colnames: [] string) {
     this.D = {1..rownames.size, 1..colnames.size};
     this.initDone();
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(N: NamedMatrix) {
     this.init(N.X);
     this.rows = N.rows;
     this.cols = N.cols;
   }

   proc init(X, rows: BiMap, cols: BiMap) {
     this.init(X);
     this.rows = rows;
     this.cols = cols;
   }
}

/*
Returns the number of rows in the matrix frame
 */
proc NamedMatrix.nrows() {
  return X.domain.dim(1).size;
}

/*
Returns the number of columns in the matrix frame
 */
proc NamedMatrix.ncols() {
  return X.domain.dim(2).size;
}

/*
 Finds the max along the row indicated by row number
 */
proc NamedMatrix.rowMax(i: int) {
  var x:[1..0] real;
  for a in this.X.domain.dimIter(2,i) {
    x.push_back(this.get(i,a));
  }
  if x.size < 1 {
    return NAN;
  } else {
    return max reduce x;
  }
}

/*
 Returns the max of each row
 */
proc NamedMatrix.rowMax() {
  var mxs: [1..this.nrows()] real;
  forall i in 1..this.nrows() {
    mxs[i] = this.rowMax(i);
  }
  return mxs;
}

/*
Loads the data from X into the internal array, also called X.  We call them all X to keep it clear.

:arg real[]: Array representing the matrix
 */
proc NamedMatrix.loadX(X:[], shape: 2*int =(-1,-1)) {
  if shape(1) > 0 && shape(2) > 0 {
    this.D = {1..shape(1), 1..shape(2)};
  }
  for (i,j) in X.domain {
    this.SD += (i,j);
    this.X(i,j) = X(i,j);
  }
}

/*
Sets the row names for the matrix X
 */
proc NamedMatrix.setRowNames(rn:[]): string throws {
  //if (X.domain.dim(1).size > 0) && (rn.size != X.domain.dim(1).size) {
  //if (1 > 0) && (rn.size != X.domain.dim(1).size) {
  if rn.size != X.domain.dim(1).size {
    const err = new DimensionMatchError(expected = X.domain.dim(1).size, actual=rn.size);
    throw err;
    return "";
  } else {
    for i in 1..rn.size {
      this.rows.add(rn[i]);
    }
    return this.rows;
  }
}

/*
Sets the column names for the matrix X
*/
proc NamedMatrix.setColNames(cn:[]): string throws {
  if cn.size != X.domain.dim(2).size then throw new Error();
  for i in 1..cn.size {
    this.cols.add(cn[i]);
  }
  return this.cols;
}

/*
 Gets the value of the (i,j) entry of the matrix X in the NamedMatrix
 */
proc NamedMatrix.get(i: int, j: int) {
   return this.X(i,j);
 }

 /*
 Get the values using the named rows and columns
  */
proc NamedMatrix.get(f: string, t: string) {
    return this.X(rows.get(f), cols.get(t));
 }

/*
 Set the value of this.X(i,j) by index row
 */
proc NamedMatrix.set(i: int, j: int, w: real) {
    if this.SD.member((i,j)) {
      this.X(i,j) = w;
    } else {
      this.SD += (i,j);
      this.X(i,j) = w;
    }
    return w;
}

/*
Set the values using the row and column names
*/
proc NamedMatrix.set(f: string, t: string, w: real) {
  return this.set(rows.get(f), cols.get(t), w);
}

/*
Update the value in X(i,j) with `w`
 */
proc NamedMatrix.update(i: int, j: int, w: real) {
  const x = this.get(i,j);
  return this.set(i,j, x + w);
}

/*
Update the value in X(i,j) with `w` using the row column names
 */
proc NamedMatrix.update(f: string, t: string, w: real) {
  const x = this.get(rows.get(f),cols.get(t));
  return this.set(rows.get(f),cols.get(t), x + w);
}

/*
Find the number of non-zeroes in the matrix
 */
proc NamedMatrix.nnz() {
  var i: int = 0;
  for ij in this.X.domain {
    i += 1;
  }
  return i;
}

/*
Calculates the sparsity of the matrix: Number of entries / frame size
 */
proc NamedMatrix.sparsity() {
  const d = this.X.shape[1]:real * this.X.shape[2]: real;
  return this.nnz():real / d;
}

/*
 Multiplies the current NamedMatrix `X` against the argument `Y`, but first it aligns
 the names of `X.cols` with `Y.rows`.  Returns an appropriately named NamedMatrix
 :arg NamedMatrix Y:
 */

proc NamedMatrix.ndot(N: NamedMatrix) {
  var C: NamedMatrix = new NamedMatrix(X = dot(this.X,N.X), this.rows, N.cols);
  return C;
}

proc NamedMatrix.alignAndMultiply(Y: NamedMatrix) {
    var rcOverlap: domain(string) = this.cols.keys & Y.rows.keys;

    var xSD: sparse subdomain(this.D) dmapped CS(),
        ySD: sparse subdomain(Y.D) dmapped CS(),
        xX: [xSD] real,  // the actual data
        yX: [ySD] real,  // the actual data
        xrows: BiMap = new BiMap(),
        yrows: BiMap = new BiMap(),
        xcols: BiMap = new BiMap(),
        ycols: BiMap = new BiMap();

    for (left_row, right_col) in zip(this.D.dim(2), Y.D.dim(1)) {
      for rc in rcOverlap {
        const j = this.cols.get(rc);
        if this.SD.member(left_row, j) {
          xSD += (left_row, j);
          xX[left_row, j] = this.X[left_row, j];
        }
        if Y.SD.member(j, right_col) {
          ySD += (j, right_col);
          yX[j, right_col] = Y.X[j, right_col];
        }
      }
    }
    const z = xX.dot(yX);
    var n = new NamedMatrix(X=z);
    n.rows = this.rows;
    n.cols = Y.cols;
    return n;
}

proc argmaxi(axis:int, id:int , X:[]) {
  var idm: (int,int),
      maximum: real = 0;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) > maximum {
        maximum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) > maximum {
        maximum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) > maximum {
        maximum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return maximum;
}

proc maxiloc(axis:int, id:int , X:[]) {
  var idm: (int,int),
      maximum: real = 0;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) > maximum {
        maximum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) > maximum {
        maximum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) > maximum {
        maximum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc argmini(axis:int, id:int , X:[]) {
  var idm: (int,int),
      minimum: real = 0;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) < minimum {
        minimum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) < minimum {
        minimum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) < minimum {
        minimum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return minimum;
}

proc miniloc(axis:int, id:int , X:[]) {
  var idm: (int,int),
      minimum: real = 0;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) < minimum {
        minimum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) < minimum {
        minimum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) < minimum {
        minimum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc tropic(A:[],B:[]) { // UNDER CONSTRUCTION
  writeln("Calculating Tropic Product");
  writeln("A is");
  writeln(A);
  writeln("With domain");
  writeln(A.domain);
  var dom: domain(2) = {A.domain.dim(1),B.domain.dim(2)};
  var sps = CSRDomain(dom);
  var BT = transpose(B);
  writeln("BT is");
  writeln(BT);
  writeln("With domain");
  writeln(BT.domain);
  writeln("Tropic will have entries in");
  writeln(dom);
  var T: [sps] real;
  for (i,j) in dom {
    var mini = 100000000: real;
    var wdom: domain(1);
    var wids: [wdom] int;
    for w in A.domain.dimIter(2,i) {
      wids.push_back(w);
      writeln("Witness in A from %n to %n".format(i,w));
    }
    writeln(wids);
    for w in BT.domain.dimIter(2,j) {
      wids.push_back(w);
      writeln("Witness in B from %n to %n".format(w,j));
    }
    writeln(wids);
    var wDom: domain(1);
    var witness: [wDom] real;
    for w in wids {
      if A(i,w) + BT(j,w) > 0 {
        witness.push_back(A(i,w) + BT(j,w));
      }
    }
    writeln("Witnesses are");
    writeln(witness);
    mini = min reduce witness;
    writeln("Minimum among witnesses is");
    writeln(mini);
    if mini < 1000000000 {
      T(i,j) = mini;
    }
  }
  return T;
}

/*
 Build a random sparse matrix.  Good for testing;
 */
proc generateRandomSparseMatrix(size: int, sparsity: real) {
  const D: domain(2) = {1..size, 1..size};
  var SD: sparse subdomain(D) dmapped CS();
  var R: [SD] real;
  var da: domain(1) = {1..size};
  var array: [da] int = 1..size;
  var dom: domain(1) = {1..0};
  var indices: [dom] 2*int;
  var N = floor(size*(1-sparsity)): int;
  forall i in array {
    forall j in array {
      indices.push_back((i,j));
    }
  }
  shuffle(indices);
  var sparseids = indices[1..N];
  SD.bulkAdd(sparseids);
  forall (i,j) in sparseids {
    R(i,j) = 1;
  }
  //ndices = zip(array1, array2)
  return R;
}

proc sparsity(X) {
  const d = X.shape[1]:real * X.shape[2]: real;
  var i: real = 0.0;
  for ij in X.domain {
    i += 1.0;
  }
  return i / d;
}

class NumSuchError : Error {
  proc init() {
    super.init();
    this.initDone();
  }
  proc message() {
    return "Generic NumSuch Error";
  }
}

class DimensionMatchError : NumSuchError {
  var expected: int,
      actual: int;

  proc init(expected: int, actual:int) {
    super.init();
    this.initDone();
    this.expected = expected;
    this.actual = actual;
  }

  proc message() {
    return "Error matching dimensions.  Expected: " + this.expected + " Actual: " + this.actual;
  }
}