This test had some indexing backwards

This test had some indexing backwards

This was caught by my experimental Array View work which apparently
does better bounds checking than the current state of the world does.

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// Use the distributions we need for this computation
//
use BlockCycDist;

//
// The number of matrices and the element type of those matrices
// indexType can be int or int(64), elemType can be real or complex
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 8;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
proc main() {
  printConfiguration();

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  const MatVectSpace: domain(2, indexType)
                      dmapped BlockCyclic(startIdx=(1,1), (blkSize,blkSize))
                    = {1..n, 1..n+1},
        MatrixSpace = MatVectSpace[.., ..n];

  var Ab : [MatVectSpace] elemType,  // the matrix A and vector b
      piv: [1..n] indexType;         // a vector of pivot values

  initAB(Ab);

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, Ab, piv);                 // compute the LU factorization

  var x = backwardSub(n, Ab);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(Ab, MatrixSpace, x);
  printResults(validAnswer, execTime);
}

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
proc LUFactorize(n: indexType, Ab: [?AbD] elemType,
                piv: [1..n] indexType) {

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(Ab, l, piv);
    updateBlockRow(Ab, tl, tr);

    //
    // update trailing submatrix (if any)
    //
    schurComplement(Ab, bl, tr, br);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The 'a' region was 'bl' in the calling
//     |aaaaa|.....|.....|.....|  function but called AD here.  Similarly,
//     +-----+-----+-----+-----+  'b' was 'tr' in the calling code, but BD
//     |aaaaa|.....|.....|.....|  here.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
proc schurComplement(Ab: [?AbD] elemType, AD: domain, BD: domain, Rest: domain) {
  //
  // Copy data into replicated array so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.  These
  // replicated distributions aren't implemented yet, but imagine that
  // they look something like the following:
  //
  //var replAbD: domain(2)
  //            dmapped new Dimensional(BlkCyc(blkSize), Replicated)) = AbD[AD];
  //
  const replAD: domain(2, indexType) = AD,
        replBD: domain(2, indexType) = BD;

  const replA : [replAD] elemType = Ab[replAD],
        replB : [replBD] elemType = Ab[replBD];

  //  writeln("Rest = ", Rest);
  //  writeln("Rest by blkSize = ", Rest by (blkSize, blkSize));
  // do local matrix-multiply on a block-by-block basis
  // TODO: re-enable parallel loop
  forall (row,col) in Rest by (blkSize, blkSize) {
    //for (row,col) in Rest by (blkSize, blkSize) {
    //
    // At this point, the dgemms should all be local, so assert that
    // fact
    //
    //    writeln("On ", here.id, ", trying to access ",
    //            (row..#blkSize, col..#blkSize));

    // TODO: enable this on distributed memory
    //    local {
      const aBlkD = replAD[row..#blkSize, ..],
            bBlkD = replBD[.., col..#blkSize],
            cBlkD = AbD[row..#blkSize, col..#blkSize];

      dgemmNativeInds(replA[aBlkD], replB[bBlkD], Ab[cBlkD]);
      /*
      dgemmReindexed(cBlkD.dim(1).length, aBlkD.dim(2).length, cBlkD.dim(2).length,
                     replA[aBlkD], replB[bBlkD], Ab[cBlkD]);
      */
      /*
      dgemmIdeal(replA[aBlkD], replB[bBlkD], Ab[cBlkD]);
      */
      //    }
  }
}

//
// calculate C = C - A * B.
//
proc dgemmNativeInds(A: [] elemType,
                    B: [] elemType,
                    C: [] elemType) {
  for (iA, iC) in zip(A.domain.dim(1), C.domain.dim(1)) do
    for (jA, iB) in zip(A.domain.dim(2), B.domain.dim(1)) do
      for (jB, jC) in zip(B.domain.dim(2), C.domain.dim(2)) do
        C[iC,jC] -= A[iA, jA] * B[iB, jB];
}

proc dgemmReindexed(p: indexType,    // number of rows in A
                   q: indexType,    // number of cols in A, number of rows in B
                   r: indexType,    // number of cols in B
                   A: [1..p, 1..q] elemType,
                   B: [1..q, 1..r] elemType,
                   C: [1..p, 1..r] elemType) {
  // Calculate (i,j) using a dot product of a row of A and a column of B.
  for i in 1..p do
    for j in 1..r do
      for k in 1..q do
        C[i,j] -= A[i, k] * B[k, j];
}

proc dgemmIdeal(A: [1.., 1..] elemType,
               B: [1.., 1..] elemType,
               C: [1.., 1..] elemType) {
  for i in C.domain.dim(1) do
    for j in C.domain.dim(2) do
      for k in A.domain.dim(2) do
        C[i,j] -= A[i, k] * B[k, j];
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
proc panelSolve(Ab: [] elemType,
               panel: domain,
               piv: [] indexType) {

  //
  // TODO: Use on clause here (or avoid using a range?
  //
  for k in panel.dim(2) {             // iterate through the columns
    var col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.numIndices == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const (_, (pivotRow, _)) = maxloc reduce zip(abs(Ab(col)), col);

    // Capture the pivot value explicitly (note that result of maxloc
    // is absolute value, so it can't be used directly).
    //
    // TODO: could introduce a maxabsloc reduction that returns the
    //       raw value with the highest absolute value
    const pivotVal = Ab[pivotRow, k];

    // Swap the current row with the pivot row and update the pivot vector
    // to reflect that
    Ab[k..k, ..] <=> Ab[pivotRow..pivotRow, ..];
    piv[k] <=> piv[pivotRow];

    if (pivotVal == 0) then
      halt("Matrix cannot be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot value
    Ab[k+1.., k..k] /= pivotVal;

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      Ab[i,j] -= Ab[i,k] * Ab[k,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
proc updateBlockRow(Ab: [] elemType,
                   tl: domain,
                   tr: domain) {

  for row in tr.dim(1) {
    const activeRow = tr[row..row, ..],
          prevRows = tr.dim(1).low..row-1;

    forall (i,j) in activeRow do
      for k in prevRows do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
  }
}

//
// compute the backwards substitution
//
proc backwardSub(n: indexType,
                 Ab: [] elemType) {
  const bd = Ab.domain.dim(1);
  var x: [bd] elemType;

  // TODO: Really want a partial reduction here
  for i in bd by -1 do
    x[i] = (Ab[i,n+1] - (+ reduce [j in i+1..bd.high] (Ab[i,j] * x[j])))
            / Ab[i,i];

  return x;
}

//
// print out the problem size and block size if requested
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
proc initAB(Ab: [] elemType) {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
proc verifyResults(Ab, MatrixSpace, x) {
  initAB(Ab);

  const axmbNorm = norm(gaxpyMinus(Ab[.., 1..n], x, Ab[.., n+1..n+1]), normType.normInf);

  const a1norm   = norm(Ab[.., 1..n], normType.norm1),
        aInfNorm = norm(Ab[.., 1..n], normType.normInf),
        x1Norm   = norm(Ab[.., n+1..n+1], normType.norm1),
        xInfNorm = norm(Ab[.., n+1..n+1], normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 1e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
proc gaxpyMinus(A: [],
                x: [?xD],
                y: [?yD]) {
  var res: [1..n] elemType;

  // TODO: really want a partial reduction here
  forall i in 1..n do
    res[i] = (+ reduce [j in xD] (A[i,j] * x[j])) - y[i,n+1];

  return res;
}

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// Use the distributions we need for this computation
//
use BlockCycDist;

//
// The number of matrices and the element type of those matrices
// indexType can be int or int(64), elemType can be real or complex
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 8;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
proc main() {
  printConfiguration();

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  const MatVectSpace: domain(2, indexType)
                      dmapped BlockCyclic(startIdx=(1,1), (blkSize,blkSize))
                    = {1..n, 1..n+1},
        MatrixSpace = MatVectSpace[.., ..n];

  var Ab : [MatVectSpace] elemType,  // the matrix A and vector b
      piv: [1..n] indexType;         // a vector of pivot values

  initAB(Ab);

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, Ab, piv);                 // compute the LU factorization

  var x = backwardSub(n, Ab);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(Ab, MatrixSpace, x);
  printResults(validAnswer, execTime);
}

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
proc LUFactorize(n: indexType, Ab: [?AbD] elemType,
                piv: [1..n] indexType) {

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(Ab, l, piv);
    updateBlockRow(Ab, tl, tr);

    //
    // update trailing submatrix (if any)
    //
    schurComplement(Ab, bl, tr, br);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The 'a' region was 'bl' in the calling
//     |aaaaa|.....|.....|.....|  function but called AD here.  Similarly,
//     +-----+-----+-----+-----+  'b' was 'tr' in the calling code, but BD
//     |aaaaa|.....|.....|.....|  here.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
proc schurComplement(Ab: [?AbD] elemType, AD: domain, BD: domain, Rest: domain) {
  //
  // Copy data into replicated array so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.  These
  // replicated distributions aren't implemented yet, but imagine that
  // they look something like the following:
  //
  //var replAbD: domain(2)
  //            dmapped new Dimensional(BlkCyc(blkSize), Replicated)) = AbD[AD];
  //
  const replAD: domain(2, indexType) = AD,
        replBD: domain(2, indexType) = BD;

  const replA : [replAD] elemType = Ab[replAD],
        replB : [replBD] elemType = Ab[replBD];

  //  writeln("Rest = ", Rest);
  //  writeln("Rest by blkSize = ", Rest by (blkSize, blkSize));
  // do local matrix-multiply on a block-by-block basis
  // TODO: re-enable parallel loop
  forall (row,col) in Rest by (blkSize, blkSize) {
    //for (row,col) in Rest by (blkSize, blkSize) {
    //
    // At this point, the dgemms should all be local, so assert that
    // fact
    //
    //    writeln("On ", here.id, ", trying to access ",
    //            (row..#blkSize, col..#blkSize));

    // TODO: enable this on distributed memory
    //    local {
      const aBlkD = replAD[row..#blkSize, ..],
            bBlkD = replBD[.., col..#blkSize],
            cBlkD = AbD[row..#blkSize, col..#blkSize];

      dgemmNativeInds(replA[aBlkD], replB[bBlkD], Ab[cBlkD]);
      /*
      dgemmReindexed(cBlkD.dim(1).length, aBlkD.dim(2).length, cBlkD.dim(2).length,
                     replA[aBlkD], replB[bBlkD], Ab[cBlkD]);
      */
      /*
      dgemmIdeal(replA[aBlkD], replB[bBlkD], Ab[cBlkD]);
      */
      //    }
  }
}

//
// calculate C = C - A * B.
//
proc dgemmNativeInds(A: [] elemType,
                    B: [] elemType,
                    C: [] elemType) {
  for (iA, iC) in zip(A.domain.dim(1), C.domain.dim(1)) do
    for (jA, iB) in zip(A.domain.dim(2), B.domain.dim(1)) do
      for (jB, jC) in zip(B.domain.dim(2), C.domain.dim(2)) do
        C[iC,jC] -= A[iA, jA] * B[iB, jB];
}

proc dgemmReindexed(p: indexType,    // number of rows in A
                   q: indexType,    // number of cols in A, number of rows in B
                   r: indexType,    // number of cols in B
                   A: [1..p, 1..q] elemType,
                   B: [1..q, 1..r] elemType,
                   C: [1..p, 1..r] elemType) {
  // Calculate (i,j) using a dot product of a row of A and a column of B.
  for i in 1..p do
    for j in 1..r do
      for k in 1..q do
        C[i,j] -= A[i, k] * B[k, j];
}

proc dgemmIdeal(A: [1.., 1..] elemType,
               B: [1.., 1..] elemType,
               C: [1.., 1..] elemType) {
  for i in C.domain.dim(1) do
    for j in C.domain.dim(2) do
      for k in A.domain.dim(2) do
        C[i,j] -= A[i, k] * B[k, j];
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
proc panelSolve(Ab: [] elemType,
               panel: domain,
               piv: [] indexType) {

  //
  // TODO: Use on clause here (or avoid using a range?
  //
  for k in panel.dim(2) {             // iterate through the columns
    var col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.numIndices == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const (_, (pivotRow, _)) = maxloc reduce zip(abs(Ab(col)), col);

    // Capture the pivot value explicitly (note that result of maxloc
    // is absolute value, so it can't be used directly).
    //
    // TODO: could introduce a maxabsloc reduction that returns the
    //       raw value with the highest absolute value
    const pivotVal = Ab[pivotRow, k];

    // Swap the current row with the pivot row and update the pivot vector
    // to reflect that
    Ab[k..k, ..] <=> Ab[pivotRow..pivotRow, ..];
    piv[k] <=> piv[pivotRow];

    if (pivotVal == 0) then
      halt("Matrix cannot be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot value
    Ab[k+1.., k..k] /= pivotVal;

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      Ab[i,j] -= Ab[i,k] * Ab[k,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
proc updateBlockRow(Ab: [] elemType,
                   tl: domain,
                   tr: domain) {

  for row in tr.dim(1) {
    const activeRow = tr[row..row, ..],
          prevRows = tr.dim(1).low..row-1;

    forall (i,j) in activeRow do
      for k in prevRows do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
  }
}

//
// compute the backwards substitution
//
proc backwardSub(n: indexType,
                 Ab: [] elemType) {
  const bd = Ab.domain.dim(1);
  var x: [bd] elemType;

  // TODO: Really want a partial reduction here
  for i in bd by -1 do
    x[i] = (Ab[n+1,i] - (+ reduce [j in i+1..bd.high] (Ab[i,j] * x[j])))
            / Ab[i,i];

  return x;
}

//
// print out the problem size and block size if requested
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
proc initAB(Ab: [] elemType) {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
proc verifyResults(Ab, MatrixSpace, x) {
  initAB(Ab);

  const axmbNorm = norm(gaxpyMinus(Ab[.., 1..n], x, Ab[.., n+1..n+1]), normType.normInf);

  const a1norm   = norm(Ab[.., 1..n], normType.norm1),
        aInfNorm = norm(Ab[.., 1..n], normType.normInf),
        x1Norm   = norm(Ab[.., n+1..n+1], normType.norm1),
        xInfNorm = norm(Ab[.., n+1..n+1], normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 1e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
proc gaxpyMinus(A: [],
                x: [?xD],
                y: [?yD]) {
  var res: [1..n] elemType;

  // TODO: really want a partial reduction here
  forall i in 1..n do
    res[i] = (+ reduce [j in xD] (A[i,j] * x[j])) - y[n+1, i];

  return res;
}


Make test locales.chpl give back allocated memory.

/*
 * Locales primer
 *
 * This example showcases Chapel's locale types.  To run this example
 * using multiple locales set up your envionment as described in
 * README.multilocale and execute it using the "-nl #" flag to specify
 * the number of locales.  For example, to run on 2 locales, run
 * "./a.out -nl 2".
 *
 */

use Memory;

//
// In Chapel, the 'locale' type refers to a unit of the machine
// resources on which your program is running.  Locales have the
// capability to store variables (i.e., they have memory) and to run
// Chapel tasks (i.e., they have processors).  The specific definition
// of 'locale' for a given target architecture is determined by the
// Chapel compiler and architecture.  In practice for a standard
// distributed memory architecture a single multicore/SMP node is
// typically considered a locale.  Shared memory architectures are
// typically considered a single locale.  See the notes in
// $CHPL_HOME/doc/platforms/ for platform-specific notes that might
// relate to locales on a given architecture.
//
// As mentioned above, the number of locales on which a Chapel program
// should be run is specified on the executable command line using the
// -nl <numLocales> (or --numLocales=<numLocales>) flag.  This will
// cause the Chapel's launcher to execute the program on that many
// locales.
//

//
// Within a Chapel program, the number of locales can be referred
// to symbolically using the built-in numLocales variable.  For
// example:
//
writeln("This program is running on ", numLocales, " locales");

//
// Chapel programs always start executing from locale #0.  The
// locale on which the current piece of code is executing can
// always be queried in a Chapel program using the builtin 'here'
// variable which has locale type.  All locale values query an
// 'id' query that returns its unique value from 0 to numLocales-1.
// Here we verify that this program began running on locale 0:
//

writeln("It began running on locale #", here.id);
writeln();

//
// Chapel also provides a built-in Locales array that represents the
// set of locale values corresponding to the machine resources on
// which the program is running.  This Locales array is the main
// way for bootstrapping distributed memory computations, whether
// explicitly (using on-clauses) or implicitly (using distributions).
//
// As an example, the following for loop iterates over the Locales
// array and then uses an on-clause to specify that each iteration
// should execute on that specific locale.  Within the body of the
// loop we use 'here' and the '.id' query to indicate what locale
// each iteration of the loop body is executing on.
//
for loc in Locales do
  on loc do
    writeln("hello locale ", here.id);
writeln();

//
// Note that in Chapel, locality and parallelism are orthogonal
// concepts.  That is, nothing about the use of the on-clause above
// introduced parallelism into the program, it only moved the current
// task (conceptually) from locale #0 to another locale.  Task and
// data parallel concepts can be intermixed with on-clauses to get
// parallelism across distributed memory machine resources, but in
// this example we focus only on the locality issues, not parallelism.
//

//
// Locale values can be assigned and stored like any other value.
// For example, the user can create their own view of the Locales
// array.  The following statement creates a 1..10 array of Locales
// in which we'll store our numLocales unique locale values
// redundantly if numLocales is < 10:
//
var MyLocaleArray: [1..10] locale;

for i in 1..10 do
  MyLocaleArray[i] = Locales[(i-1)%numLocales];

for i in 1..10 do
  on MyLocaleArray[i] do
    writeln("MyLocaleArray[", i, "] is really locale ", here.id);
writeln();

//
// Similarly, the user could arrange the locales into a
// multidimensional virtual locale grid by storing the locale
// values into a higher-dimensional array of locales (the array
// reshape() method can be particularly useful for this purpose).
//

//
// In addition to the .id query mentioned above, locales support a
// number of other queries about their properties.  For example:
//
//   locale.name     : returns a string indicating the locale's name
//   locale.numCores : returns the number of processor cores on the locale
//   locale.physicalMemory() : returns the amount of memory on the locale
//   locale.maxTaskPar : returns the likely maximum parallelism available
//                       on the locale
//

config const printLocaleInfo = true;  // permit testing to turn this off

if printLocaleInfo then
  for loc in Locales do
    on loc {
      writeln("locale #", here.id, "...");
      writeln("  ...is named: ", here.name);
      writeln("  ...has ", here.numCores, " processor cores");
      writeln("  ...has ", here.physicalMemory(unit=MemUnits.GB, retType=real),
              " GB of memory");
      writeln("  ...has ", here.maxTaskPar, " maximum parallelism");
    }
writeln();

//
// Chapel variables are stored using the memory of the locale
// executing the task that encounters the variable declaration.
// Thus, in the following code, x is declared on locale 0 and
// y is declared on locale 1 if numLocales > 1 (0 otherwise).
//

{
  var x: int = 2;
  on Locales[1 % numLocales] {
    var y: int = 3;
    writeln("From locale ", here.id, ", x is: ", x, " and y is: ", y);
    on Locales[0] {
      writeln("From locale 0, x is: ", x, " and y is: ", y);
    }
  }
  writeln();
}

//
// Note in the code above that a task can refer to any
// lexically-visible variable in Chapel regardless of the locales on
// which the variable is stored and the task is executing.  This is
// what is known as a global namespace (or global address space)
// quality in a language.  The fact that locality within a Chapel
// program can be semantically reasoned about by the programmer (i.e.,
// "x is on Locale[0], y is on Locale[1]") makes it a PGAS or
// Partitioned Global Address Space language (though we prefer the
// less broadly-used term "Partitioned Global Namespace").
//
// Not only can a programmer reason about a variable or task's
// location abstractly, they can also query these values directly
// in the language.  In particular, all Chapel variables support a
// .locale query which returns the locale value that is storing that
// variable.  Modifying the above example slightly:
//

{
  var x: int = 2;
  on Locales[1 % numLocales] {
    var y: int = 3;
    writeln("x is stored on locale ", x.locale.id, ", while y lives on ",
            y.locale.id);
  }
  writeln();
}

//
// Moreover, Chapel's on-clauses can be controlled by any expression
// with storage associated with it.  "on x" effectively says "execute
// this on whichever locale owns x" or equivalently "on x.locale".
// Here's a variation of the example above using this idiom:
//

{
  var x: int = 2;
  on Locales[1 % numLocales] {
    var y: int = 3;
    on x do
      writeln("Using a data-driven on-clause, I'm now executing on locale ",
              here.id);
  }
  writeln();
}

//
// Next, let's create a simple distributed data structure using the
// principles above.  In particular, we'll create a linked list that
// spans the locales with a list node per locale.
//

//
// First, define a standard Node class for linked lists.  Note that
// even though the 'next' fields will span locales, they don't need
// to be declared specially in any way, again thanks to the [P]GAS
// nature of the language.
//
class Node {
  var data: real;
  var next: Node;
}

//
// Next, we'll iterate through the locales, allocating a node on each.
// It's worth reviewing that a class variable (like the 'next' field
// above) is just a reference in Chapel and the object it points to
// is a separate thing allocated by a call to 'new'.  So in general,
// the reference and object can live on different locales (as they
// will here.
//
// So, in the following loop, 'current' will always point at the last
// node we created while our on-clause will move the task ahead to
// the next locale.  That way, our execution of 'new' will create
// the new object on that "next" locale.
//
var head = new Node(0);
var current = head;
for i in 1..numLocales-1 do
  on Locales[i] {
    current.next = new Node(i);
    current = current.next;
  }

//
// Loop over the linked list and output each node and the locale on
// which it exists using .locale.id to determine the locale.  This
// computation takes place entirely on Locales(0) and accesses remote
// memory as necessary.
//
current = head;
while current {
  writeln("node with data = ", current.data, " on locale ", current.locale.id);
  current = current.next;
}
writeln();

//
// Alternatively, we could use a data-driven execution to walk the
// linked list.  Note that on-clauses, when applied to class
// variables, resolve to the locale of the object, not the reference.
//
current = head;
while current {
  on current {
    writeln("node with data = ", current.data, " on locale ", here.id);
    current = current.next;
  }
}

//
// Give back the memory allocated by the node list.
//
delete head;

//
// For more information about Locales, refer to the Locales chapter of
// the language spec.
//

/*
 * Locales primer
 *
 * This example showcases Chapel's locale types.  To run this example
 * using multiple locales set up your envionment as described in
 * README.multilocale and execute it using the "-nl #" flag to specify
 * the number of locales.  For example, to run on 2 locales, run
 * "./a.out -nl 2".
 *
 */

use Memory;

//
// In Chapel, the 'locale' type refers to a unit of the machine
// resources on which your program is running.  Locales have the
// capability to store variables (i.e., they have memory) and to run
// Chapel tasks (i.e., they have processors).  The specific definition
// of 'locale' for a given target architecture is determined by the
// Chapel compiler and architecture.  In practice for a standard
// distributed memory architecture a single multicore/SMP node is
// typically considered a locale.  Shared memory architectures are
// typically considered a single locale.  See the notes in
// $CHPL_HOME/doc/platforms/ for platform-specific notes that might
// relate to locales on a given architecture.
//
// As mentioned above, the number of locales on which a Chapel program
// should be run is specified on the executable command line using the
// -nl <numLocales> (or --numLocales=<numLocales>) flag.  This will
// cause the Chapel's launcher to execute the program on that many
// locales.
//

//
// Within a Chapel program, the number of locales can be referred
// to symbolically using the built-in numLocales variable.  For
// example:
//
writeln("This program is running on ", numLocales, " locales");

//
// Chapel programs always start executing from locale #0.  The
// locale on which the current piece of code is executing can
// always be queried in a Chapel program using the builtin 'here'
// variable which has locale type.  All locale values query an
// 'id' query that returns its unique value from 0 to numLocales-1.
// Here we verify that this program began running on locale 0:
//

writeln("It began running on locale #", here.id);
writeln();

//
// Chapel also provides a built-in Locales array that represents the
// set of locale values corresponding to the machine resources on
// which the program is running.  This Locales array is the main
// way for bootstrapping distributed memory computations, whether
// explicitly (using on-clauses) or implicitly (using distributions).
//
// As an example, the following for loop iterates over the Locales
// array and then uses an on-clause to specify that each iteration
// should execute on that specific locale.  Within the body of the
// loop we use 'here' and the '.id' query to indicate what locale
// each iteration of the loop body is executing on.
//
for loc in Locales do
  on loc do
    writeln("hello locale ", here.id);
writeln();

//
// Note that in Chapel, locality and parallelism are orthogonal
// concepts.  That is, nothing about the use of the on-clause above
// introduced parallelism into the program, it only moved the current
// task (conceptually) from locale #0 to another locale.  Task and
// data parallel concepts can be intermixed with on-clauses to get
// parallelism across distributed memory machine resources, but in
// this example we focus only on the locality issues, not parallelism.
//

//
// Locale values can be assigned and stored like any other value.
// For example, the user can create their own view of the Locales
// array.  The following statement creates a 1..10 array of Locales
// in which we'll store our numLocales unique locale values
// redundantly if numLocales is < 10:
//
var MyLocaleArray: [1..10] locale;

for i in 1..10 do
  MyLocaleArray[i] = Locales[(i-1)%numLocales];

for i in 1..10 do
  on MyLocaleArray[i] do
    writeln("MyLocaleArray[", i, "] is really locale ", here.id);
writeln();

//
// Similarly, the user could arrange the locales into a
// multidimensional virtual locale grid by storing the locale
// values into a higher-dimensional array of locales (the array
// reshape() method can be particularly useful for this purpose).
//

//
// In addition to the .id query mentioned above, locales support a
// number of other queries about their properties.  For example:
//
//   locale.name     : returns a string indicating the locale's name
//   locale.numCores : returns the number of processor cores on the locale
//   locale.physicalMemory() : returns the amount of memory on the locale
//   locale.maxTaskPar : returns the likely maximum parallelism available
//                       on the locale
//

config const printLocaleInfo = true;  // permit testing to turn this off

if printLocaleInfo then
  for loc in Locales do
    on loc {
      writeln("locale #", here.id, "...");
      writeln("  ...is named: ", here.name);
      writeln("  ...has ", here.numCores, " processor cores");
      writeln("  ...has ", here.physicalMemory(unit=MemUnits.GB, retType=real),
              " GB of memory");
      writeln("  ...has ", here.maxTaskPar, " maximum parallelism");
    }
writeln();

//
// Chapel variables are stored using the memory of the locale
// executing the task that encounters the variable declaration.
// Thus, in the following code, x is declared on locale 0 and
// y is declared on locale 1 if numLocales > 1 (0 otherwise).
//

{
  var x: int = 2;
  on Locales[1 % numLocales] {
    var y: int = 3;
    writeln("From locale ", here.id, ", x is: ", x, " and y is: ", y);
    on Locales[0] {
      writeln("From locale 0, x is: ", x, " and y is: ", y);
    }
  }
  writeln();
}

//
// Note in the code above that a task can refer to any
// lexically-visible variable in Chapel regardless of the locales on
// which the variable is stored and the task is executing.  This is
// what is known as a global namespace (or global address space)
// quality in a language.  The fact that locality within a Chapel
// program can be semantically reasoned about by the programmer (i.e.,
// "x is on Locale[0], y is on Locale[1]") makes it a PGAS or
// Partitioned Global Address Space language (though we prefer the
// less broadly-used term "Partitioned Global Namespace").
//
// Not only can a programmer reason about a variable or task's
// location abstractly, they can also query these values directly
// in the language.  In particular, all Chapel variables support a
// .locale query which returns the locale value that is storing that
// variable.  Modifying the above example slightly:
//

{
  var x: int = 2;
  on Locales[1 % numLocales] {
    var y: int = 3;
    writeln("x is stored on locale ", x.locale.id, ", while y lives on ",
            y.locale.id);
  }
  writeln();
}

//
// Moreover, Chapel's on-clauses can be controlled by any expression
// with storage associated with it.  "on x" effectively says "execute
// this on whichever locale owns x" or equivalently "on x.locale".
// Here's a variation of the example above using this idiom:
//

{
  var x: int = 2;
  on Locales[1 % numLocales] {
    var y: int = 3;
    on x do
      writeln("Using a data-driven on-clause, I'm now executing on locale ",
              here.id);
  }
  writeln();
}

//
// Next, let's create a simple distributed data structure using the
// principles above.  In particular, we'll create a linked list that
// spans the locales with a list node per locale.
//

//
// First, define a standard Node class for linked lists.  Note that
// even though the 'next' fields will span locales, they don't need
// to be declared specially in any way, again thanks to the [P]GAS
// nature of the language.
//
class Node {
  var data: real;
  var next: Node;
}

//
// Next, we'll iterate through the locales, allocating a node on each.
// It's worth reviewing that a class variable (like the 'next' field
// above) is just a reference in Chapel and the object it points to
// is a separate thing allocated by a call to 'new'.  So in general,
// the reference and object can live on different locales (as they
// will here.
//
// So, in the following loop, 'current' will always point at the last
// node we created while our on-clause will move the task ahead to
// the next locale.  That way, our execution of 'new' will create
// the new object on that "next" locale.
//
var head = new Node(0);
var current = head;
for i in 1..numLocales-1 do
  on Locales[i] {
    current.next = new Node(i);
    current = current.next;
  }

//
// Loop over the linked list and output each node and the locale on
// which it exists using .locale.id to determine the locale.  This
// computation takes place entirely on Locales(0) and accesses remote
// memory as necessary.
//
current = head;
while current {
  writeln("node with data = ", current.data, " on locale ", current.locale.id);
  current = current.next;
}
writeln();

//
// Alternatively, we could use a data-driven execution to walk the
// linked list.  Note that on-clauses, when applied to class
// variables, resolve to the locale of the object, not the reference.
//
current = head;
while current {
  on current {
    writeln("node with data = ", current.data, " on locale ", here.id);
    current = current.next;
  }
}

//
// For more information about Locales, refer to the Locales chapter of
// the language spec.
//


Apply private to some symbols in the CommDiagnostics standard module

When chpldoc recognizes private, the "no doc" pragmas can be removed

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  This module provides support for reporting and counting
  communication operations between network-connected locales.  The
  operations include various kinds of remote reads (GETs), remote
  writes (PUTs), and remote forks (which are actually more like remote
  procedure calls, and are used to implement ``on`` statements).
  Callers can request on-the-fly output each time a remote operation
  occurs, or count such operations as they occur and retrieve the
  counts later.  The former gives more detailed information but has
  much more overhead.  The latter has much less overhead but only
  provides aggregate information.

  **On-the-fly Reporting**

  All forms of communication reporting and counting are done between
  pairs of function calls that turn it on and off.  On-the-fly
  reporting across all locales is done like this::

    startVerboseComm();
    // between start/stop calls, report comm ops initiated on any locale
    stopVerboseComm();

  On-the-fly reporting for just the calling locale is similar.  Only
  the procedure names change::

    startVerboseCommHere();
    // between start/stop calls, report comm ops initiated on this locale
    stopVerboseCommHere();

  In either case, the output produced consists of a line written to
  ``stdout`` for each communication operation.  (Here ``stdout`` means
  the file associated with the process, not the Chapel channel with the
  same name.)

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startVerboseComm();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopVerboseComm();
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    0: remote task created on 1
    1: t.chpl:6: remote get from 0
    1: t.chpl:6: remote put to 0

  The initial number refers to the locale reporting the communication
  event.  The file name and line number point to the place in the
  code that triggered the communication event.  (For remote forks,
  file name and line number information is not yet reported.)

  **Counting Communication Operations**

  Counting communication operations requires a few more calls then just
  reporting them does.  In particular, the counts have to be retrieved
  after they are collected and, if they have been used previously, the
  internal counters have to be reset before counting is turned on.
  Counting across all locales is done like this::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnostics();
    startCommDiagnostics();
    // between start/stop calls, count comm ops initiated on any locale
    stopCommDiagnostics();
    // retrieve the counts and report the results
    writeln(getCommDiagnostics());

  Counting on just the calling locale is similar.  Just as for
  on-the-fly reporting, only the procedure names change::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnosticsHere();
    startCommDiagnosticsHere();
    // between start/stop calls, count comm ops initiated on this locale
    stopCommDiagnosticsHere();
    // retrieve the counts and report the results
    writeln(getCommDiagnosticsHere());

  The optional call to reset the counters is only needed when a program
  collects counts more than once.  In this case, the counters have to
  be set back to zero before starting the second and succeeding
  counting periods.  By far the most common situation is that programs
  only collect communication counts once per run, in which case this
  step is not needed.

  Note that the same internal mechanisms and counters are used for
  counting on all locales and counting on just the calling locale, so
  trying to do both at once may lead to surprising turn-on/turn-off
  behavior and/or incorrect results.

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startCommDiagnostics();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopCommDiagnostics();
      writeln(getCommDiagnostics());
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    (get = 0, get_nb = 0, put = 0, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 1, fork_fast = 0, fork_nb = 0) (get = 1, get_nb = 0, put = 1, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 0, fork_fast = 0, fork_nb = 0)

  The first parenthesized group contains the counts for locale 0, and
  the second contains the counts for locale 1.  So, for the
  instrumented section of this program we can say that a remote fork
  was executed on locale 0, and a remote get and a remote put were
  executed on locale 1.

  **Studying Communication During Module Initialization**

  It is hard for a programmer to determine exactly what happens during
  initialization or teardown of a module, because the code that runs
  then does so only implicitly, as a result of the declarations
  present.  And even if that code can be identified, doing debug
  output or logging data for later reporting might not work because
  the Chapel capabilities needed to do so could be unavailable due to
  being implemented by built-in modules which themselves are not yet
  initialized, or have already been torn down.

  To help with that problem, this module provides built-in support for
  studying communication operations during module initialization and
  teardown.  To use it, set either or both of the config params
  :param:`printInitVerboseComm` and :param:`printInitCommCounts`,
  described below.  You can do this by using appropriate
  ``-sconfigParamName=value`` command line options when you compile
  your program.

  The reporting and/or counting enabled by these covers all of program
  execution, from just before the first module is initialized until
  just after the last one is torn down.  This is almost always a
  superset of the part of the program that is of interest, which is
  often just a single module.  To learn what communication is being
  done by a single module during its initialization and teardown it is
  often necessary to run a small test program twice, once with that
  module present and once without it.
 */
module CommDiagnostics
{
  // There should be a type like this declared in chpl-comm.h with a single
  // function that returns the C struct.  We're not doing it that way yet
  // due to some shortcomings in our extern records implementation.
  // Once that gets sorted out, we can turn this into an extern record,
  // and remove the 8 or so individual functions below that return the
  // various counters.

  /* Aggregated communication operation counts.  This record type is
     defined in the same way by both the underlying comm layer(s) and
     this module, because we don't have a good way to inherit types back
     and forth between the two.  This first definition duplicates the
     one in the comm layer(s).
   */
  extern record chpl_commDiagnostics {
    /*
      blocking GETs, in which initiator waits for completion
     */
    var get: uint(64);
    /*
      non-blocking GETs
     */
    var get_nb: uint(64);
    /*
      blocking PUTs, in which initiator waits for completion
     */
    var put: uint(64);
    /*
      non-blocking PUTs
     */
    var put_nb: uint(64);
    /*
      tests for non-blocking GET/PUT completions
     */
    var test_nb: uint(64);
    /*
      blocking waits for non-blocking GET/PUT completions
     */
    var wait_nb: uint(64);
    /*
      non-blocking waits for non-blocking GET/PUT completions
     */
    var try_nb: uint(64);
    /*
      blocking remote executions, in which initiator waits for completion
     */
    var fork: uint(64);
    /*
      blocking remote executions performed by the target locale's
      Active Message handler
     */
    var fork_fast: uint(64);
    /*
      non-blocking remote executions
     */
    var fork_nb: uint(64);
  };

  /*
    The Chapel record type inherits the comm layer definition of it.
   */
  type commDiagnostics = chpl_commDiagnostics;

  pragma "no doc"
  private extern proc chpl_startVerboseComm();

  pragma "no doc"
  private extern proc chpl_stopVerboseComm();

  pragma "no doc"
  private extern proc chpl_startVerboseCommHere();

  pragma "no doc"
  private extern proc chpl_stopVerboseCommHere();

  pragma "no doc"
  private extern proc chpl_gen_startCommDiagnostics();

  pragma "no doc"
  private extern proc chpl_gen_stopCommDiagnostics();

  pragma "no doc"
  private extern proc chpl_gen_startCommDiagnosticsHere();

  pragma "no doc"
  private extern proc chpl_gen_stopCommDiagnosticsHere();

  pragma "no doc"
  private extern proc chpl_resetCommDiagnosticsHere();

  pragma "no doc"
  private extern proc chpl_getCommDiagnosticsHere(out cd: commDiagnostics);

  /*
    Start on-the-fly reporting of communication initiated on any locale.
   */
  proc startVerboseComm() { chpl_startVerboseComm(); }

  /*
    Stop on-the-fly reporting of communication initiated on any locale.
   */
  proc stopVerboseComm() { chpl_stopVerboseComm(); }

  /*
    Start on-the-fly reporting of communication initiated on this locale.
   */
  proc startVerboseCommHere() { chpl_startVerboseCommHere(); }

  /*
    Stop on-the-fly reporting of communication initiated on this locale.
   */
  proc stopVerboseCommHere() { chpl_stopVerboseCommHere(); }

  /*
    Start counting communication operations across the whole program.
   */
  proc startCommDiagnostics() {
    chpl_gen_startCommDiagnostics();
  }

  /*
    Stop counting communication operations across the whole program.
   */
  proc stopCommDiagnostics() {
    chpl_gen_stopCommDiagnostics();
  }

  /*
    Start counting communication operations initiated on this locale.
   */
  proc startCommDiagnosticsHere() {
    chpl_gen_startCommDiagnosticsHere();
  }

  /*
    Stop counting communication operations initiated on this locale.
   */
  proc stopCommDiagnosticsHere() {
    chpl_gen_stopCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts across the whole program.
   */
  proc resetCommDiagnostics() {
    for loc in Locales do on loc do
      resetCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts on the calling locale.
   */
  inline proc resetCommDiagnosticsHere() {
    chpl_resetCommDiagnosticsHere();
  }

  // See note above regarding extern records
  pragma "no doc"
  private extern proc chpl_numCommGets(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommNBGets(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommPuts(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommNBPuts(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommTestNB(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommWaitNB(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommTryNB(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommForks(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommFastForks(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommNBForks(): uint(64);

  /*
    Retrieve aggregate communication counts for the whole program.

    :returns: array of counts of comm ops initiated on each locale
    :rtype: `[LocaleSpace] commDiagnostics`
   */
  proc getCommDiagnostics() {
    var D: [LocaleSpace] commDiagnostics;
    for loc in Locales do on loc {
      // See note above regarding extern records
      D(loc.id).get = chpl_numCommGets();
      D(loc.id).get_nb = chpl_numCommNBGets();
      D(loc.id).put = chpl_numCommPuts();
      D(loc.id).put_nb = chpl_numCommNBPuts();
      D(loc.id).test_nb = chpl_numCommTestNB();
      D(loc.id).wait_nb = chpl_numCommWaitNB();
      D(loc.id).try_nb = chpl_numCommTryNB();
      D(loc.id).fork = chpl_numCommForks();
      D(loc.id).fork_fast = chpl_numCommFastForks();
      D(loc.id).fork_nb = chpl_numCommNBForks();
    }
    return D;
  }

  /*
    Retrieve aggregate communication counts for this locale.

    :returns: counts of comm ops initiated on this locale
    :rtype: `commDiagnostics`
   */
  proc getCommDiagnosticsHere() {
    var cd: commDiagnostics;
    cd.get = chpl_numCommGets();
    cd.get_nb = chpl_numCommNBGets();
    cd.put = chpl_numCommPuts();
    cd.put_nb = chpl_numCommNBPuts();
    cd.test_nb = chpl_numCommTestNB();
    cd.wait_nb = chpl_numCommWaitNB();
    cd.try_nb = chpl_numCommTryNB();
    cd.fork = chpl_numCommForks();
    cd.fork_fast = chpl_numCommFastForks();
    cd.fork_nb = chpl_numCommNBForks();
    return cd;
  }

  /*
    If this is set, on-the-fly reporting of communication operations
    will be turned on before any module initialization begins and
    turned off after all module teardown ends.  See procedures
    :proc:`startVerboseComm` and :proc:`stopVerboseComm` for more
    information.
   */
  config param printInitVerboseComm = false;

  /*
    If this is set, communication operations are counted from before
    any module initialization begins until after all module teardown
    ends, and then the aggregate counts are printed.  See procedures
    :proc:`startCommDiagnostics`, :proc:`stopCommDiagnostics`, and
    :proc:`getCommDiagnostics` for more information.
   */
  config param printInitCommCounts = false;

}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  This module provides support for reporting and counting
  communication operations between network-connected locales.  The
  operations include various kinds of remote reads (GETs), remote
  writes (PUTs), and remote forks (which are actually more like remote
  procedure calls, and are used to implement ``on`` statements).
  Callers can request on-the-fly output each time a remote operation
  occurs, or count such operations as they occur and retrieve the
  counts later.  The former gives more detailed information but has
  much more overhead.  The latter has much less overhead but only
  provides aggregate information.

  **On-the-fly Reporting**

  All forms of communication reporting and counting are done between
  pairs of function calls that turn it on and off.  On-the-fly
  reporting across all locales is done like this::

    startVerboseComm();
    // between start/stop calls, report comm ops initiated on any locale
    stopVerboseComm();

  On-the-fly reporting for just the calling locale is similar.  Only
  the procedure names change::

    startVerboseCommHere();
    // between start/stop calls, report comm ops initiated on this locale
    stopVerboseCommHere();

  In either case, the output produced consists of a line written to
  ``stdout`` for each communication operation.  (Here ``stdout`` means
  the file associated with the process, not the Chapel channel with the
  same name.)

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startVerboseComm();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopVerboseComm();
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    0: remote task created on 1
    1: t.chpl:6: remote get from 0
    1: t.chpl:6: remote put to 0

  The initial number refers to the locale reporting the communication
  event.  The file name and line number point to the place in the
  code that triggered the communication event.  (For remote forks,
  file name and line number information is not yet reported.)

  **Counting Communication Operations**

  Counting communication operations requires a few more calls then just
  reporting them does.  In particular, the counts have to be retrieved
  after they are collected and, if they have been used previously, the
  internal counters have to be reset before counting is turned on.
  Counting across all locales is done like this::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnostics();
    startCommDiagnostics();
    // between start/stop calls, count comm ops initiated on any locale
    stopCommDiagnostics();
    // retrieve the counts and report the results
    writeln(getCommDiagnostics());

  Counting on just the calling locale is similar.  Just as for
  on-the-fly reporting, only the procedure names change::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnosticsHere();
    startCommDiagnosticsHere();
    // between start/stop calls, count comm ops initiated on this locale
    stopCommDiagnosticsHere();
    // retrieve the counts and report the results
    writeln(getCommDiagnosticsHere());

  The optional call to reset the counters is only needed when a program
  collects counts more than once.  In this case, the counters have to
  be set back to zero before starting the second and succeeding
  counting periods.  By far the most common situation is that programs
  only collect communication counts once per run, in which case this
  step is not needed.

  Note that the same internal mechanisms and counters are used for
  counting on all locales and counting on just the calling locale, so
  trying to do both at once may lead to surprising turn-on/turn-off
  behavior and/or incorrect results.

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startCommDiagnostics();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopCommDiagnostics();
      writeln(getCommDiagnostics());
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    (get = 0, get_nb = 0, put = 0, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 1, fork_fast = 0, fork_nb = 0) (get = 1, get_nb = 0, put = 1, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 0, fork_fast = 0, fork_nb = 0)

  The first parenthesized group contains the counts for locale 0, and
  the second contains the counts for locale 1.  So, for the
  instrumented section of this program we can say that a remote fork
  was executed on locale 0, and a remote get and a remote put were
  executed on locale 1.

  **Studying Communication During Module Initialization**

  It is hard for a programmer to determine exactly what happens during
  initialization or teardown of a module, because the code that runs
  then does so only implicitly, as a result of the declarations
  present.  And even if that code can be identified, doing debug
  output or logging data for later reporting might not work because
  the Chapel capabilities needed to do so could be unavailable due to
  being implemented by built-in modules which themselves are not yet
  initialized, or have already been torn down.

  To help with that problem, this module provides built-in support for
  studying communication operations during module initialization and
  teardown.  To use it, set either or both of the config params
  :param:`printInitVerboseComm` and :param:`printInitCommCounts`,
  described below.  You can do this by using appropriate
  ``-sconfigParamName=value`` command line options when you compile
  your program.

  The reporting and/or counting enabled by these covers all of program
  execution, from just before the first module is initialized until
  just after the last one is torn down.  This is almost always a
  superset of the part of the program that is of interest, which is
  often just a single module.  To learn what communication is being
  done by a single module during its initialization and teardown it is
  often necessary to run a small test program twice, once with that
  module present and once without it.
 */
module CommDiagnostics
{
  // There should be a type like this declared in chpl-comm.h with a single
  // function that returns the C struct.  We're not doing it that way yet
  // due to some shortcomings in our extern records implementation.
  // Once that gets sorted out, we can turn this into an extern record,
  // and remove the 8 or so individual functions below that return the
  // various counters.

  /* Aggregated communication operation counts.  This record type is
     defined in the same way by both the underlying comm layer(s) and
     this module, because we don't have a good way to inherit types back
     and forth between the two.  This first definition duplicates the
     one in the comm layer(s).
   */
  extern record chpl_commDiagnostics {
    /*
      blocking GETs, in which initiator waits for completion
     */
    var get: uint(64);
    /*
      non-blocking GETs
     */
    var get_nb: uint(64);
    /*
      blocking PUTs, in which initiator waits for completion
     */
    var put: uint(64);
    /*
      non-blocking PUTs
     */
    var put_nb: uint(64);
    /*
      tests for non-blocking GET/PUT completions
     */
    var test_nb: uint(64);
    /*
      blocking waits for non-blocking GET/PUT completions
     */
    var wait_nb: uint(64);
    /*
      non-blocking waits for non-blocking GET/PUT completions
     */
    var try_nb: uint(64);
    /*
      blocking remote executions, in which initiator waits for completion
     */
    var fork: uint(64);
    /*
      blocking remote executions performed by the target locale's
      Active Message handler
     */
    var fork_fast: uint(64);
    /*
      non-blocking remote executions
     */
    var fork_nb: uint(64);
  };

  /*
    The Chapel record type inherits the comm layer definition of it.
   */
  type commDiagnostics = chpl_commDiagnostics;

  pragma "no doc"
  extern proc chpl_startVerboseComm();

  pragma "no doc"
  extern proc chpl_stopVerboseComm();

  pragma "no doc"
  extern proc chpl_startVerboseCommHere();

  pragma "no doc"
  extern proc chpl_stopVerboseCommHere();

  pragma "no doc"
  extern proc chpl_gen_startCommDiagnostics();

  pragma "no doc"
  extern proc chpl_gen_stopCommDiagnostics();

  pragma "no doc"
  extern proc chpl_gen_startCommDiagnosticsHere();

  pragma "no doc"
  extern proc chpl_gen_stopCommDiagnosticsHere();

  pragma "no doc"
  extern proc chpl_resetCommDiagnosticsHere();

  pragma "no doc"
  extern proc chpl_getCommDiagnosticsHere(out cd: commDiagnostics);

  /*
    Start on-the-fly reporting of communication initiated on any locale.
   */
  proc startVerboseComm() { chpl_startVerboseComm(); }

  /*
    Stop on-the-fly reporting of communication initiated on any locale.
   */
  proc stopVerboseComm() { chpl_stopVerboseComm(); }

  /*
    Start on-the-fly reporting of communication initiated on this locale.
   */
  proc startVerboseCommHere() { chpl_startVerboseCommHere(); }

  /*
    Stop on-the-fly reporting of communication initiated on this locale.
   */
  proc stopVerboseCommHere() { chpl_stopVerboseCommHere(); }

  /*
    Start counting communication operations across the whole program.
   */
  proc startCommDiagnostics() {
    chpl_gen_startCommDiagnostics();
  }

  /*
    Stop counting communication operations across the whole program.
   */
  proc stopCommDiagnostics() {
    chpl_gen_stopCommDiagnostics();
  }

  /*
    Start counting communication operations initiated on this locale.
   */
  proc startCommDiagnosticsHere() {
    chpl_gen_startCommDiagnosticsHere();
  }

  /*
    Stop counting communication operations initiated on this locale.
   */
  proc stopCommDiagnosticsHere() {
    chpl_gen_stopCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts across the whole program.
   */
  proc resetCommDiagnostics() {
    for loc in Locales do on loc do
      resetCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts on the calling locale.
   */
  inline proc resetCommDiagnosticsHere() {
    chpl_resetCommDiagnosticsHere();
  }

  // See note above regarding extern records
  pragma "no doc"
  extern proc chpl_numCommGets(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommNBGets(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommPuts(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommNBPuts(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommTestNB(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommWaitNB(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommTryNB(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommForks(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommFastForks(): uint(64);

  pragma "no doc"
  extern proc chpl_numCommNBForks(): uint(64);

  /*
    Retrieve aggregate communication counts for the whole program.

    :returns: array of counts of comm ops initiated on each locale
    :rtype: `[LocaleSpace] commDiagnostics`
   */
  proc getCommDiagnostics() {
    var D: [LocaleSpace] commDiagnostics;
    for loc in Locales do on loc {
      // See note above regarding extern records
      D(loc.id).get = chpl_numCommGets();
      D(loc.id).get_nb = chpl_numCommNBGets();
      D(loc.id).put = chpl_numCommPuts();
      D(loc.id).put_nb = chpl_numCommNBPuts();
      D(loc.id).test_nb = chpl_numCommTestNB();
      D(loc.id).wait_nb = chpl_numCommWaitNB();
      D(loc.id).try_nb = chpl_numCommTryNB();
      D(loc.id).fork = chpl_numCommForks();
      D(loc.id).fork_fast = chpl_numCommFastForks();
      D(loc.id).fork_nb = chpl_numCommNBForks();
    }
    return D;
  }

  /*
    Retrieve aggregate communication counts for this locale.

    :returns: counts of comm ops initiated on this locale
    :rtype: `commDiagnostics`
   */
  proc getCommDiagnosticsHere() {
    var cd: commDiagnostics;
    cd.get = chpl_numCommGets();
    cd.get_nb = chpl_numCommNBGets();
    cd.put = chpl_numCommPuts();
    cd.put_nb = chpl_numCommNBPuts();
    cd.test_nb = chpl_numCommTestNB();
    cd.wait_nb = chpl_numCommWaitNB();
    cd.try_nb = chpl_numCommTryNB();
    cd.fork = chpl_numCommForks();
    cd.fork_fast = chpl_numCommFastForks();
    cd.fork_nb = chpl_numCommNBForks();
    return cd;
  }

  /*
    If this is set, on-the-fly reporting of communication operations
    will be turned on before any module initialization begins and
    turned off after all module teardown ends.  See procedures
    :proc:`startVerboseComm` and :proc:`stopVerboseComm` for more
    information.
   */
  config param printInitVerboseComm = false;

  /*
    If this is set, communication operations are counted from before
    any module initialization begins until after all module teardown
    ends, and then the aggregate counts are printed.  See procedures
    :proc:`startCommDiagnostics`, :proc:`stopCommDiagnostics`, and
    :proc:`getCommDiagnostics` for more information.
   */
  config param printInitCommCounts = false;

}