Changing to a whole-array read

Changing to a whole-array read

Michael caught this just before the release and after the code freeze.
Now that whole-array reads are supported, it's much nicer to show them
off here rather than using the loop-based form.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@23242 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
 * File I/O Primer
 */

/*
   First example: Textual Array I/O
   Second and Third examples: Binary I/O
   Fourth example: reading UTF-8 lines and printing them out.
   Fifth example: error handling.
   Sixth example: object-at-a-time writing.
   Seventh example: binary I/O with bits at a time
*/

config var n = 9,            // the problem size for example 1
           filename = "Arr.dat";  // the filename for writing/reading the array
config const num = 128*1024;
config const example = 0;
config const testfile = "test.bin";

use IO;

/* Example 1
 * This is a simple example of using file I/O in Chapel.
 *
 * It initializes an array and writes its size and data to a file.  It
 * then opens the file, uses the size in the file to declare a new
 * domain and array, and reads in the array data.
 */
if example == 0 || example == 1 {
  const ADom = {1..n, 1..n};  // Create a domain of the specified problem size

  // Create and initialize an array of the specified size
  var A: [ADom] real = [(i,j) in ADom] i + j/10.0;

  // Write the problem size and array out to the specified filename
  writeSquareArray(n, A, filename);

  // Read an array in from the specified filename, storing in a new variable, B
  var B = readArray(filename);

  // Print out B as a debugging step
  writeln("B is:\n", B);

  //
  // verify that A and B contain the same values and print success or failure
  //
  const numErrors = + reduce [i in ADom] (A(i) != B(i));

  if (numErrors > 0) {
    writeln("FAILURE");
  } else {
    writeln("SUCCESS");
  }
}

//
// this procedure writes a square array out to a file
//
proc writeSquareArray(n, X, filename) {
  // Create and open an output file with the specified filename in write mode
  var outfile = open(filename, iomode.cw);
  var writer = outfile.writer();

  // Write the problem size in each dimension to the file
  writer.writeln(n, " ", n);

  // write out the array itself
  writer.write(X);

  // close the file
  writer.close();
  outfile.close();
}

//
// This procedure reads a new array out of a file and returns it
//
proc readArray(filename) {
   // Open an input file with the specified filename in read mode
  var infile = open(filename, iomode.r);
  var reader = infile.reader();

  // Read the number of rows and columns in the array in from the file
  var m = reader.read(int),
      n = reader.read(int);

  // Declare an array of the specified dimensions
  var X: [1..m, 1..n] real;

  //
  // Read in the array (row-major order is used for whole-array reads
  // like this)
  //
  reader.read(X);

  // Close the file
  reader.close();
  infile.close();

  // Return the array
  return X;
}

/*
   In Examples 2 and 3, we will write numbers 0..n-1 to a file in binary,
   and then we'll open it up, and read the numbers in reverse,
   just to show how to 'seek' in a file.

   We show two versions of this example; a simple version, and
   a slightly more complicated version that has some performance hints.
*/

if example == 0 || example == 2 {
  writeln("Running Example 2");

  // Here comes the simple version!

  // First, open up a test file. Chapel's I/O interface allows
  // us to open regular files, temporary files, memory, or file descriptors;
  var f = open(testfile, iomode.cwr);

  /* Since the typical 'file position' design leads to race conditions
   * all over, the Chapel I/O design separates a file from a channel.
   * A channel is a buffer to a particular spot in a file. Channels
   * can have a start and and end, so that if you're doing parallel I/O
   * to different parts of a file with different channels, you can
   * partition the file to be assured that they do not interfere.
   */

  {
    var w = f.writer(kind=ionative); // get a binary writing channel for the start of the file.

    for i in 0..#num { // for each i in [0,n)
      var tmp:uint(64) = i:uint(64);
      w.write(tmp); // writing a uint(64) will write 8 bytes.
    }

    // Now w goes out of scope, it will flush anything that is buffered.
    // Channels are reference-counted, so it should be easy, but if you want to make sure,
    // you can close the channel.
    w.close();
  }

  // OK, now we have written our data file. Now read it backwards.
  // Note -- this could be a forall loop to do I/O in parallel!
  for i in 0..#num by -1 {
    var r = f.reader(kind=ionative, start=8*i, end=8*i+8);
    var tmp:uint(64);
    r.read(tmp);
    assert(tmp == i:uint(64));
    r.close();
  }

  // Now close the file. Like channels, files are reference-counted,
  // and so should close when they go out of scope, but we close it
  // here anyway to be sure.
  f.close();

  // Finally, we can remove the test file.
  unlink(testfile);
}

if example == 0 || example == 3 {
  writeln("Running Example 3");

  /* Here comes a faster version of example 2, using some hints.
     This time we're not going to use a temporary file, because
     we want to open it twice for performance reasons.
     If you want to measure the performance difference, try:
       time ./fielIOv2 --example=2
       time ./fielIOv2 --example=3
   */

  // First, open up a file and write to it.
  {
    var f = open(testfile, iomode.cwr);
    /* When we create the writer, suppling locking=false will do unlocked I/O.
       That's fine as long as the channel is not shared between tasks.
      */
    var w = f.writer(kind=ionative, locking=false);

    for i in 0..#num {
      var tmp:uint(64) = i:uint(64);
      w.write(tmp);
    }

    w.close();
    f.close();
  }

  /* Now that we've created the file, when we open it for
     read access and hint 'random access' and 'keep data cached/assume data is cached',
     we can optimize better (using mmap, if you like details).
   */
  {
    var f = open(testfile, iomode.r,
                 hints=IOHINT_RANDOM|IOHINT_CACHED|IOHINT_PARALLEL);

    // Note -- this could be a forall loop to do I/O in parallel!
    forall i in 0..#num by -1 {
      /* When we create the reader, suppling locking=false will do unlocked I/O.
         That's fine as long as the channel is not shared between tasks;
         here it's just used as a local variable, so we are O.K.
        */
      var r = f.reader(kind=ionative, locking=false, start=8*i, end=8*i+8);
      var tmp:uint(64);
      r.read(tmp);
      assert(tmp == i:uint(64));
      r.close();
    }

    f.close();
  }

  // Finally, we can remove the test file.
  unlink(testfile);
}

/*
   In Example 4, we show that it's easy to read the lines in a file,
   including UTF-8 characters.
*/
if example == 0 || example == 4 {
  writeln("Running Example 4");

  var f = open(testfile, iomode.cwr);
  var w = f.writer();

  w.writeln("Hello");
  w.writeln("This");
  w.writeln(" is ");
  w.writeln(" a test ");
  // We only write the UTF-8 characters if unicode is supported,
  // and that depends on the current unix locale environment
  // (e.g. setting the environment variable LC_ALL=C will disable unicode support).
  // Note that since UTF-8 strings are C strings, this should work
  // even in a C locale. We don't do it all the time for testing sanity reasons.
  if unicodeSupported() then w.writeln(" of UTF-8 Euro Sign: €");

  // flush buffers, close the channel.
  w.close();

  var r = f.reader();
  var line:string;
  while( r.readline(line) ) {
    write("Read line: ", line);
  }
  r.close();

  // Or, if we just want all the lines in the file,
  // we can use file.lines, and we don't even have to make a reader:
  for line in f.lines() {
    write("Read line: ", line);
  }

  f.close();
  unlink(testfile);
}

/*
   In Example 5, we show that error handling is possible
   with the new I/O routines. Maybe one day this strategy will
   be replaced by exceptions, but until then... Chapel programs
   can still respond to errors.

*/
if example == 0 || example == 5 {
  writeln("Running Example 5");

  // Error handling.
  var err:syserr;
  // Who knows, maybe 1st unlink succeeds.
  unlink(testfile, error=err);

  // File does not exist by now, for sure.
  unlink(testfile, error=err);
  assert(err == ENOENT);

  // What happens if we try to open a non-existant file?
  var f = open(testfile, iomode.r, error=err);
  assert(err == ENOENT);

  /* Note that if an error= argument is not supplied to an
     I/O function, it will call ioerror, which will
     in turn halt with an error message.
   */
}

/*
   In Example 6, we demonstrate that output from multiple tasks
   can use the same channel, and each write() call will be
   completed entirely before another one is allowed to begin.
*/
if example == 0 || example == 6 {
  writeln("Running Example 6");

  forall i in 1..4 {
    writeln("This should be a chunk: {", "\n a", "\n b", "\n}");
  }

  record MyThing {
    proc writeThis(w:Writer) {
      w.writeln("This should be a chunk: {");
      w.writeln(" a");
      w.writeln(" b");
      w.writeln("}");
    }
  }

  forall i in 1..4 {
    var t:MyThing;
    write(t);
  }
}

/*
   In Example 7, we demonstrate bit-level I/O.
 */
if example == 0 || example == 7 {
  writeln("Running Example 7");

  var f = open(testfile, iomode.cwr);

  {
    var w = f.writer(kind=ionative);

    // Write 011 0110 011110000
    w.writebits(0b011, 3);
    w.writebits(0b0110, 4);
    w.writebits(0b011110000, 9);
    w.close();
  }

  // Try reading it back the way we wrote it.
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(64);

    r.readbits(tmp, 3);
    assert(tmp == 0b011);

    r.readbits(tmp, 4);
    assert(tmp == 0b0110);

    r.readbits(tmp, 9);
    assert(tmp == 0b011110000);

    r.close();
  }

  // Try reading it back all as one big chunk.
  // Read 01101100 11110000
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(8);

    r.read(tmp);
    assert(tmp == 0b01101100);

    r.read(tmp);
    assert(tmp == 0b11110000);

    r.close();
  }

  f.close();
}

/*
 * File I/O Primer
 */

/*
   First example: Textual Array I/O
   Second and Third examples: Binary I/O
   Fourth example: reading UTF-8 lines and printing them out.
   Fifth example: error handling.
   Sixth example: object-at-a-time writing.
   Seventh example: binary I/O with bits at a time
*/

config var n = 9,            // the problem size for example 1
           filename = "Arr.dat";  // the filename for writing/reading the array
config const num = 128*1024;
config const example = 0;
config const testfile = "test.bin";

use IO;

/* Example 1
 * This is a simple example of using file I/O in Chapel.
 *
 * It initializes an array and writes its size and data to a file.  It
 * then opens the file, uses the size in the file to declare a new
 * domain and array, and reads in the array data.
 */
if example == 0 || example == 1 {
  const ADom = {1..n, 1..n};  // Create a domain of the specified problem size

  // Create and initialize an array of the specified size
  var A: [ADom] real = [(i,j) in ADom] i + j/10.0;

  // Write the problem size and array out to the specified filename
  writeSquareArray(n, A, filename);

  // Read an array in from the specified filename, storing in a new variable, B
  var B = readArray(filename);

  // Print out B as a debugging step
  writeln("B is:\n", B);

  //
  // verify that A and B contain the same values and print success or failure
  //
  const numErrors = + reduce [i in ADom] (A(i) != B(i));

  if (numErrors > 0) {
    writeln("FAILURE");
  } else {
    writeln("SUCCESS");
  }
}

//
// this procedure writes a square array out to a file
//
proc writeSquareArray(n, X, filename) {
  // Create and open an output file with the specified filename in write mode
  var outfile = open(filename, iomode.cw);
  var writer = outfile.writer();

  // Write the problem size in each dimension to the file
  writer.writeln(n, " ", n);

  // write out the array itself
  writer.write(X);

  // close the file
  writer.close();
  outfile.close();
}

//
// This procedure reads a new array out of a file and returns it
//
proc readArray(filename) {
   // Open an input file with the specified filename in read mode
  var infile = open(filename, iomode.r);
  var reader = infile.reader();

  // Read the number of rows and columns in the array in from the file
  var m = reader.read(int),
      n = reader.read(int);

  // Declare an array of the specified dimensions
  var X: [1..m, 1..n] real;

  //
  // Read in the array elements one by one.
  // (Whole-array reads are also supported.)
  //
  for i in 1..m do
    for j in 1..n do
      reader.read(X(i,j));

  // Close the file
  reader.close();
  infile.close();

  // Return the array
  return X;
}

/*
   In Examples 2 and 3, we will write numbers 0..n-1 to a file in binary,
   and then we'll open it up, and read the numbers in reverse,
   just to show how to 'seek' in a file.

   We show two versions of this example; a simple version, and
   a slightly more complicated version that has some performance hints.
*/

if example == 0 || example == 2 {
  writeln("Running Example 2");

  // Here comes the simple version!

  // First, open up a test file. Chapel's I/O interface allows
  // us to open regular files, temporary files, memory, or file descriptors;
  var f = open(testfile, iomode.cwr);

  /* Since the typical 'file position' design leads to race conditions
   * all over, the Chapel I/O design separates a file from a channel.
   * A channel is a buffer to a particular spot in a file. Channels
   * can have a start and and end, so that if you're doing parallel I/O
   * to different parts of a file with different channels, you can
   * partition the file to be assured that they do not interfere.
   */

  {
    var w = f.writer(kind=ionative); // get a binary writing channel for the start of the file.

    for i in 0..#num { // for each i in [0,n)
      var tmp:uint(64) = i:uint(64);
      w.write(tmp); // writing a uint(64) will write 8 bytes.
    }

    // Now w goes out of scope, it will flush anything that is buffered.
    // Channels are reference-counted, so it should be easy, but if you want to make sure,
    // you can close the channel.
    w.close();
  }

  // OK, now we have written our data file. Now read it backwards.
  // Note -- this could be a forall loop to do I/O in parallel!
  for i in 0..#num by -1 {
    var r = f.reader(kind=ionative, start=8*i, end=8*i+8);
    var tmp:uint(64);
    r.read(tmp);
    assert(tmp == i:uint(64));
    r.close();
  }

  // Now close the file. Like channels, files are reference-counted,
  // and so should close when they go out of scope, but we close it
  // here anyway to be sure.
  f.close();

  // Finally, we can remove the test file.
  unlink(testfile);
}

if example == 0 || example == 3 {
  writeln("Running Example 3");

  /* Here comes a faster version of example 2, using some hints.
     This time we're not going to use a temporary file, because
     we want to open it twice for performance reasons.
     If you want to measure the performance difference, try:
       time ./fielIOv2 --example=2
       time ./fielIOv2 --example=3
   */

  // First, open up a file and write to it.
  {
    var f = open(testfile, iomode.cwr);
    /* When we create the writer, suppling locking=false will do unlocked I/O.
       That's fine as long as the channel is not shared between tasks.
      */
    var w = f.writer(kind=ionative, locking=false);

    for i in 0..#num {
      var tmp:uint(64) = i:uint(64);
      w.write(tmp);
    }

    w.close();
    f.close();
  }

  /* Now that we've created the file, when we open it for
     read access and hint 'random access' and 'keep data cached/assume data is cached',
     we can optimize better (using mmap, if you like details).
   */
  {
    var f = open(testfile, iomode.r,
                 hints=IOHINT_RANDOM|IOHINT_CACHED|IOHINT_PARALLEL);

    // Note -- this could be a forall loop to do I/O in parallel!
    forall i in 0..#num by -1 {
      /* When we create the reader, suppling locking=false will do unlocked I/O.
         That's fine as long as the channel is not shared between tasks;
         here it's just used as a local variable, so we are O.K.
        */
      var r = f.reader(kind=ionative, locking=false, start=8*i, end=8*i+8);
      var tmp:uint(64);
      r.read(tmp);
      assert(tmp == i:uint(64));
      r.close();
    }

    f.close();
  }

  // Finally, we can remove the test file.
  unlink(testfile);
}

/*
   In Example 4, we show that it's easy to read the lines in a file,
   including UTF-8 characters.
*/
if example == 0 || example == 4 {
  writeln("Running Example 4");

  var f = open(testfile, iomode.cwr);
  var w = f.writer();

  w.writeln("Hello");
  w.writeln("This");
  w.writeln(" is ");
  w.writeln(" a test ");
  // We only write the UTF-8 characters if unicode is supported,
  // and that depends on the current unix locale environment
  // (e.g. setting the environment variable LC_ALL=C will disable unicode support).
  // Note that since UTF-8 strings are C strings, this should work
  // even in a C locale. We don't do it all the time for testing sanity reasons.
  if unicodeSupported() then w.writeln(" of UTF-8 Euro Sign: €");

  // flush buffers, close the channel.
  w.close();

  var r = f.reader();
  var line:string;
  while( r.readline(line) ) {
    write("Read line: ", line);
  }
  r.close();

  // Or, if we just want all the lines in the file,
  // we can use file.lines, and we don't even have to make a reader:
  for line in f.lines() {
    write("Read line: ", line);
  }

  f.close();
  unlink(testfile);
}

/*
   In Example 5, we show that error handling is possible
   with the new I/O routines. Maybe one day this strategy will
   be replaced by exceptions, but until then... Chapel programs
   can still respond to errors.

*/
if example == 0 || example == 5 {
  writeln("Running Example 5");

  // Error handling.
  var err:syserr;
  // Who knows, maybe 1st unlink succeeds.
  unlink(testfile, error=err);

  // File does not exist by now, for sure.
  unlink(testfile, error=err);
  assert(err == ENOENT);

  // What happens if we try to open a non-existant file?
  var f = open(testfile, iomode.r, error=err);
  assert(err == ENOENT);

  /* Note that if an error= argument is not supplied to an
     I/O function, it will call ioerror, which will
     in turn halt with an error message.
   */
}

/*
   In Example 6, we demonstrate that output from multiple tasks
   can use the same channel, and each write() call will be
   completed entirely before another one is allowed to begin.
*/
if example == 0 || example == 6 {
  writeln("Running Example 6");

  forall i in 1..4 {
    writeln("This should be a chunk: {", "\n a", "\n b", "\n}");
  }

  record MyThing {
    proc writeThis(w:Writer) {
      w.writeln("This should be a chunk: {");
      w.writeln(" a");
      w.writeln(" b");
      w.writeln("}");
    }
  }

  forall i in 1..4 {
    var t:MyThing;
    write(t);
  }
}

/*
   In Example 7, we demonstrate bit-level I/O.
 */
if example == 0 || example == 7 {
  writeln("Running Example 7");

  var f = open(testfile, iomode.cwr);

  {
    var w = f.writer(kind=ionative);

    // Write 011 0110 011110000
    w.writebits(0b011, 3);
    w.writebits(0b0110, 4);
    w.writebits(0b011110000, 9);
    w.close();
  }

  // Try reading it back the way we wrote it.
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(64);

    r.readbits(tmp, 3);
    assert(tmp == 0b011);

    r.readbits(tmp, 4);
    assert(tmp == 0b0110);

    r.readbits(tmp, 9);
    assert(tmp == 0b011110000);

    r.close();
  }

  // Try reading it back all as one big chunk.
  // Read 01101100 11110000
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(8);

    r.read(tmp);
    assert(tmp == 0b01101100);

    r.read(tmp);
    assert(tmp == 0b11110000);

    r.close();
  }

  f.close();
}


Improved replication for HPL with Dimensional.

* Replicate the sub-row used in the last computation in panelSolve().

** Removed comments with suggestions on the above.
(They are still retained in hpl-blc+vass-dimensional.chpl.)

** Wrapped the computation in the loop body with 'local'.

* In schurComplement() and panelSolve(), replicate only the relevant
array elements over only the relevant locales.

** I wrote computeReplLocales() that returns an array of "relevant
locales". It assumes (a) the block-cyclic distribution and (b)
the entire row of a panel fits within one locale.
(It could be an iterator as well, except that having an array
allows us to cache the outcome.)

Discussion:

* The good: replicate over fewer locales.

* The questionable: in order to replicate only the relevant elements,
I added a few array slices. Given that the slicee arrays all support
privatization, each slice will add comm to all locales. That may
easily negate the benefits of replicating fewer elements.

* TODO: try taking out as many slices as possible and see if
we get speedup.

* TODO: add replication to updateBlockRow.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@19370 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// Use the distributions we need for this computation
//
use d, r, f;

//
// The number of matrices and the element type of those matrices
// indexType can be int or int(64), elemType can be real or complex
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 8;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

// These are solely to make the testing system happy given the COMPOPTS file.
// To be removed once COMPOPTS becomes a non-issue.
config var reproducible = false, verbose = false;

//
// The program entry point, currently module initialization
//
  printConfiguration();

  //
  // Compute targetLocales - required for Dimensional.
  // We hard-code 2 dimensions.
  //
  var targetIds: domain(2);
  var targetLocales: [targetIds] locale;
  setupTargetLocalesArray(targetIds, targetLocales, Locales);

  // writeln("targetLocales\n", targetLocales, "\n");

  // Here are the dimensions of our grid of locales.
  const tl1 = targetIds.dim(1).length,
        tl2 = targetIds.dim(2).length;
  if printParams && printStats then
    writeln("target locales ", tl1, " x ", tl2);

  // Create the dimensional descriptors
  const
    bdim1 = new idist(lowIdx=1, blockSize=blkSize, numLocales=tl1),
    rdim1 = new vdist(tl1),

    bdim2 = new idist(lowIdx=1, blockSize=blkSize, numLocales=tl2),
    rdim2 = new vdist(tl2);

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  // We use 'AbD' instead of 'MatVectSpace' throughout.
  //
  const AbD: domain(2, indexType)
          dmapped DimensionalDist(targetLocales, bdim1, bdim2, "dim")
          = [1..n, 1..n+1],
        MatrixSpace = AbD[.., ..n];

  var Ab : [AbD] elemType,           // the matrix A and vector b
      piv: [1..n] indexType;         // a vector of pivot values

  //
  // Create the replicated arrays for schurComplement() and panelSolve().
  //
  const
    replAD = [1..n, 1..blkSize]
      dmapped DimensionalDist(targetLocales, bdim1, rdim2, "distBR"),
    replBD = [1..blkSize, 1..n+1]
      dmapped DimensionalDist(targetLocales, rdim1, bdim2, "distRB"),
    replKD = [0..0, 1..blkSize-1]
      dmapped DimensionalDist(targetLocales, rdim1, rdim2, "distRR");

  var replA: [replAD] elemType,
      replB: [replBD] elemType,
      replK: [replKD] elemType;

  initAB();

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, piv);                 // compute the LU factorization

  var x = backwardSub(n);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(x);
  printResults(validAnswer, execTime);

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
proc LUFactorize(n: indexType,
                piv: [1..n] indexType) {

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    if printStats then writeln("processing block ", blk);

    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(l, piv);
    updateBlockRow(tl, tr);

    //
    // update trailing submatrix (if any)
    //
    schurComplement(bl, tr, br);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The 'a' region was 'bl' in the calling
//     |aaaaa|.....|.....|.....|  function but called AD here.  Similarly,
//     +-----+-----+-----+-----+  'b' was 'tr' in the calling code, but BD
//     |aaaaa|.....|.....|.....|  here.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
proc schurComplement(AD: domain, BD: domain, Rest: domain) {

  // Prevent replication of unequal-sized slices
  if Rest.numIndices == 0 then return;

  //
  // Copy data into replicated arrays so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.
  //
  coforall dest in computeReplLocales(2, Rest.dim(2), Rest.dim(1).low) do
    on dest do
      // replA on tgLocales[d1,i] gets a copy of Ab from tgLocales[d1,..]
      replA[Rest.dim(1), ..] = Ab[Rest.dim(1), AD.dim(2)];

  coforall dest in computeReplLocales(1, Rest.dim(1), Rest.dim(2).low) do
    on dest do
      // replB on tgLocales[i,d2] gets a copy of Ab from tgLocales[..,d2]
      replB[.., Rest.dim(2)] = Ab[BD.dim(1), Rest.dim(2)];

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in Rest by (blkSize, blkSize) {
    // workaround: localize Rest explicitly
    const RestLcl = Rest;

    local {
      for a in (RestLcl.dim(1))(row..#blkSize) do
        for w in 1..blkSize do
          for b in (RestLcl.dim(2))(col..#blkSize) do
            Ab[a,b] -= replA[a,w] * replB[w,b];
    }
  }
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
proc panelSolve(
               panel: domain,
               piv: [] indexType) {

  // Used in replication below.
  const replKLocales = computeReplLocales(1, panel.dim(1), panel.dim(2).low);
  const rowEnd = panel.dim(2).high;

  for k in panel.dim(2) {             // iterate through the columns
    const col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.numIndices == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const ( , (pivotRow, )) = maxloc reduce(abs(Ab(col)), col);

    // Capture the pivot value explicitly (note that result of maxloc
    // is absolute value, so it can't be used directly).
    //
    const pivotVal = Ab[pivotRow, k];

    // Swap the current row with the pivot row and update the pivot vector
    // to reflect that
    Ab[k..k, ..] <=> Ab[pivotRow..pivotRow, ..];
    piv[k] <=> piv[pivotRow];

    if (pivotVal == 0) then
      halt("Matrix cannot be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot value
    Ab[k+1.., k..k] /= pivotVal;

    // Replicate some values for the last computation.
    coforall dest in replKLocales do
      on dest do
        replK[0..0, 1..rowEnd-k] = Ab[k..k, k+1..rowEnd];

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      // this 'local' relies on the entire row of the panel being on one locale
      local {
        Ab[i,j] -= Ab[i,k] * replK[0,j-k]; // replK[] was: Ab[k,j]
      }
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
proc updateBlockRow(
                   tl: domain,
                   tr: domain) {

  for row in tr.dim(1) {
    const activeRow = tr[row..row, ..],
          prevRows = tr.dim(1).low..row-1;

    forall (i,j) in activeRow do
      for k in prevRows do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
  }
}

//
// compute the backwards substitution
//
proc backwardSub(n: indexType) {
  const bd = Ab.domain.dim(1);  // or simply 1..n
  var x: [bd] elemType;

  for i in bd by -1 do
    x[i] = (Ab[i,n+1] - (+ reduce [j in i+1..bd.high] (Ab[i,j] * x[j])))
            / Ab[i,i];

  return x;
}

//
// compute the column of targetLocales that's hosting
// the current panel. this is specific to the distribution
// for the second dimension of Ab
//
proc computeReplLocales(param replDim, replStartEnd, nonReplIx)
{
  param nonreplDim =
    if replDim == 1 then 2 else 1;
  proc numTgLocales(param dim)
    return if dim == 1 then tl1 else tl2;
  proc make2dIx(replIx, nonreplIx)
    return if replDim == 1 then (replIx, nonreplIx) else (nonreplIx, replIx);
  proc tgLocalesIndexForAbIndex(param dim, abIx)
    return (abIx / blkSize) % numTgLocales(dim);

  const tgNonrepl = tgLocalesIndexForAbIndex(nonreplDim, nonReplIx);
  assert(Ab[make2dIx(1, 1 + tgNonrepl * blkSize)].locale ==
         targetLocales[make2dIx(0, tgNonrepl)]);

  const startB = replStartEnd.first / blkSize,
        endB   = replStartEnd.last / blkSize;

  if endB - startB + 1 >= numTgLocales(replDim) {
    return
      if replDim == 1 then targetLocales[.., tgNonrepl]
      else                 targetLocales[tgNonrepl, ..];
  } else {
    var result: [startB..endB] locale = [b in startB..endB]
      targetLocales[make2dIx(b % numTgLocales(replDim), tgNonrepl)];
    return result;
  }
}

//
// print out the problem size and block size if requested
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
proc initAB() {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
proc verifyResults(x) {
  initAB();

  const axmbNorm = norm(gaxpyMinus(Ab[.., 1..n], x, Ab[.., n+1..n+1]), normType.normInf);

  const a1norm   = norm(Ab[.., 1..n], normType.norm1),
        aInfNorm = norm(Ab[.., 1..n], normType.normInf),
        x1Norm   = norm(Ab[.., n+1..n+1], normType.norm1),
        xInfNorm = norm(Ab[.., n+1..n+1], normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
proc gaxpyMinus(A: [],
                x: [?xD],
                y: [?yD]) {
  var res: [1..n] elemType;

  forall i in 1..n do
    res[i] = (+ reduce [j in xD] (A[i,j] * x[j])) - y[i,n+1];

  return res;
}

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// Use the distributions we need for this computation
//
use d, r, f;

//
// The number of matrices and the element type of those matrices
// indexType can be int or int(64), elemType can be real or complex
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 8;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

// These are solely to make the testing system happy given the COMPOPTS file.
// To be removed once COMPOPTS becomes a non-issue.
config var reproducible = false, verbose = false;

//
// The program entry point, currently module initialization
//
  printConfiguration();

  //
  // Compute targetLocales - required for Dimensional.
  // We hard-code 2 dimensions.
  //
  var targetIds: domain(2);
  var targetLocales: [targetIds] locale;
  setupTargetLocalesArray(targetIds, targetLocales, Locales);

  // Here are the dimensions of our grid of locales.
  const tl1 = targetIds.dim(1).length,
        tl2 = targetIds.dim(2).length;
  if printParams && printStats then
    writeln("target locales ", tl1, " x ", tl2);

  // Create the dimensional descriptors
  const
    bdim1 = new idist(lowIdx=1, blockSize=blkSize, numLocales=tl1),
    rdim1 = new vdist(tl1),

    bdim2 = new idist(lowIdx=1, blockSize=blkSize, numLocales=tl2),
    rdim2 = new vdist(tl2);

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  // We use 'AbD' instead of 'MatVectSpace' throughout.
  //
  const AbD: domain(2, indexType)
          dmapped DimensionalDist(targetLocales, bdim1, bdim2, "dim")
          = [1..n, 1..n+1],
        MatrixSpace = AbD[.., ..n];

  var Ab : [AbD] elemType,           // the matrix A and vector b
      piv: [1..n] indexType;         // a vector of pivot values

  //
  // Create the 1-d replicated arrays for schurComplement().
  //
  const
    replAD = [1..n, 1..blkSize]
      dmapped DimensionalDist(targetLocales, bdim1, rdim2, "distBR"),
    replBD = [1..blkSize, 1..n+1]
      dmapped DimensionalDist(targetLocales, rdim1, bdim2, "distRB");

  var replA: [replAD] elemType,
      replB: [replBD] elemType;

  initAB();

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, piv);                 // compute the LU factorization

  var x = backwardSub(n);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(x);
  printResults(validAnswer, execTime);

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
proc LUFactorize(n: indexType,
                piv: [1..n] indexType) {

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    if printStats then writeln("processing block ", blk);

    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(l, piv);
    updateBlockRow(tl, tr);

    //
    // update trailing submatrix (if any)
    //
    schurComplement(bl, tr, br);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The 'a' region was 'bl' in the calling
//     |aaaaa|.....|.....|.....|  function but called AD here.  Similarly,
//     +-----+-----+-----+-----+  'b' was 'tr' in the calling code, but BD
//     |aaaaa|.....|.....|.....|  here.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
proc schurComplement(AD: domain, BD: domain, Rest: domain) {

  // Prevent replication of unequal-sized slices
  if Rest.numIndices == 0 then return;

  //
  // Copy data into replicated arrays so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.
  //
  coforall dest in targetLocales[targetIds.dim(1).high, targetIds.dim(2)] do
    on dest do
      // replA on tgLocales[d1,i] gets a copy of Ab from tgLocales[d1,..]
      replA = Ab[1..n, AD.dim(2)];

  coforall dest in targetLocales[targetIds.dim(1), targetIds.dim(2).high] do
    on dest do
      // replB on tgLocales[i,d2] gets a copy of Ab from tgLocales[..,d2]
      replB = Ab[BD.dim(1), 1..n+1];

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in Rest by (blkSize, blkSize) {
    // workaround: localize Rest explicitly
    const RestLcl = Rest;

    local {
      for a in (RestLcl.dim(1))(row..#blkSize) do
        for w in 1..blkSize do
          for b in (RestLcl.dim(2))(col..#blkSize) do
            Ab[a,b] -= replA[a,w] * replB[w,b];
    }
  }
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
proc panelSolve(
               panel: domain,
               piv: [] indexType) {

  for k in panel.dim(2) {             // iterate through the columns
    var col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.numIndices == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const ( , (pivotRow, )) = maxloc reduce(abs(Ab(col)), col);

    // Capture the pivot value explicitly (note that result of maxloc
    // is absolute value, so it can't be used directly).
    //
    const pivotVal = Ab[pivotRow, k];

    // Swap the current row with the pivot row and update the pivot vector
    // to reflect that
    Ab[k..k, ..] <=> Ab[pivotRow..pivotRow, ..];
    piv[k] <=> piv[pivotRow];

    if (pivotVal == 0) then
      halt("Matrix cannot be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot value
    Ab[k+1.., k..k] /= pivotVal;

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      Ab[i,j] -= Ab[i,k] * Ab[k,j];

/*  Prospective improvements to the above:

    // Replicate Ab[panel[k, k+1..]] across column of locales
    // * Could store as Dimensional(Replicated, BlockCyclic)
    // * Or as completely replicated block-size row vector

    //
    // Option 1: most flexible w.r.t. Ab's distribution
    //  Note: assignment to AbRowRepl assumes all replicants will
    //        be assigned
    //

    const AbRowReplDom: domain(2)
                        dmapped Dimensional((Replicated, Ab.dimdist(2)),
                                            targetLocales=Ab.targetLocales)
                      = panel[k..k, k+1..];
    const AbRowRepl: [AbRowReplDom] real = Ab[AbRowReplDom];

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      Ab[i,j] -= Ab[i,k] * AbRowRepl[k,j];

    //
    // Option 2: build in assumption that block column is local to a single
    // locale
    //
    const AbRowReplDom: domain(1)
                        dmapped Replicated(targetLocales=Ab.targetLocales[.., Ab.domain.indexToLocaleDim(dim=2, idx=k)])
                      = [k+1..panel.dim(2).high];

    var AbRowRepl: [AbRowReplDom] = Ab[k, AbRowReplDom.dim(1)];

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      Ab[i,j] -= Ab[i,k] * AbRowRepl[j];

    //
    // Option 3: allocate storage up-front (and hoist out of loop over k)
    //
    const AbRowReplDom: domain(1)
                        dmapped Replicated(targetLocales=Ab.targetLocales[.., Ab.domain.indexToLocaleDim(dim=2, idx=k)])
                      = [panel.dim(2)];

    var AbRowRepl: [AbRowReplDom] = Ab[k, AbRowReplDom.dim(1)];

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      Ab[i,j] -= Ab[i,k] * AbRowRepl[j];

    //
    // Option 4: allocate storage up-front (and hoist out of loop over k)
    // across all locales, but restrict assignment to the active column;
    // hoist completely out of this routine; but have to specify coordinates
    // using some fixed scheme (e.g., 0..#blocksize) and adjust global
    // indexing scheme to match
    //
    const AbRowReplDom: domain(1)
                        dmapped Replicated()
                      = [0..#blkSize];

    //
    // TODO: How do we restrict this to only certain locales?
    // idea: it would be nice to have dimensional support some queries
    // along these lines:  "Which locale subarray owns this subdomain
    // of indices.  One particular challenge is that the answer may be
    // disjoint or not expressible using a dense/regular array of locales.
    //
    var AbRowRepl: [AbRowReplDom] = Ab[k, panel[k, ..].dim(2)];

    // update all other values below the pivot
    forall (i,j) in panel[k+1.., k+1..] do
      Ab[i,j] -= Ab[i,k] * AbRowRepl[j%blkSize];
*/
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
proc updateBlockRow(
                   tl: domain,
                   tr: domain) {

  for row in tr.dim(1) {
    const activeRow = tr[row..row, ..],
          prevRows = tr.dim(1).low..row-1;

    forall (i,j) in activeRow do
      for k in prevRows do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
  }
}

//
// compute the backwards substitution
//
proc backwardSub(n: indexType) {
  const bd = Ab.domain.dim(1);  // or simply 1..n
  var x: [bd] elemType;

  for i in bd by -1 do
    x[i] = (Ab[i,n+1] - (+ reduce [j in i+1..bd.high] (Ab[i,j] * x[j])))
            / Ab[i,i];

  return x;
}

//
// print out the problem size and block size if requested
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
proc initAB() {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
proc verifyResults(x) {
  initAB();

  const axmbNorm = norm(gaxpyMinus(Ab[.., 1..n], x, Ab[.., n+1..n+1]), normType.normInf);

  const a1norm   = norm(Ab[.., 1..n], normType.norm1),
        aInfNorm = norm(Ab[.., 1..n], normType.normInf),
        x1Norm   = norm(Ab[.., n+1..n+1], normType.norm1),
        xInfNorm = norm(Ab[.., n+1..n+1], normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
proc gaxpyMinus(A: [],
                x: [?xD],
                y: [?yD]) {
  var res: [1..n] elemType;

  forall i in 1..n do
    res[i] = (+ reduce [j in xD] (A[i,j] * x[j])) - y[i,n+1];

  return res;
}


Some updates relative to code reviews.

Some updates relative to code reviews.

git-svn-id: 5aabc979e906cfa1cd4a03d781c193b99fe6e46f@21472 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Distributions Primer
//
// This primer demonstrates uses of some of Chapel's standard
// distributions.  To use these distributions in a Chapel program,
// the respective module must be used:
//
use BlockDist, CyclicDist, BlockCycDist, ReplicatedDist;
use DimensionalDist2D, ReplicatedDim, BlockCycDim;

//
// For each distribution, we'll create a distributed domain and array
// and then initialize it just to give a brief flavor of how the
// distribution maps across locales.  Running this example on 6
// locales does a nice job of illustrating the distribution
// characteristics.
//
// All of these distributions support options to map to a different
// virtual locale grid than the one used by default (a
// multidimensional factoring of the built-in Locales array), as well
// as to control the amount of parallelism used in data parallel
// loops.  See the Standard Distributions chapter of the language spec
// for more details.
//

//
// Make the program size configurable from the command line.
//
config const n = 8;

//
// Declare a 2-dimensional domain Space that we will later use to
// initialize the distributed domains.
//
const Space = {1..n, 1..n};

//
// The Block distribution distributes a bounding box from
// n-dimensional space across the target locale array viewed as an
// n-dimensional virtual locale grid.  The bounding box is blocked
// into roughly equal portions across the locales.  Note that domains
// declared over a Block distribution can also store indices outside
// of the bounding box; the bounding box is merely used to compute
// the blocking of space.
//
// In this example, we declare a 2-dimensional Block-distributed
// domain BlockSpace and a Block-distributed array BA declared over
// the domain.
//
const BlockSpace = Space dmapped Block(boundingBox=Space);
var BA: [BlockSpace] int;

//
// To illustrate how the index set is distributed across locales,
// we'll use a forall loop to initialize each array element to the
// locale ID that stores that index/element/iteration.
//
forall ba in BA do
  ba = here.id;

//
// Output the Block-distributed array to visually see how the elements
// are partitioned across the locales.
//
writeln("Block Array Index Map");
writeln(BA);
writeln();

//
// Most of Chapel's standard distributions support an optional
// targetLocales argument that permits you to pass in your own
// array of locales to be targeted.  In general, the targetLocales
// argument should match the rank of the distribution.  So for
// example, to map a Block to a [numLocales x 1] view of the
// locale set, one could do something like this:

//
// We start by creating our own array of the locale values.  Here
// we use the standard array reshape function for convenience,
// but more generally, this array could be accessed/assigned like any
// other.
//

var MyLocaleView = {0..#numLocales, 1..1};
var MyLocales: [MyLocaleView] locale = reshape(Locales, MyLocaleView);

//
// Then we'll declare a distributed domain/array that targets
// this view of the locales:
//

const BlockSpace2 = Space dmapped Block(boundingBox=Space,
                                        targetLocales=MyLocales);
var BA2: [BlockSpace2] int;

//
// Then we'll do a similar computation as before to verify where
// everything ended up:
//
forall ba in BA2 do
  ba = here.id;

writeln("Block Array Index Map");
writeln(BA2);
writeln();

//
// Next, we'll perform a similar computation for the Cyclic distribution.
// Cyclic distributions start at a designated n-dimensional index and
// distribute the n-dimensional space across an n-dimensional array
// of locales in a round-robin fashion (in each dimension).  As with
// the Block distribution, domains may be declared using the
// distribution who have lower indices that the starting index; that
// value should just be considered a parameterization of how the
// distribution is defined.
//
const CyclicSpace = Space dmapped Cyclic(startIdx=Space.low);
var CA: [CyclicSpace] int;

forall ca in CA do
  ca = here.id;

writeln("Cyclic Array Index Map");
writeln(CA);
writeln();

//
// Next, we'll declare a Block-Cyclic distribution.  These
// distributions also deal out indices in a round-robin fashion,
// but rather than dealing out singleton indices, they deal out blocks
// of indices.  Thus, the BlockCyclic distribution is parameterized
// by a starting index (as with Cyclic) and a block size (per
// dimension) specifying how large the chunks to be dealt out are.
//
const BlkCycSpace = Space dmapped BlockCyclic(startIdx=Space.low,
                                              blocksize=(2, 3));
var BCA: [BlkCycSpace] int;

forall bca in BCA do
  bca = here.id;

writeln("Block-Cyclic Array Index Map");
writeln(BCA);
writeln();

//
// The ReplicatedDist distribution is different: each of the
// original domain's indices - and the corresponding array elements -
// is replicated onto each locale. (Note: consistency among these
// array replicands is NOT maintained automatically.)
//
// This replication is observable in some cases but not others,
// as shown below. Note: this behavior may change in the future.
//
const ReplicatedSpace = Space dmapped ReplicatedDist();
var RA: [ReplicatedSpace] int;

// The replication is observable - this visits each replicand.
forall ra in RA do
  ra = here.id;

writeln("Replicated Array Index Map, ", RA.numElements, " elements total");
writeln(RA);
writeln();

//
// The replication is observable when the replicated array is
// on the left-hand side. If the right-hand side is not replicated,
// it is copied into each replicand.
// We illustrate this using a non-distributed array.
//
var A: [Space] int = [(i,j) in Space] i*100 + j;
RA = A;
writeln("Replicated Array after being array-assigned into");
writeln(RA);
writeln();

//
// Analogously, each replicand will be visited and
// other participated expressions will be computed on each locale
// (a) when the replicated array is assigned a scalar:
//       RA = 5;
// (b) when it appears first in a zippered forall loop:
//       forall (ra, a) in zip(RA, A) do ...;
// (c) when it appears in a for loop:
//       for ra in RA do ...;
//
// Zippering (RA,A) or (A,RA) in a 'for' loop will generate
// an error due to their different number of elements.

// Let RA store the Index Map again, for the examples below.
forall ra in RA do
  ra = here.id;

//
// Only the local replicand is accessed - replication is NOT observable
// and consistency is NOT maintained - when:
// (a) the replicated array is indexed - an individual element is read...
//
on Locales(0) do
  writeln("on ", here, ": ", RA(Space.low));
on Locales(LocaleSpace.high) do
  writeln("on ", here, ": ", RA(Space.low));
writeln();

// ...or an individual element is written;
on Locales(LocaleSpace.high) do
  RA(Space.low) = 7777;

writeln("Replicated Array after being indexed into");
writeln(RA);
writeln();

//
// (b) the replicated array is on the right-hand side of an assignment...
//
on Locales(LocaleSpace.high) do
  A = RA + 4;
writeln("Non-Replicated Array after assignment from Replicated Array + 4");
writeln(A);
writeln();

//
// (c) ...or, generally, the replicated array or domain participates
//     in a zippered forall loop, but not in the first position.
//     The loop could look like:
//
//       forall (a, (i,j), ra) in (A, ReplicatedSpace, RA) do ...;
//

//
// The DimensionalDist2D distribution lets us build a 2D distribution
// as a composition of specifiers for individual dimensions.
// Under such a "dimensional" distribution each dimension is handled
// independently of the other.
//
// The dimension specifiers are similar to the corresponding multi-dimensional
// distributions in constructor arguments and index-to-locale mapping rules.
// However, instead of an array of locales, a specifier constructor
// accepts just the number of locales that the indices in the corresponding
// dimension will be distributed across.
//
// The DimensionalDist2D constructor requires:
// * an [0..nl1-1, 0..nl2-1] array of locales, where
//   nl1 and nl2 are the number of locales in each dimension, and
// * two dimension specifiers, created for nl1 and nl2 locale counts, resp.
//
// Presently, the following dimension specifiers are available
// (shown here with their constructor arguments):
//
// * ReplicatedDim(numLocales)
// * BlockDim(numLocales, boundingBoxLow, boundingBoxHigh)
// * BlockCyclicDim(lowIdx, blockSize, numLocales)
//

//
// The following example creates a dimensional distribution that
// replicates over 2 locales (when available) in the first dimemsion
// and distributes using block-cyclic distribution in the second dimension.
// The example computes nl1 and nl2 and reshapes MyLocales correspondingly.
//

var (nl1, nl2) = if numLocales == 1 then (1, 1) else (2, numLocales/2);
MyLocaleView = {0..#nl1, 0..#nl2};
MyLocales = reshape(Locales[0..#nl1*nl2], MyLocaleView);

const DimReplicatedBlockcyclicSpace = Space
  dmapped DimensionalDist2D(MyLocales,
                            new ReplicatedDim(numLocales = nl1),
                            new BlockCyclicDim(numLocales = nl2,
                                               lowIdx = 1, blockSize = 2));

var DRBA: [DimReplicatedBlockcyclicSpace] int;

// The ReplicatedDim specifier always accesses the local replicand.
// (This differs from how the ReplicatedDist distribution works.)
//
// This example visits each replicand. The behavior is the same
// regardless of the second index into MyLocales below.

for locId1 in 0..#nl1 do on MyLocales[locId1, 0] {

  forall drba in DRBA do
    drba = here.id;

  writeln("Dimensional2D(Replicated,BlockCyclic) Array Index Map",
          " from ", here);

  // Technicality: 'writeln(DRBA)' would read DRBA always on Locale 0.
  // Since we want to see what DRBA contains on the current locale,
  // we use 'Helper' that is mapped using the default distribution.
  // 'Helper = DRBA' captures the view of DRBA on the current locale,
  // which we then print out.

  const Helper: [Space] int = DRBA;
  writeln(Helper);
  writeln();

}

//
// Distributions Primer
//
// This primer demonstrates uses of some of Chapel's standard
// distributions.  To use these distributions in a Chapel program,
// the respective module must be used:
//
use BlockDist, CyclicDist, BlockCycDist, ReplicatedDist;
use DimensionalDist2D, ReplicatedDim, BlockCycDim;

//
// For each distribution, we'll create a distributed domain and array
// and then initialize it just to give a brief flavor of how the
// distribution maps across locales.  Running this example on 6
// locales does a nice job of illustrating the distribution
// characteristics.
//
// All of these distributions support options to map to a different
// virtual locale grid than the one used by default (a
// multidimensional factoring of the built-in Locales array), as well
// as to control the amount of parallelism used in data parallel
// loops.  See the Standard Distributions chapter of the language spec
// for more details.
//

//
// Make the program size configurable from the command line.
//
config const n = 8;

//
// Declare a 2-dimensional domain Space that we will later use to
// initialize the distributed domains.
//
const Space = {1..n, 1..n};

//
// The Block distribution distributes a bounding box from
// n-dimensional space across the target locale array viewed as an
// n-dimensional virtual locale grid.  The bounding box is blocked
// into roughly equal portions across the locales.  Note that domains
// declared over a Block distribution can also store indices outside
// of the bounding box; the bounding box is merely used to compute
// the blocking of space.
//
// In this example, we declare a 2-dimensional Block-distributed
// domain BlockSpace and a Block-distributed array BA declared over
// the domain.
//
const BlockSpace = Space dmapped Block(boundingBox=Space);
var BA: [BlockSpace] int;

//
// To illustrate how the index set is distributed across locales,
// we'll use a forall loop to initialize each array element to the
// locale ID that stores that index/element/iteration.
//
forall ba in BA do
  ba = __primitive("_wide_get_node", here);

//
// Output the Block-distributed array to visually see how the elements
// are partitioned across the locales.
//
writeln("Block Array Index Map");
writeln(BA);
writeln();

//
// Most of Chapel's standard distributions support an optional
// targetLocales argument that permits you to pass in your own
// array of locales to be targeted.  In general, the targetLocales
// argument should match the rank of the distribution.  So for
// example, to map a Block to a [numLocales x 1] view of the
// locale set, one could do something like this:

//
// We start by creating our own array of the locale values.  Here
// we use the standard array reshape function for convenience,
// but more generally, this array could be accessed/assigned like any
// other.
//

var MyLocaleView = {0..#numLocales, 1..1};
var MyLocales: [MyLocaleView] locale = reshape(Locales, MyLocaleView);

//
// Then we'll declare a distributed domain/array that targets
// this view of the locales:
//

const BlockSpace2 = Space dmapped Block(boundingBox=Space,
                                        targetLocales=MyLocales);
var BA2: [BlockSpace2] int;

//
// Then we'll do a similar computation as before to verify where
// everything ended up:
//
forall ba in BA2 do
  ba = here.id;

writeln("Block Array Index Map");
writeln(BA2);
writeln();

//
// Next, we'll perform a similar computation for the Cyclic distribution.
// Cyclic distributions start at a designated n-dimensional index and
// distribute the n-dimensional space across an n-dimensional array
// of locales in a round-robin fashion (in each dimension).  As with
// the Block distribution, domains may be declared using the
// distribution who have lower indices that the starting index; that
// value should just be considered a parameterization of how the
// distribution is defined.
//
const CyclicSpace = Space dmapped Cyclic(startIdx=Space.low);
var CA: [CyclicSpace] int;

forall ca in CA do
  ca = here.id;

writeln("Cyclic Array Index Map");
writeln(CA);
writeln();

//
// Next, we'll declare a Block-Cyclic distribution.  These
// distributions also deal out indices in a round-robin fashion,
// but rather than dealing out singleton indices, they deal out blocks
// of indices.  Thus, the BlockCyclic distribution is parameterized
// by a starting index (as with Cyclic) and a block size (per
// dimension) specifying how large the chunks to be dealt out are.
//
const BlkCycSpace = Space dmapped BlockCyclic(startIdx=Space.low,
                                              blocksize=(2, 3));
var BCA: [BlkCycSpace] int;

forall bca in BCA do
  bca = here.id;

writeln("Block-Cyclic Array Index Map");
writeln(BCA);
writeln();

//
// The ReplicatedDist distribution is different: each of the
// original domain's indices - and the corresponding array elements -
// is replicated onto each locale. (Note: consistency among these
// array replicands is NOT maintained automatically.)
//
// This replication is observable in some cases but not others,
// as shown below. Note: this behavior may change in the future.
//
const ReplicatedSpace = Space dmapped ReplicatedDist();
var RA: [ReplicatedSpace] int;

// The replication is observable - this visits each replicand.
forall ra in RA do
  ra = here.id;

writeln("Replicated Array Index Map, ", RA.numElements, " elements total");
writeln(RA);
writeln();

//
// The replication is observable when the replicated array is
// on the left-hand side. If the right-hand side is not replicated,
// it is copied into each replicand.
// We illustrate this using a non-distributed array.
//
var A: [Space] int = [(i,j) in Space] i*100 + j;
RA = A;
writeln("Replicated Array after being array-assigned into");
writeln(RA);
writeln();

//
// Analogously, each replicand will be visited and
// other participated expressions will be computed on each locale
// (a) when the replicated array is assigned a scalar:
//       RA = 5;
// (b) when it appears first in a zippered forall loop:
//       forall (ra, a) in zip(RA, A) do ...;
// (c) when it appears in a for loop:
//       for ra in RA do ...;
//
// Zippering (RA,A) or (A,RA) in a 'for' loop will generate
// an error due to their different number of elements.

// Let RA store the Index Map again, for the examples below.
forall ra in RA do
  ra = here.id;

//
// Only the local replicand is accessed - replication is NOT observable
// and consistency is NOT maintained - when:
// (a) the replicated array is indexed - an individual element is read...
//
on Locales(0) do
  writeln("on ", here, ": ", RA(Space.low));
on Locales(LocaleSpace.high) do
  writeln("on ", here, ": ", RA(Space.low));
writeln();

// ...or an individual element is written;
on Locales(LocaleSpace.high) do
  RA(Space.low) = 7777;

writeln("Replicated Array after being indexed into");
writeln(RA);
writeln();

//
// (b) the replicated array is on the right-hand side of an assignment...
//
on Locales(LocaleSpace.high) do
  A = RA + 4;
writeln("Non-Replicated Array after assignment from Replicated Array + 4");
writeln(A);
writeln();

//
// (c) ...or, generally, the replicated array or domain participates
//     in a zippered forall loop, but not in the first position.
//     The loop could look like:
//
//       forall (a, (i,j), ra) in (A, ReplicatedSpace, RA) do ...;
//

//
// The DimensionalDist2D distribution lets us build a 2D distribution
// as a composition of specifiers for individual dimensions.
// Under such a "dimensional" distribution each dimension is handled
// independently of the other.
//
// The dimension specifiers are similar to the corresponding multi-dimensional
// distributions in constructor arguments and index-to-locale mapping rules.
// However, instead of an array of locales, a specifier constructor
// accepts just the number of locales that the indices in the corresponding
// dimension will be distributed across.
//
// The DimensionalDist2D constructor requires:
// * an [0..nl1-1, 0..nl2-1] array of locales, where
//   nl1 and nl2 are the number of locales in each dimension, and
// * two dimension specifiers, created for nl1 and nl2 locale counts, resp.
//
// Presently, the following dimension specifiers are available
// (shown here with their constructor arguments):
//
// * ReplicatedDim(numLocales)
// * BlockDim(numLocales, boundingBoxLow, boundingBoxHigh)
// * BlockCyclicDim(lowIdx, blockSize, numLocales)
//

//
// The following example creates a dimensional distribution that
// replicates over 2 locales (when available) in the first dimemsion
// and distributes using block-cyclic distribution in the second dimension.
// The example computes nl1 and nl2 and reshapes MyLocales correspondingly.
//

var (nl1, nl2) = if numLocales == 1 then (1, 1) else (2, numLocales/2);
MyLocaleView = {0..#nl1, 0..#nl2};
MyLocales = reshape(Locales[0..#nl1*nl2], MyLocaleView);

const DimReplicatedBlockcyclicSpace = Space
  dmapped DimensionalDist2D(MyLocales,
                            new ReplicatedDim(numLocales = nl1),
                            new BlockCyclicDim(numLocales = nl2,
                                               lowIdx = 1, blockSize = 2));

var DRBA: [DimReplicatedBlockcyclicSpace] int;

// The ReplicatedDim specifier always accesses the local replicand.
// (This differs from how the ReplicatedDist distribution works.)
//
// This example visits each replicand. The behavior is the same
// regardless of the second index into MyLocales below.

for locId1 in 0..#nl1 do on MyLocales[locId1, 0] {

  forall drba in DRBA do
    drba = here.id;

  writeln("Dimensional2D(Replicated,BlockCyclic) Array Index Map",
          " from ", here);

  // Technicality: 'writeln(DRBA)' would read DRBA always on Locale 0.
  // Since we want to see what DRBA contains on the current locale,
  // we use 'Helper' that is mapped using the default distribution.
  // 'Helper = DRBA' captures the view of DRBA on the current locale,
  // which we then print out.

  const Helper: [Space] int = DRBA;
  writeln(Helper);
  writeln();

}