Enable skipping the verification step.

Enable skipping the verification step.

Sometimes verification is unnecessary or not wanted.  Allow the user to
skip it in those cases.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20905 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for Block distributions and Timing routines
//
use BlockDist, Time;

//
// Use the user modules for computing HPCC problem sizes and for
// defining RA's random stream of values
//
use HPCCProblemSize, RARandomStream;

//
// The number of tables as well as the element and index types of
// that table
//
const numTables = 1;
type elemType = randType,
     indexType = randType;

//
// Configuration constants defining log2(problem size) -- n -- and
// the number of updates -- N_U
//
config const n = computeProblemSize(numTables, elemType,
                                    returnLog2=true, retType=indexType),
             N_U = 2**(n+2);

//
// Constants defining the problem size (m) and a bit mask for table
// indexing
//
const m = 2**n,
      indexMask = m-1;

//
// Do verification?
//
config const verify = true;

//
// Unless we protect against it, the verification pass can get wrong
// answers due to races in the same way that the main computation can.
// This can lead to a total error count that exceeds the threshold,
// causing a spurious failure report.  We protect against errors in
// verification by using sync variables to single-thread accesses to
// blocks of table elements, with no more than verifyBlockSize elements
// per block.  As a rule of thumb, verifyBlockSize should be chosen such
// that there are a few times as many blocks (thus sync variables) as
// the maximum number of tasks that could actually be running at any one
// time.  The "few times" multiplier, named coreConcurrencyFactor here,
// accounts for two effects.  One is that task switching during critical
// sections can make it seem as though more tasks are competing for
// access to those critical sections than there are hardware cores.  The
// other is that a smaller block size reduces the likelihood that two
// tasks will try to work on table elements in the same block at the
// same time.  Of course, reducing the protection block size increases
// the number of protection blocks and thus the amount of memory needed
// for sync variables.  So we have the classic tradeoff: using a higher
// coreConcurrencyFactor increases performance because it reduces the
// number of single-threading conflicts, but it means that the program
// requires more memory.  Here we set coreConcurrencyFactor to 4 by
// default, but that value was chosen intuitively rather than as a
// result of careful study.
//
config const coreConcurrencyFactor = 4;
config const verifyBlockSize = m / (numLocales
                                    * here.numCores
                                    * coreConcurrencyFactor):indexType;

//
// Configuration constant defining the number of errors to allow (as a
// fraction of the number of updates, N_U)
//
config const errorTolerance = 1e-2;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// Use "on" statement in kernel, or remote memory references?
//
config const useOn = (CHPL_COMM != "ugni");

//
// TableDist is a 1D block distribution for domains storing indices
// of type "indexType", and it is computed by blocking the bounding
// box 0..m-1 across the set of locales.  UpdateDist is a similar
// distribution that is computed by blocking the indices 0..N_U-1
// across the locales.
//
const TableDist = new dmap(new Block(boundingBox={0..m-1})),
      UpdateDist = new dmap(new Block(boundingBox={0..N_U-1}));

//
// TableSpace describes the index set for the table.  It is a 1D
// domain storing indices of type indexType, it is distributed
// according to TableDist, and it contains the indices 0..m-1.
// Updates is an index set describing the set of updates to be made.
// It is distributed according to UpdateDist and contains the
// indices 0..N_U-1.
//
const TableSpace: domain(1, indexType) dmapped TableDist = {0..m-1},
      Updates: domain(1, indexType) dmapped UpdateDist = {0..N_U-1};

//
// The program entry point
//
proc main() {
  //
  // T is the distributed table itself, storing a variable of type
  // elemType for each index in TableSpace.
  //
  var T: [TableSpace] elemType;

  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // In parallel, initialize the table such that each position
  // contains its index.  "[i in TableSpace]" is shorthand for "forall
  // i in TableSpace"
  //
  [i in TableSpace] T(i) = i;

  const startTime = getCurrentTime();              // capture the start time

  //
  // The main computation: Iterate over the set of updates and the
  // stream of random values in a parallel, zippered manner, dropping
  // the update index on the ground and storing the random value
  // in r.  Depending on the value of the useOn config variable, we
  // either use an on-clause to force the table updates to be executed
  // on the locales which own the table elements in question, or we
  // do the updates using direct GETs and PUTs.  Either way could be
  // faster, depending on the characteristics of the paricular comm
  // layer in use.  Compute the update using r both to compute the
  // index and as the update value.
  //
  if (useOn) then
    forall (_, r) in zip(Updates, RAStream()) do
      on TableDist.idxToLocale(r & indexMask) do {
        const myR = r;
        local {
          T(myR & indexMask) ^= myR;
        }
      }
  else
    forall (_, r) in zip(Updates, RAStream()) do
      T(r & indexMask) ^= r;

  const execTime = getCurrentTime() - startTime;   // capture the elapsed time

  const validAnswer = verifyResults(T);            // verify the updates
  printResults(validAnswer, execTime);             // print the results
}

//
// Print the problem size and number of updates
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numTables, m);
    writeln("Number of updates = ", N_U, "\n");
  }
}

//
// Verify that the computation is correct
//
proc verifyResults(T) {
  if (!verify) then return true;

  //
  // We protect against errors in verification by using sync variables
  // to protect accesses to blocks of table elements.  The configuration
  // constant verifyBlockSize is the initial, user adjustable, setting
  // of the size of a protection block.  Its actual size is the largest
  // power of 2 less than or equal to verifyBlockSize.  This allows us
  // to use an inexpensive mask operation to compute the strided sync
  // var array index that corresponds to a given table index.  It may
  // also cause us to use more memory for sync variables than we would
  // if we used verifyBlockSize directly, but if so it will also provide
  // more conflict avoidance.
  //
  const lockStride = if verifyBlockSize < 1
                     then 1
                     else 2 ** log2(verifyBlockSize),
        lockIndexMask = indexMask & ~(lockStride - 1);
  const lockSpace = TableSpace by lockStride;
  var locks$: [lockSpace] sync bool;

  //
  // Print the table, if requested
  //
  if (printArrays) then writeln("After updates, T is: ", T, "\n");

  //
  // Reverse the updates by recomputing them, this time using sync
  // variables to ensure no conflicting updates.  The sync variable
  // that protects a given table element will usually be on the same
  // locale with that table element, but not always, so we cannot
  // reference it safely in the "local" statement.
  //
  if (useOn) then
    forall (_, r) in zip(Updates, RAStream()) do
      on TableDist.idxToLocale(r & indexMask) do {
        const myR = r;
        locks$(myR & lockIndexMask) = true;
        local {
          T(myR & indexMask) ^= myR;
        }
        locks$(myR & lockIndexMask);
      }
  else
    forall (_, r) in zip(Updates, RAStream()) do {
      locks$(r & lockIndexMask) = true;
      T(r & indexMask) ^= r;
      locks$(r & lockIndexMask);
    }

  //
  // Print the table again after the updates have been reversed
  //
  if (printArrays) then writeln("After verification, T is: ", T, "\n");

  //
  // Compute the number of table positions that weren't reverted
  // correctly.  This is an indication of the number of conflicting
  // updates.
  //
  const numErrors = + reduce [i in TableSpace] (T(i) != i);
  if (printStats) then writeln("Number of errors is: ", numErrors, "\n");

  //
  // Return whether or not the number of errors was within the benchmark's
  // tolerance.
  //
  return numErrors <= (errorTolerance * N_U);
}

//
// Print out success/failure, the execution time, and the GUPS value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (GUPS) = ", (N_U / execTime) * 1e-9);
  }
}

//
// Use standard modules for Block distributions and Timing routines
//
use BlockDist, Time;

//
// Use the user modules for computing HPCC problem sizes and for
// defining RA's random stream of values
//
use HPCCProblemSize, RARandomStream;

//
// The number of tables as well as the element and index types of
// that table
//
const numTables = 1;
type elemType = randType,
     indexType = randType;

//
// Configuration constants defining log2(problem size) -- n -- and
// the number of updates -- N_U
//
config const n = computeProblemSize(numTables, elemType,
                                    returnLog2=true, retType=indexType),
             N_U = 2**(n+2);

//
// Constants defining the problem size (m) and a bit mask for table
// indexing
//
const m = 2**n,
      indexMask = m-1;

//
// Unless we protect against it, the verification pass can get wrong
// answers due to races in the same way that the main computation can.
// This can lead to a total error count that exceeds the threshold,
// causing a spurious failure report.  We protect against errors in
// verification by using sync variables to single-thread accesses to
// blocks of table elements, with no more than verifyBlockSize elements
// per block.  As a rule of thumb, verifyBlockSize should be chosen such
// that there are a few times as many blocks (thus sync variables) as
// the maximum number of tasks that could actually be running at any one
// time.  The "few times" multiplier, named coreConcurrencyFactor here,
// accounts for two effects.  One is that task switching during critical
// sections can make it seem as though more tasks are competing for
// access to those critical sections than there are hardware cores.  The
// other is that a smaller block size reduces the likelihood that two
// tasks will try to work on table elements in the same block at the
// same time.  Of course, reducing the protection block size increases
// the number of protection blocks and thus the amount of memory needed
// for sync variables.  So we have the classic tradeoff: using a higher
// coreConcurrencyFactor increases performance because it reduces the
// number of single-threading conflicts, but it means that the program
// requires more memory.  Here we set coreConcurrencyFactor to 4 by
// default, but that value was chosen intuitively rather than as a
// result of careful study.
//
config const coreConcurrencyFactor = 4;
config const verifyBlockSize = m / (numLocales
                                    * here.numCores
                                    * coreConcurrencyFactor):indexType;

//
// Configuration constant defining the number of errors to allow (as a
// fraction of the number of updates, N_U)
//
config const errorTolerance = 1e-2;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// Use "on" statement in kernel, or remote memory references?
//
config const useOn = (CHPL_COMM != "ugni");

//
// TableDist is a 1D block distribution for domains storing indices
// of type "indexType", and it is computed by blocking the bounding
// box 0..m-1 across the set of locales.  UpdateDist is a similar
// distribution that is computed by blocking the indices 0..N_U-1
// across the locales.
//
const TableDist = new dmap(new Block(boundingBox={0..m-1})),
      UpdateDist = new dmap(new Block(boundingBox={0..N_U-1}));

//
// TableSpace describes the index set for the table.  It is a 1D
// domain storing indices of type indexType, it is distributed
// according to TableDist, and it contains the indices 0..m-1.
// Updates is an index set describing the set of updates to be made.
// It is distributed according to UpdateDist and contains the
// indices 0..N_U-1.
//
const TableSpace: domain(1, indexType) dmapped TableDist = {0..m-1},
      Updates: domain(1, indexType) dmapped UpdateDist = {0..N_U-1};

//
// The program entry point
//
proc main() {
  //
  // T is the distributed table itself, storing a variable of type
  // elemType for each index in TableSpace.
  //
  var T: [TableSpace] elemType;

  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // In parallel, initialize the table such that each position
  // contains its index.  "[i in TableSpace]" is shorthand for "forall
  // i in TableSpace"
  //
  [i in TableSpace] T(i) = i;

  const startTime = getCurrentTime();              // capture the start time

  //
  // The main computation: Iterate over the set of updates and the
  // stream of random values in a parallel, zippered manner, dropping
  // the update index on the ground and storing the random value
  // in r.  Depending on the value of the useOn config variable, we
  // either use an on-clause to force the table updates to be executed
  // on the locales which own the table elements in question, or we
  // do the updates using direct GETs and PUTs.  Either way could be
  // faster, depending on the characteristics of the paricular comm
  // layer in use.  Compute the update using r both to compute the
  // index and as the update value.
  //
  if (useOn) then
    forall (_, r) in zip(Updates, RAStream()) do
      on TableDist.idxToLocale(r & indexMask) do {
        const myR = r;
        local {
          T(myR & indexMask) ^= myR;
        }
      }
  else
    forall (_, r) in zip(Updates, RAStream()) do
      T(r & indexMask) ^= r;

  const execTime = getCurrentTime() - startTime;   // capture the elapsed time

  const validAnswer = verifyResults(T);            // verify the updates
  printResults(validAnswer, execTime);             // print the results
}

//
// Print the problem size and number of updates
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numTables, m);
    writeln("Number of updates = ", N_U, "\n");
  }
}

//
// Verify that the computation is correct
//
proc verifyResults(T) {
  //
  // We protect against errors in verification by using sync variables
  // to protect accesses to blocks of table elements.  The configuration
  // constant verifyBlockSize is the initial, user adjustable, setting
  // of the size of a protection block.  Its actual size is the largest
  // power of 2 less than or equal to verifyBlockSize.  This allows us
  // to use an inexpensive mask operation to compute the strided sync
  // var array index that corresponds to a given table index.  It may
  // also cause us to use more memory for sync variables than we would
  // if we used verifyBlockSize directly, but if so it will also provide
  // more conflict avoidance.
  //
  const lockStride = if verifyBlockSize < 1
                     then 1
                     else 2 ** log2(verifyBlockSize),
        lockIndexMask = indexMask & ~(lockStride - 1);
  const lockSpace = TableSpace by lockStride;
  var locks$: [lockSpace] sync bool;

  //
  // Print the table, if requested
  //
  if (printArrays) then writeln("After updates, T is: ", T, "\n");

  //
  // Reverse the updates by recomputing them, this time using sync
  // variables to ensure no conflicting updates.  The sync variable
  // that protects a given table element will usually be on the same
  // locale with that table element, but not always, so we cannot
  // reference it safely in the "local" statement.
  //
  if (useOn) then
    forall (_, r) in zip(Updates, RAStream()) do
      on TableDist.idxToLocale(r & indexMask) do {
        const myR = r;
        locks$(myR & lockIndexMask) = true;
        local {
          T(myR & indexMask) ^= myR;
        }
        locks$(myR & lockIndexMask);
      }
  else
    forall (_, r) in zip(Updates, RAStream()) do {
      locks$(r & lockIndexMask) = true;
      T(r & indexMask) ^= r;
      locks$(r & lockIndexMask);
    }

  //
  // Print the table again after the updates have been reversed
  //
  if (printArrays) then writeln("After verification, T is: ", T, "\n");

  //
  // Compute the number of table positions that weren't reverted
  // correctly.  This is an indication of the number of conflicting
  // updates.
  //
  const numErrors = + reduce [i in TableSpace] (T(i) != i);
  if (printStats) then writeln("Number of errors is: ", numErrors, "\n");

  //
  // Return whether or not the number of errors was within the benchmark's
  // tolerance.
  //
  return numErrors <= (errorTolerance * N_U);
}

//
// Print out success/failure, the execution time, and the GUPS value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (GUPS) = ", (N_U / execTime) * 1e-9);
  }
}


Added 'stridable' parameter to the Block1D array and domain classes.
This seems to get this distribution up and running.

Removed buildEmptyDomain method.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14600 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    writeln("Created a Block1DDist:\n", this);
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDomain(param rank: int, type idxType, param stridable: bool) {
    return new Block1DDom(idxType=idxType, dist=this, stridable=stridable);
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseDenseArithmeticDomain {

  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  // CONSTRUCTORS:

  //
  // TODO: Would like to create the following constructor, but this
  //       is problematic because the class' default constructor
  //       sets dist to nil and then tries to derefernce it.  :P
  //
  /*
  def Block1DDom(type idxType, dist: Block1DDist(idxType)) {
    writeln("in my constructor");
    this.dist = dist;
    setupLocalDoms();
  }
  */
  //
  // so doing this instead:
  //
  def initialize() {
    setupLocalDoms();
  }

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    return new Block1DArr(idxType, elemType, stridable, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setupLocalDoms();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setupLocalDoms();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist() {
    return dist;
  }

  def setupLocalDoms() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, stridable, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1, stridable);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1, stridable);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType, stridable);

  // CONSTRUCTORS:

  def initialize() {
    for localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, stridable, dom.locDoms(localeIdx));
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    writeln("Created a Block1DDist:\n", this);
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDomain(param rank: int, type idxType, param stridable: bool) {
    return new Block1DDom(idxType=idxType, dist=this);
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseDenseArithmeticDomain {

  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  //
  // TODO: use stridable (sjd: this parameter is required)
  //
  param stridable: bool = true;

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  // CONSTRUCTORS:

  //
  // TODO: Would like to create the following constructor, but this
  //       is problematic because the class' default constructor
  //       sets dist to nil and then tries to derefernce it.  :P
  //
  /*
  def Block1DDom(type idxType, dist: Block1DDist(idxType)) {
    writeln("in my constructor");
    this.dist = dist;
    setupLocalDoms();
  }
  */
  //
  // so doing this instead:
  //
  def initialize() {
    setupLocalDoms();
  }

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    return new Block1DArr(idxType, elemType, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: These are the routines that I hadn't implemented
  // myself in BradsBlock1DPar.chpl:
  //
  def buildEmptyDomain() {
    if this != nil then
      return new Block1DDom(idxType=idxType, dist=dist);
    else {
      //
      // TODO: Add constructors -- now that domains/arrays inherit, it
      // is really annoying to have arguments match in order to all the
      // parent classes first.
      //
      // TODO: This is a complete hack -- the nil distribution is going
      // to get us into trouble, and isn't really what we want to do here.
      // distributions should be creating empty domains
      //
      return new Block1DDom(idxType=idxType,
                            dist= new Block1DDist(idxType, [1..0:idxType],
                                                  Locales));
    }
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setupLocalDoms();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setupLocalDoms();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist() {
    return dist;
  }

  def setupLocalDoms() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType);

  // CONSTRUCTORS:

  def initialize() {
    for localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, dom.locDoms(localeIdx));
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}


Fixed a misleading function name. Aka

Fixed a misleading function name.
Aka

// Testing divceilpos() and divfloorpos() from UtilMath (a standard module)

use UtilMath;

var testcount = 0, errcount = 0;

proc test(type T, m: T, n: T) {
  testcount += 1;
  // actual
  const rc = divceilpos(m,n);
  const rf = divfloorpos(m,n);
  writeln(typeToString(T), "  ", m, ",", n, "  ceil ", rc, "  floor ", rf);
  // expected
  const floor = m/n;
  const ceil = floor + if floor * n == m then 0:T else 1:T;
  // verify
  if rc != ceil {
    errcount += 1;
    writeln(" *** error in divceilpos:  expected ", ceil, "  got ", rc);
  }
  if rf != floor {
    errcount += 1;
    writeln(" *** error in divfloorpos:  expected ", floor, "  got ", rf);
  }
}

// produce a bunch of numbers for use in testing
iter numbers() {
  yield 1; yield 2;  yield 3;  yield 4;
  yield 5;  yield 7;  yield 11;
  yield 64;  yield 1023;
  yield 16384; yield 32767;
}

proc tests(type TT) {
  const mm = max(TT);

  test(TT, mm, mm);
  test(TT, mm-1, mm);
  test(TT, mm, mm-1);
  test(TT, mm-1, mm-1);
  test(TT, mm, 1);
// divceilpos() and divfloorpos() do not handle 0
//test(TT, 0:TT, mm);
  for d in numbers() do
    test(TT, mm, d:TT);
  for d in numbers() do
    test(TT, d:TT, mm);
  writeln();
}

tests(int(64));
tests(int(32));
// divceilpos() and divfloorpos() handle signed ints only
//tests(uint(64));
//tests(uint(32));

//writeln(max(uint(64)));
//writeln(max(int(64)));
//writeln(max(uint(32)));
//writeln(max(int(32)));

writeln("done  ", testcount, " tests");
if errcount != 0 then
  writeln(errcount, " ERRORS");

// Testing divceilpos() and divfloorpos() from UtilMath (a standard module)

use UtilMath;

var testcount = 0, errcount = 0;

proc test(type T, m: T, n: T) {
  testcount += 1;
  // actual
  const rc = divceilpos(m,n);
  const rf = divfloorpos(m,n);
  writeln(typeToString(T), "  ", m, ",", n, "  ceil ", rc, "  floor ", rf);
  // expected
  const floor = m/n;
  const ceil = floor + if floor * n == m then 0:T else 1:T;
  // verify
  if rc != ceil {
    errcount += 1;
    writeln(" *** error in divceilpos:  expected ", ceil, "  got ", rc);
  }
  if rf != floor {
    errcount += 1;
    writeln(" *** error in divfloorpos:  expected ", floor, "  got ", rf);
  }
}

// produce a bunch of numbers for use in testing
iter nombres() {
  yield 1; yield 2;  yield 3;  yield 4;
  yield 5;  yield 7;  yield 11;
  yield 64;  yield 1023;
  yield 16384; yield 32767;
}

proc tests(type TT) {
  const mm = max(TT);

  test(TT, mm, mm);
  test(TT, mm-1, mm);
  test(TT, mm, mm-1);
  test(TT, mm-1, mm-1);
  test(TT, mm, 1);
// divceilpos() and divfloorpos() do not handle 0
//test(TT, 0:TT, mm);
  for d in nombres() do
    test(TT, mm, d:TT);
  for d in nombres() do
    test(TT, d:TT, mm);
  writeln();
}

tests(int(64));
tests(int(32));
// divceilpos() and divfloorpos() handle signed ints only
//tests(uint(64));
//tests(uint(32));

//writeln(max(uint(64)));
//writeln(max(int(64)));
//writeln(max(uint(32)));
//writeln(max(int(32)));

writeln("done  ", testcount, " tests");
if errcount != 0 then
  writeln(errcount, " ERRORS");


Made stream-ep report max, avg, and min GB/s results on a local basis.  To get
the total, you need to multiply by the number of locales, like the reference
version.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@16410 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Embarassingly Parallel Implementation of STREAM Triad
//
// This version of the stream benchmark is not as elegant as
// stream.chpl.  It is a per-locale code with no communication in the
// actual computation.  It highlights the ability of a Chapel
// programmer to escape the global-view programming model and write
// codes with a fragmented, per-locale model.
//
// Comments marked with '***' point out differences with the global
// version of this benchmark in stream.chpl.
//

//
// *** Since this benchmark is written on a per-locale basis, there is no
// *** need to use the Block distribution.  The following use omits
// *** BlockDist, included in stream.chpl.
//
// Use standard modules for Timing routines, Type utility functions,
// and Random numbers
//
use Time, Types, Random;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of vectors and element type of those vectors
//
const numVectors = 3;
type elemType = real(64);

//
// *** To ensure a local problem size, we spoof the number of vectors
// *** passed to the computeProblemSize function to be the number of
// *** vectors times the number of locales.
//
// Configuration constants to set the problem size (m) and the scalar
// multiplier, alpha
//
config const m = computeProblemSize(numVectors*numLocales, elemType),
             alpha = 3.0;

//
// Configuration constants to set the number of trials to run and the
// amount of error to permit in the verification
//
config const numTrials = 10,
             epsilon = 0.0;

//
// *** There isn't (yet) a way to set the number of tasks to use for
// *** implementing a forall loop over a default array.  When there is
// *** such a way, we will want to set it via this configuration
// *** constant to get functionality like we have with stream.chpl.
//
// config const tasksPerLocale = here.numCores;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

//
// *** To ensure determinism of output, there is no more printing of the
// *** arrays in initVectors and verifyResults.
//
//
// Configuration constants to control what's printed -- benchmark
// parameters and/or statistics
//
config const printParams = true,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // *** Aggregates for collecting per-locale results for the minimum
  // *** execution time per trial, and whether verification passed
  //
  var minTimes: [LocaleSpace] real;
  var validAnswers: [LocaleSpace] bool;

  //
  // *** Fragment control so that we have a single task running on
  // *** every locale.
  //
  coforall loc in Locales do on loc {

    //
    // *** We declare these variables outside of the local block since
    // *** we'll need to access them when we write back to the global
    // *** aggregates declared above.
    //
    var validAnswer: bool;
    var execTime: [1..numTrials] real;

    //
    // *** Indicates that all of the code in this block is local to
    // *** this locale.  There is no communication.  A violation will
    // *** result in an error, though error checking is disabled with
    // *** --fast or --no-checks.
    //
    local {

      //
      // *** A, B, and C are the three local vectors
      //
      var A, B, C: [1..m] elemType;

      initVectors(B, C);    // Initialize the input vectors, B and C

      for trial in 1..numTrials {                        // loop over the trials
        const startTime = getCurrentTime();              // capture the start time

        //
        // *** The main loop looks identical to stream.chpl.  However,
        // *** in this version we are iterating over arrays that are
        // *** not distributed.
        //
        forall (a, b, c) in (A, B, C) do
          a = b + alpha * c;

        execTime(trial) = getCurrentTime() - startTime;  // store the elapsed time
      }

      validAnswer = verifyResults(A, B, C);              // verify...
    }

    //
    // *** Write times and verification result into aggregates
    // *** declared above.  These are declared over LocaleSpace so we
    // *** can write to them in parallel.
    //
    minTimes[here.id] = min reduce execTime;
    validAnswers[here.id] = validAnswer;
  }

  //
  // *** Pass minimum, average, and maximum times to printResults
  //
  printResults(&& reduce validAnswers, minTimes);
}

//
// Print the problem size and number of trials
//
def printConfiguration() {
  if (printParams) {
    //
    // *** Here we multiply m by the number of locales so that we can
    // *** print out the global problem size.
    //
    printProblemSize(elemType, numVectors, m * numLocales);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

//
// *** Both initVectors and verifyResults are almost identical to
// *** stream.chpl even though they are called with arrays that are
// *** not distributed.  For initialization, the same random stream is
// *** used on each locale.  In the global version, a single logical
// *** stream of random numbers is used across all of the locales.
// ***
// *** In this version, we've omitted a way to print the arrays.  This
// *** ensures determinism of output.  Printing the arrays also
// *** violates the locality constraint imposed by the local block
// *** from which these functions are called.
//
// Initialize vectors B and C using a random stream of values
//
def initVectors(B, C) {
  var randlist = new RandomStream(seed);

  randlist.fillRandom(B);
  randlist.fillRandom(C);

  delete randlist;
}

//
// Verify that the computation is correct
//
def verifyResults(A, B, C) {

  //
  // recompute the computation, destructively storing into B to save space
  //
  forall (b, c) in (B, C) do
    b += alpha *c;

  //
  // Compute the infinity-norm by computing the maximum reduction of the
  // absolute value of A's elements minus the new result computed in B.
  // "[i in I]" represents an expression-level loop: "forall i in I"
  //
  const infNorm = max reduce [(a,b) in (A,B)] abs(a - b);

  return (infNorm <= epsilon);    // return whether the error is acceptable
}

//
// Print out success/failure, the timings, and the GB/s value.
//
// *** Here we report maximum, average, and minimum times instead of
// *** total, average, and minimum.
//
def printResults(successful, minTimes) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    const maxTime = max reduce minTimes,
          avgTime = + reduce minTimes / numLocales,
          minTime = min reduce minTimes;
    writeln("Execution time:");
    writeln("  max = ", maxTime);
    writeln("  avg = ", avgTime);
    writeln("  min = ", minTime);

    const maxGBPerSec = numVectors * numBytes(elemType) * (m / minTime) * 1e-9,
          avgGBPerSec = numVectors * numBytes(elemType) * (m / avgTime) * 1e-9,
          minGBPerSec = numVectors * numBytes(elemType) * (m / maxTime) * 1e-9;
    writeln("Performance (GB/s):");
    writeln("  max = ", maxGBPerSec);
    writeln("  avg = ", avgGBPerSec);
    writeln("  min = ", minGBPerSec);
  }
}

//
// Embarassingly Parallel Implementation of STREAM Triad
//
// This version of the stream benchmark is not as elegant as
// stream.chpl.  It is a per-locale code with no communication in the
// actual computation.  It highlights the ability of a Chapel
// programmer to escape the global-view programming model and write
// codes with a fragmented, per-locale model.
//
// Comments marked with '***' point out differences with the global
// version of this benchmark in stream.chpl.
//

//
// *** Since this benchmark is written on a per-locale basis, there is no
// *** need to use the Block distribution.  The following use omits
// *** BlockDist, included in stream.chpl.
//
// Use standard modules for Timing routines, Type utility functions,
// and Random numbers
//
use Time, Types, Random;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of vectors and element type of those vectors
//
const numVectors = 3;
type elemType = real(64);

//
// *** To ensure a local problem size, we spoof the number of vectors
// *** passed to the computeProblemSize function to be the number of
// *** vectors times the number of locales.
//
// Configuration constants to set the problem size (m) and the scalar
// multiplier, alpha
//
config const m = computeProblemSize(numVectors*numLocales, elemType),
             alpha = 3.0;

//
// Configuration constants to set the number of trials to run and the
// amount of error to permit in the verification
//
config const numTrials = 10,
             epsilon = 0.0;

//
// *** There isn't (yet) a way to set the number of tasks to use for
// *** implementing a forall loop over a default array.  When there is
// *** such a way, we will want to set it via this configuration
// *** constant to get functionality like we have with stream.chpl.
//
// config const tasksPerLocale = here.numCores;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

//
// *** To ensure determinism of output, there is no more printing of the
// *** arrays in initVectors and verifyResults.
//
//
// Configuration constants to control what's printed -- benchmark
// parameters and/or statistics
//
config const printParams = true,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // *** Aggregates for collecting per-locale results for the minimum
  // *** execution time per trial, and whether verification passed
  //
  var minTimes: [LocaleSpace] real;
  var validAnswers: [LocaleSpace] bool;

  //
  // *** Fragment control so that we have a single task running on
  // *** every locale.
  //
  coforall loc in Locales do on loc {

    //
    // *** We declare these variables outside of the local block since
    // *** we'll need to access them when we write back to the global
    // *** aggregates declared above.
    //
    var validAnswer: bool;
    var execTime: [1..numTrials] real;

    //
    // *** Indicates that all of the code in this block is local to
    // *** this locale.  There is no communication.  A violation will
    // *** result in an error, though error checking is disabled with
    // *** --fast or --no-checks.
    //
    local {

      //
      // *** A, B, and C are the three local vectors
      //
      var A, B, C: [1..m] elemType;

      initVectors(B, C);    // Initialize the input vectors, B and C

      for trial in 1..numTrials {                        // loop over the trials
        const startTime = getCurrentTime();              // capture the start time

        //
        // *** The main loop looks identical to stream.chpl.  However,
        // *** in this version we are iterating over arrays that are
        // *** not distributed.
        //
        forall (a, b, c) in (A, B, C) do
          a = b + alpha * c;

        execTime(trial) = getCurrentTime() - startTime;  // store the elapsed time
      }

      validAnswer = verifyResults(A, B, C);              // verify...
    }

    //
    // *** Write times and verification result into aggregates
    // *** declared above.  These are declared over LocaleSpace so we
    // *** can write to them in parallel.
    //
    minTimes[here.id] = min reduce execTime;
    validAnswers[here.id] = validAnswer;
  }

  //
  // *** Pass minimum, average, and maximum times to printResults
  //
  printResults(&& reduce validAnswers, minTimes);
}

//
// Print the problem size and number of trials
//
def printConfiguration() {
  if (printParams) {
    //
    // *** Here we multiply m by the number of locales so that we can
    // *** print out the global problem size.
    //
    printProblemSize(elemType, numVectors, m * numLocales);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

//
// *** Both initVectors and verifyResults are almost identical to
// *** stream.chpl even though they are called with arrays that are
// *** not distributed.  For initialization, the same random stream is
// *** used on each locale.  In the global version, a single logical
// *** stream of random numbers is used across all of the locales.
// ***
// *** In this version, we've omitted a way to print the arrays.  This
// *** ensures determinism of output.  Printing the arrays also
// *** violates the locality constraint imposed by the local block
// *** from which these functions are called.
//
// Initialize vectors B and C using a random stream of values
//
def initVectors(B, C) {
  var randlist = new RandomStream(seed);

  randlist.fillRandom(B);
  randlist.fillRandom(C);

  delete randlist;
}

//
// Verify that the computation is correct
//
def verifyResults(A, B, C) {

  //
  // recompute the computation, destructively storing into B to save space
  //
  forall (b, c) in (B, C) do
    b += alpha *c;

  //
  // Compute the infinity-norm by computing the maximum reduction of the
  // absolute value of A's elements minus the new result computed in B.
  // "[i in I]" represents an expression-level loop: "forall i in I"
  //
  const infNorm = max reduce [(a,b) in (A,B)] abs(a - b);

  return (infNorm <= epsilon);    // return whether the error is acceptable
}

//
// Print out success/failure, the timings, and the GB/s value.
//
// *** Here we report maximum, average, and minimum times instead of
// *** total, average, and minimum.
//
def printResults(successful, minTimes) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    const maxTime = max reduce minTimes,
          avgTime = + reduce minTimes / numLocales,
          minTime = min reduce minTimes;
    writeln("Execution time:");
    writeln("  max = ", maxTime);
    writeln("  avg = ", avgTime);
    writeln("  min = ", minTime);

    const GBPerSec = numVectors * numBytes(elemType) * (numLocales * m / minTime) * 1e-9;
    writeln("Performance (GB/s) = ", GBPerSec);
  }
}