Removing unused extern declaration

Removing unused extern declaration

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20576 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for Block distributions and Timing routines
//
use BlockDist, Time;

//
// Use the user modules for computing HPCC problem sizes and for
// defining RA's random stream of values
//
use HPCCProblemSize, RARandomStream;

//
// The number of tables as well as the element and index types of
// that table
//
const numTables = 1;
type elemType = randType,
     indexType = randType;

//
// Configuration constants defining log2(problem size) -- n -- and
// the number of updates -- N_U
//
config const n = computeProblemSize(numTables, elemType,
                                    returnLog2=true, retType=indexType),
             N_U = 2**(n+2);

//
// Constants defining the problem size (m) and a bit mask for table
// indexing
//
const m = 2**n,
      indexMask = m-1;

//
// Unless we protect against it, the verification pass can get wrong
// answers due to races in the same way that the main computation can.
// This can lead to a total error count that exceeds the threshold,
// causing a spurious failure report.  We protect against errors in
// verification by using sync variables to single-thread accesses to
// blocks of table elements, with no more than verifyBlockSize elements
// per block.  As a rule of thumb, verifyBlockSize should be chosen such
// that there are a few times as many blocks (thus sync variables) as
// the maximum number of tasks that could actually be running at any one
// time.  The "few times" multiplier, named coreConcurrencyFactor here,
// accounts for two effects.  One is that task switching during critical
// sections can make it seem as though more tasks are competing for
// access to those critical sections than there are hardware cores.  The
// other is that a smaller block size reduces the likelihood that two
// tasks will try to work on table elements in the same block at the
// same time.  Of course, reducing the protection block size increases
// the number of protection blocks and thus the amount of memory needed
// for sync variables.  So we have the classic tradeoff: using a higher
// coreConcurrencyFactor increases performance because it reduces the
// number of single-threading conflicts, but it means that the program
// requires more memory.  Here we set coreConcurrencyFactor to 4 by
// default, but that value was chosen intuitively rather than as a
// result of careful study.
//
config const coreConcurrencyFactor = 4;
config const verifyBlockSize = m / (numLocales
                                    * here.numCores
                                    * coreConcurrencyFactor):indexType;

//
// Configuration constant defining the number of errors to allow (as a
// fraction of the number of updates, N_U)
//
config const errorTolerance = 1e-2;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// TableDist is a 1D block distribution for domains storing indices
// of type "indexType", and it is computed by blocking the bounding
// box 0..m-1 across the set of locales.  UpdateDist is a similar
// distribution that is computed by blocking the indices 0..N_U-1
// across the locales.
//
const TableDist = new dmap(new Block(boundingBox={0..m-1})),
      UpdateDist = new dmap(new Block(boundingBox={0..N_U-1}));

//
// TableSpace describes the index set for the table.  It is a 1D
// domain storing indices of type indexType, it is distributed
// according to TableDist, and it contains the indices 0..m-1.
// Updates is an index set describing the set of updates to be made.
// It is distributed according to UpdateDist and contains the
// indices 0..N_U-1.
//
const TableSpace: domain(1, indexType) dmapped TableDist = {0..m-1},
      Updates: domain(1, indexType) dmapped UpdateDist = {0..N_U-1};

//
// The program entry point
//
proc main() {
  //
  // T is the distributed table itself, storing a variable of type
  // elemType for each index in TableSpace.
  //
  var T: [TableSpace] atomic elemType;

  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // In parallel, initialize the table such that each position
  // contains its index.  "[i in TableSpace]" is shorthand for "forall
  // i in TableSpace"
  //
  [i in TableSpace] T(i).poke(i);

  const startTime = getCurrentTime();              // capture the start time

  //
  // The main computation: Iterate over the set of updates and the
  // stream of random values in a parallel, zippered manner, dropping
  // the update index on the ground and storing the random value
  // in r.  Compute the update using r both to compute the index and
  // as the update value.
  //
  forall (_, r) in zip(Updates, RAStream()) do
    T(r & indexMask).xor(r);

  const execTime = getCurrentTime() - startTime;   // capture the elapsed time

  const validAnswer = verifyResults(T);            // verify the updates
  printResults(validAnswer, execTime);             // print the results
}

//
// Print the problem size and number of updates
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numTables, m);
    writeln("Number of updates = ", N_U, "\n");
  }
}

//
// Verify that the computation is correct
//
proc verifyResults(T) {
  //
  // We protect against errors in verification by using sync variables
  // to protect accesses to blocks of table elements.  The configuration
  // constant verifyBlockSize is the initial, user adjustable, setting
  // of the size of a protection block.  Its actual size is the largest
  // power of 2 less than or equal to verifyBlockSize.  This allows us
  // to use an inexpensive mask operation to compute the strided sync
  // var array index that corresponds to a given table index.  It may
  // also cause us to use more memory for sync variables than we would
  // if we used verifyBlockSize directly, but if so it will also provide
  // more conflict avoidance.
  //
  const lockStride = if verifyBlockSize < 1
                     then 1
                     else 2 ** log2(verifyBlockSize),
        lockIndexMask = indexMask & ~(lockStride - 1);
  const lockSpace = TableSpace by lockStride;
  var locks$: [lockSpace] sync bool;

  //
  // Print the table, if requested
  //
  if (printArrays) then writeln("After updates, T is: ", T, "\n");

  //
  // Reverse the updates by recomputing them, this time using sync
  // variables to ensure no conflicting updates.  The sync variable
  // that protects a given table element will usually be on the same
  // locale with that table element, but not always, so we cannot
  // reference it safely in the "local" statement.
  //
   forall (_, r) in zip(Updates, RAStream()) do
     T(r & indexMask).xor(r);

  //
  // Print the table again after the updates have been reversed
  //
  if (printArrays) then writeln("After verification, T is: ", T, "\n");

  //
  // Compute the number of table positions that weren't reverted
  // correctly.  This is an indication of the number of conflicting
  // updates.
  //
  const numErrors = + reduce [i in TableSpace] (T(i).peek() != i);
  if (printStats) then writeln("Number of errors is: ", numErrors, "\n");

  //
  // Return whether or not the number of errors was within the benchmark's
  // tolerance.
  //
  return numErrors <= (errorTolerance * N_U);
}

//
// Print out success/failure, the execution time, and the GUPS value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (GUPS) = ", (N_U / execTime) * 1e-9);
  }
}

//
// Use standard modules for Block distributions and Timing routines
//
use BlockDist, Time;

//
// Use the user modules for computing HPCC problem sizes and for
// defining RA's random stream of values
//
use HPCCProblemSize, RARandomStream;

//
// The number of tables as well as the element and index types of
// that table
//
const numTables = 1;
type elemType = randType,
     indexType = randType;

//
// Configuration constants defining log2(problem size) -- n -- and
// the number of updates -- N_U
//
config const n = computeProblemSize(numTables, elemType,
                                    returnLog2=true, retType=indexType),
             N_U = 2**(n+2);

//
// Constants defining the problem size (m) and a bit mask for table
// indexing
//
const m = 2**n,
      indexMask = m-1;

//
// Unless we protect against it, the verification pass can get wrong
// answers due to races in the same way that the main computation can.
// This can lead to a total error count that exceeds the threshold,
// causing a spurious failure report.  We protect against errors in
// verification by using sync variables to single-thread accesses to
// blocks of table elements, with no more than verifyBlockSize elements
// per block.  As a rule of thumb, verifyBlockSize should be chosen such
// that there are a few times as many blocks (thus sync variables) as
// the maximum number of tasks that could actually be running at any one
// time.  The "few times" multiplier, named coreConcurrencyFactor here,
// accounts for two effects.  One is that task switching during critical
// sections can make it seem as though more tasks are competing for
// access to those critical sections than there are hardware cores.  The
// other is that a smaller block size reduces the likelihood that two
// tasks will try to work on table elements in the same block at the
// same time.  Of course, reducing the protection block size increases
// the number of protection blocks and thus the amount of memory needed
// for sync variables.  So we have the classic tradeoff: using a higher
// coreConcurrencyFactor increases performance because it reduces the
// number of single-threading conflicts, but it means that the program
// requires more memory.  Here we set coreConcurrencyFactor to 4 by
// default, but that value was chosen intuitively rather than as a
// result of careful study.
//
config const coreConcurrencyFactor = 4;
config const verifyBlockSize = m / (numLocales
                                    * here.numCores
                                    * coreConcurrencyFactor):indexType;

//
// Configuration constant defining the number of errors to allow (as a
// fraction of the number of updates, N_U)
//
config const errorTolerance = 1e-2;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// TableDist is a 1D block distribution for domains storing indices
// of type "indexType", and it is computed by blocking the bounding
// box 0..m-1 across the set of locales.  UpdateDist is a similar
// distribution that is computed by blocking the indices 0..N_U-1
// across the locales.
//
const TableDist = new dmap(new Block(boundingBox={0..m-1})),
      UpdateDist = new dmap(new Block(boundingBox={0..N_U-1}));

//
// TableSpace describes the index set for the table.  It is a 1D
// domain storing indices of type indexType, it is distributed
// according to TableDist, and it contains the indices 0..m-1.
// Updates is an index set describing the set of updates to be made.
// It is distributed according to UpdateDist and contains the
// indices 0..N_U-1.
//
const TableSpace: domain(1, indexType) dmapped TableDist = {0..m-1},
      Updates: domain(1, indexType) dmapped UpdateDist = {0..N_U-1};

extern proc ra_init(ref data: atomic elemType, numElems: int(64));

//
// The program entry point
//
proc main() {
  //
  // T is the distributed table itself, storing a variable of type
  // elemType for each index in TableSpace.
  //
  var T: [TableSpace] atomic elemType;

  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // In parallel, initialize the table such that each position
  // contains its index.  "[i in TableSpace]" is shorthand for "forall
  // i in TableSpace"
  //
  [i in TableSpace] T(i).poke(i);

  const startTime = getCurrentTime();              // capture the start time

  //
  // The main computation: Iterate over the set of updates and the
  // stream of random values in a parallel, zippered manner, dropping
  // the update index on the ground and storing the random value
  // in r.  Compute the update using r both to compute the index and
  // as the update value.
  //
  forall (_, r) in zip(Updates, RAStream()) do
    T(r & indexMask).xor(r);

  const execTime = getCurrentTime() - startTime;   // capture the elapsed time

  const validAnswer = verifyResults(T);            // verify the updates
  printResults(validAnswer, execTime);             // print the results
}

//
// Print the problem size and number of updates
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numTables, m);
    writeln("Number of updates = ", N_U, "\n");
  }
}

//
// Verify that the computation is correct
//
proc verifyResults(T) {
  //
  // We protect against errors in verification by using sync variables
  // to protect accesses to blocks of table elements.  The configuration
  // constant verifyBlockSize is the initial, user adjustable, setting
  // of the size of a protection block.  Its actual size is the largest
  // power of 2 less than or equal to verifyBlockSize.  This allows us
  // to use an inexpensive mask operation to compute the strided sync
  // var array index that corresponds to a given table index.  It may
  // also cause us to use more memory for sync variables than we would
  // if we used verifyBlockSize directly, but if so it will also provide
  // more conflict avoidance.
  //
  const lockStride = if verifyBlockSize < 1
                     then 1
                     else 2 ** log2(verifyBlockSize),
        lockIndexMask = indexMask & ~(lockStride - 1);
  const lockSpace = TableSpace by lockStride;
  var locks$: [lockSpace] sync bool;

  //
  // Print the table, if requested
  //
  if (printArrays) then writeln("After updates, T is: ", T, "\n");

  //
  // Reverse the updates by recomputing them, this time using sync
  // variables to ensure no conflicting updates.  The sync variable
  // that protects a given table element will usually be on the same
  // locale with that table element, but not always, so we cannot
  // reference it safely in the "local" statement.
  //
   forall (_, r) in zip(Updates, RAStream()) do
     T(r & indexMask).xor(r);

  //
  // Print the table again after the updates have been reversed
  //
  if (printArrays) then writeln("After verification, T is: ", T, "\n");

  //
  // Compute the number of table positions that weren't reverted
  // correctly.  This is an indication of the number of conflicting
  // updates.
  //
  const numErrors = + reduce [i in TableSpace] (T(i).peek() != i);
  if (printStats) then writeln("Number of errors is: ", numErrors, "\n");

  //
  // Return whether or not the number of errors was within the benchmark's
  // tolerance.
  //
  return numErrors <= (errorTolerance * N_U);
}

//
// Print out success/failure, the execution time, and the GUPS value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (GUPS) = ", (N_U / execTime) * 1e-9);
  }
}


Minor update to comment

Minor update to comment

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial and single-locale, though the ultimate intention
// is to also support parallel and multi-locale options.
//
// These interfaces should be considered experimental and subject to
// change for the 1.10 release.  For that reason, send in feedback if
// you would like to see changes or additional arguments/features.
// And, as always, send in bug reports if you think we got something
// wrong.
//

/* iter listdir(path: string, hidden=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * hidden: should hidden files/directories be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, hidden=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (hidden || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          //
          // use FileSystem;  // Doesn't work, see comment below
          //
          const fullpath = path + "/" + filename;
          {
            //
            // The use of this compound statement to restrict the
            // impact of the 'use' of FileSystem is unfortunate
            // (compared to placing it in the more logical place
            // above), yet seemingly required at present; otherwise
            // the 'path' argument gets shadowed by a
            // (compiler-introduced?) method coming from one of the
            // standard or internal modules.  See
            // test/modules/bradc/useFileSystemShadowsPath.chpl for
            // a smaller standalone test exhibiting the issue (or
            // uncomment the 'use' above to see it here).
            //
            use FileSystem;

            if (listlinks || !isLink(fullpath)) {
              if (dirs && isDir(fullpath)) then
                yield filename;
              else if (files && isFile(fullpath)) then
                yield filename;
            }
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 hidden=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.

     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * hidden: indicates whether to enter hidden directories
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden,
                             followlinks, sort) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

//
// Here's a parallel version
//
iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false, param tag: iterKind): string
       where tag == iterKind.standalone {

  if (sort) then
    warning("sorting has no effect for parallel invocations of walkdirs()");

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    forall subdir in subdirs {
      const fullpath = path + "/" + subdir;
      //
      // Call standalone walkdirs() iterator recursively; set sort=false since it is
      // not useful and we've already printed the warning
      //
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden, followlinks, sort=false, iterKind.standalone) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()

     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

module Chpl_rt_glob {
  extern type glob_t;
  extern proc chpl_glob(pattern:c_string, flags: c_int,
                        ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);
}

iter glob(pattern="*") {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    for i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.standalone {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    forall i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: The following leader/follower iterator assumes that
// the file system state won't change between calls because
// it calls the glob routines twice and assumes the same
// result will come back.  In leader-follower 2.0, we would
// like to store such state between calls in which case this
// should be rewritten to do so (and would require freeing
// the state at the end of the call).
//
iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.leader {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  //
  // cast is used here to ensure we create an int-based leader
  //
  const num = Chpl_rt_glob.chpl_glob_num(glb).safeCast(int);
  if (num) {
    Chpl_rt_glob.globfree(glb);

    //
    // Forward to the range type's leader
    //
    for followThis in (0..num-1).these(tag) do
      yield followThis;
  }
}

iter glob(pattern:string="*", followThis, param tag: iterKind)
       where tag == iterKind.follower {
  var glb : Chpl_rt_glob.glob_t;
  if (followThis.size != 1) then
    compilerError("glob() iterator can only be zipped with 1D iterators");
  var (r,) = followThis;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (r.high > num.safeCast(int)) then
    halt("glob() iterator zipped with something too big");
  if (num) then
    for i in r do
      //
      // safe cast is used here to turn an int into a size_t
      //
      yield Chpl_rt_glob.chpl_glob_index(glb, i.safeCast(size_t)): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: It'd be nice if we could do a leader-follower version of
// glob() -- and we almost could -- except that the glob_t state
// has to be communicated from leader to follower.
//

/* iter findfiles(startdir = ".", recursive=false, hidden=false)

   findfiles() is a simple find-like utility implemented using the
   above routines

     * startdir: where to start when looking for files
     * recursive: tells whether or not to descend recurisvely
     * hidden: tells whether or not to yield hidden
*/

iter findfiles(startdir = ".", recursive=false, hidden=false) {
  if (recursive) then
    for subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

iter findfiles(startdir = ".", recursive=false, hidden=false, param tag: iterKind) where tag == iterKind.standalone {
  if (recursive) then
    forall subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial and single-locale, though the ultimate intention
// is to also support parallel and multi-locale options.
//
// These interfaces should be considered experimental and subject to
// change for the 1.10 release.  For that reason, send in feedback if
// you would like to see changes or additional arguments/features.
// And, as always, send in bug reports if you think we got something
// wrong.
//

/* iter listdir(path: string, hidden=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * hidden: should hidden files/directories be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, hidden=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (hidden || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          //
          // use FileSystem;  // Doesn't work, see comment below
          //
          const fullpath = path + "/" + filename;
          {
            //
            // The use of this compound statement to restrict the
            // impact of the 'use' of FileSystem is unfortunate
            // (compared to placing it in the more logical place
            // above), yet seemingly required at present; otherwise
            // the 'path' argument gets shadowed by a
            // (compiler-introduced?) method coming from one of the
            // standard or internal modules.  See
            // test/modules/bradc/useFileSystemShadowsPath.chpl for
            // a smaller standalone test exhibiting the issue (or
            // uncomment the 'use' above to see it here).
            //
            use FileSystem;

            if (listlinks || !isLink(fullpath)) {
              if (dirs && isDir(fullpath)) then
                yield filename;
              else if (files && isFile(fullpath)) then
                yield filename;
            }
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 hidden=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.

     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * hidden: indicates whether to enter hidden directories
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden,
                             followlinks, sort) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

//
// Here's a parallel version
//
iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false, param tag: iterKind): string
       where tag == iterKind.standalone {

  if (sort) then
    warning("sorting has no effect for parallel invocations of walkdirs()");

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    forall subdir in subdirs {
      const fullpath = path + "/" + subdir;
      //
      // Call standalone walkdirs() iterator recursively
      //
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden, followlinks, sort=false, iterKind.standalone) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()

     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

module Chpl_rt_glob {
  extern type glob_t;
  extern proc chpl_glob(pattern:c_string, flags: c_int,
                        ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);
}

iter glob(pattern="*") {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    for i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.standalone {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    forall i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: The following leader/follower iterator assumes that
// the file system state won't change between calls because
// it calls the glob routines twice and assumes the same
// result will come back.  In leader-follower 2.0, we would
// like to store such state between calls in which case this
// should be rewritten to do so (and would require freeing
// the state at the end of the call).
//
iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.leader {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  //
  // cast is used here to ensure we create an int-based leader
  //
  const num = Chpl_rt_glob.chpl_glob_num(glb).safeCast(int);
  if (num) {
    Chpl_rt_glob.globfree(glb);

    //
    // Forward to the range type's leader
    //
    for followThis in (0..num-1).these(tag) do
      yield followThis;
  }
}

iter glob(pattern:string="*", followThis, param tag: iterKind)
       where tag == iterKind.follower {
  var glb : Chpl_rt_glob.glob_t;
  if (followThis.size != 1) then
    compilerError("glob() iterator can only be zipped with 1D iterators");
  var (r,) = followThis;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (r.high > num.safeCast(int)) then
    halt("glob() iterator zipped with something too big");
  if (num) then
    for i in r do
      //
      // safe cast is used here to turn an int into a size_t
      //
      yield Chpl_rt_glob.chpl_glob_index(glb, i.safeCast(size_t)): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: It'd be nice if we could do a leader-follower version of
// glob() -- and we almost could -- except that the glob_t state
// has to be communicated from leader to follower.
//

/* iter findfiles(startdir = ".", recursive=false, hidden=false)

   findfiles() is a simple find-like utility implemented using the
   above routines

     * startdir: where to start when looking for files
     * recursive: tells whether or not to descend recurisvely
     * hidden: tells whether or not to yield hidden
*/

iter findfiles(startdir = ".", recursive=false, hidden=false) {
  if (recursive) then
    for subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

iter findfiles(startdir = ".", recursive=false, hidden=false, param tag: iterKind) where tag == iterKind.standalone {
  if (recursive) then
    forall subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}


Adding exampels for the 'localSubdomain(s)' functions

//
// Distributions Primer
//
// This primer demonstrates uses of some of Chapel's standard
// distributions.  To use these distributions in a Chapel program,
// the respective module must be used:
//
use BlockDist, CyclicDist, BlockCycDist, ReplicatedDist;
use DimensionalDist2D, ReplicatedDim, BlockCycDim;

//
// For each distribution, we'll create a distributed domain and array
// and then initialize it just to give a brief flavor of how the
// distribution maps across locales.  Running this example on 6
// locales does a nice job of illustrating the distribution
// characteristics.
//
// All of these distributions support options to map to a different
// virtual locale grid than the one used by default (a
// multidimensional factoring of the built-in Locales array), as well
// as to control the amount of parallelism used in data parallel
// loops.  See the Standard Distributions chapter of the language spec
// for more details.
//

//
// Make the program size configurable from the command line.
//
config const n = 8;

//
// Declare a 2-dimensional domain Space that we will later use to
// initialize the distributed domains.
//
const Space = {1..n, 1..n};

//
// The Block distribution distributes a bounding box from
// n-dimensional space across the target locale array viewed as an
// n-dimensional virtual locale grid.  The bounding box is blocked
// into roughly equal portions across the locales.  Note that domains
// declared over a Block distribution can also store indices outside
// of the bounding box; the bounding box is merely used to compute
// the blocking of space.
//
// In this example, we declare a 2-dimensional Block-distributed
// domain BlockSpace and a Block-distributed array BA declared over
// the domain.
//
const BlockSpace = Space dmapped Block(boundingBox=Space);
var BA: [BlockSpace] int;

//
// To illustrate how the index set is distributed across locales,
// we'll use a forall loop to initialize each array element to the
// locale ID that stores that index/element/iteration.
//
forall ba in BA do
  ba = here.id;

//
// The 'hasSingleLocalSubdomain' method on arrays will return true if the
// index set for a locale can be represented by a single domain.
//
if !BA.hasSingleLocalSubdomain() then
  halt("For a Block distribution, the index set per locale should be \
      represented by a single domain");

//
// If the distribution's subdomains can be represented as single subdomain,
// we can use the 'localSubdomain' method to get the index set for the
// current locale.
//
// Below, we'll use the index set to confirm that the array elements have the
// correct locale id.
//

for L in Locales {
  on L {
    const indices = BA.localSubdomain();
    for i in indices {
      if BA[i] != L.id then
        halt("Error: incorrect locale id");
    }
  }
}

//
// Output the Block-distributed array to visually see how the elements
// are partitioned across the locales.
//
writeln("Block Array Index Map");
writeln(BA);
writeln();

//
// Most of Chapel's standard distributions support an optional
// targetLocales argument that permits you to pass in your own
// array of locales to be targeted.  In general, the targetLocales
// argument should match the rank of the distribution.  So for
// example, to map a Block to a [numLocales x 1] view of the
// locale set, one could do something like this:

//
// We start by creating our own array of the locale values.  Here
// we use the standard array reshape function for convenience,
// but more generally, this array could be accessed/assigned like any
// other.
//

var MyLocaleView = {0..#numLocales, 1..1};
var MyLocales: [MyLocaleView] locale = reshape(Locales, MyLocaleView);

//
// Then we'll declare a distributed domain/array that targets
// this view of the locales:
//

const BlockSpace2 = Space dmapped Block(boundingBox=Space,
                                        targetLocales=MyLocales);
var BA2: [BlockSpace2] int;

//
// Then we'll do a similar computation as before to verify where
// everything ended up:
//
forall ba in BA2 do
  ba = here.id;

writeln("Block Array Index Map");
writeln(BA2);
writeln();

//
// We can use the 'targetLocales' method available on an array to get the
// locales array used as targets.
//
for (L, ML) in zip(BA2.targetLocales(), MyLocales) do
  if L != ML then
    halt("Error: BA2.targetLocales() should equal MyLocales");

//
// Next, we'll perform a similar computation for the Cyclic distribution.
// Cyclic distributions start at a designated n-dimensional index and
// distribute the n-dimensional space across an n-dimensional array
// of locales in a round-robin fashion (in each dimension).  As with
// the Block distribution, domains may be declared using the
// distribution who have lower indices that the starting index; that
// value should just be considered a parameterization of how the
// distribution is defined.
//
const CyclicSpace = Space dmapped Cyclic(startIdx=Space.low);
var CA: [CyclicSpace] int;

forall ca in CA do
  ca = here.id;

writeln("Cyclic Array Index Map");
writeln(CA);
writeln();

//
// Next, we'll declare a Block-Cyclic distribution.  These
// distributions also deal out indices in a round-robin fashion,
// but rather than dealing out singleton indices, they deal out blocks
// of indices.  Thus, the BlockCyclic distribution is parameterized
// by a starting index (as with Cyclic) and a block size (per
// dimension) specifying how large the chunks to be dealt out are.
//
const BlkCycSpace = Space dmapped BlockCyclic(startIdx=Space.low,
                                              blocksize=(2, 3));
var BCA: [BlkCycSpace] int;

forall bca in BCA do
  bca = here.id;

writeln("Block-Cyclic Array Index Map");
writeln(BCA);
writeln();

//
// A locale's index set for a Block-Cyclic distribution cannot be represented
// by a single subdomain.
//
if BCA.hasSingleLocalSubdomain() then
  halt("A Block-Cyclic index set cannot be represented by a single subdomain");

//
// If the local index set cannot be represented by a single subdomain,
// we can use the 'localSubdomains' iterator to yield a number of domains
// that represent the whole index set.
//
for L in Locales {
  on L {
    for indices in BCA.localSubdomains() {
      for i in indices {
        if BCA[i] != L.id then
          halt("Error: incorrect locale id");
      }
    }
  }
}

//
// The ReplicatedDist distribution is different: each of the
// original domain's indices - and the corresponding array elements -
// is replicated onto each locale. (Note: consistency among these
// array replicands is NOT maintained automatically.)
//
// This replication is observable in some cases but not others,
// as shown below. Note: this behavior may change in the future.
//
const ReplicatedSpace = Space dmapped ReplicatedDist();
var RA: [ReplicatedSpace] int;

// The replication is observable - this visits each replicand.
forall ra in RA do
  ra = here.id;

writeln("Replicated Array Index Map, ", RA.numElements, " elements total");
writeln(RA);
writeln();

//
// The replication is observable when the replicated array is
// on the left-hand side. If the right-hand side is not replicated,
// it is copied into each replicand.
// We illustrate this using a non-distributed array.
//
var A: [Space] int = [(i,j) in Space] i*100 + j;
RA = A;
writeln("Replicated Array after being array-assigned into");
writeln(RA);
writeln();

//
// Analogously, each replicand will be visited and
// other participated expressions will be computed on each locale
// (a) when the replicated array is assigned a scalar:
//       RA = 5;
// (b) when it appears first in a zippered forall loop:
//       forall (ra, a) in zip(RA, A) do ...;
// (c) when it appears in a for loop:
//       for ra in RA do ...;
//
// Zippering (RA,A) or (A,RA) in a 'for' loop will generate
// an error due to their different number of elements.

// Let RA store the Index Map again, for the examples below.
forall ra in RA do
  ra = here.id;

//
// Only the local replicand is accessed - replication is NOT observable
// and consistency is NOT maintained - when:
// (a) the replicated array is indexed - an individual element is read...
//
on Locales(0) do
  writeln("on ", here, ": ", RA(Space.low));
on Locales(LocaleSpace.high) do
  writeln("on ", here, ": ", RA(Space.low));
writeln();

// ...or an individual element is written;
on Locales(LocaleSpace.high) do
  RA(Space.low) = 7777;

writeln("Replicated Array after being indexed into");
writeln(RA);
writeln();

//
// (b) the replicated array is on the right-hand side of an assignment...
//
on Locales(LocaleSpace.high) do
  A = RA + 4;
writeln("Non-Replicated Array after assignment from Replicated Array + 4");
writeln(A);
writeln();

//
// (c) ...or, generally, the replicated array or domain participates
//     in a zippered forall loop, but not in the first position.
//     The loop could look like:
//
//       forall (a, (i,j), ra) in (A, ReplicatedSpace, RA) do ...;
//

//
// The DimensionalDist2D distribution lets us build a 2D distribution
// as a composition of specifiers for individual dimensions.
// Under such a "dimensional" distribution each dimension is handled
// independently of the other.
//
// The dimension specifiers are similar to the corresponding multi-dimensional
// distributions in constructor arguments and index-to-locale mapping rules.
// However, instead of an array of locales, a specifier constructor
// accepts just the number of locales that the indices in the corresponding
// dimension will be distributed across.
//
// The DimensionalDist2D constructor requires:
// * an [0..nl1-1, 0..nl2-1] array of locales, where
//   nl1 and nl2 are the number of locales in each dimension, and
// * two dimension specifiers, created for nl1 and nl2 locale counts, resp.
//
// Presently, the following dimension specifiers are available
// (shown here with their constructor arguments):
//
// * ReplicatedDim(numLocales)
// * BlockDim(numLocales, boundingBoxLow, boundingBoxHigh)
// * BlockCyclicDim(lowIdx, blockSize, numLocales)
//

//
// The following example creates a dimensional distribution that
// replicates over 2 locales (when available) in the first dimemsion
// and distributes using block-cyclic distribution in the second dimension.
// The example computes nl1 and nl2 and reshapes MyLocales correspondingly.
//

var (nl1, nl2) = if numLocales == 1 then (1, 1) else (2, numLocales/2);
MyLocaleView = {0..#nl1, 0..#nl2};
MyLocales = reshape(Locales[0..#nl1*nl2], MyLocaleView);

const DimReplicatedBlockcyclicSpace = Space
  dmapped DimensionalDist2D(MyLocales,
                            new ReplicatedDim(numLocales = nl1),
                            new BlockCyclicDim(numLocales = nl2,
                                               lowIdx = 1, blockSize = 2));

var DRBA: [DimReplicatedBlockcyclicSpace] int;

// The ReplicatedDim specifier always accesses the local replicand.
// (This differs from how the ReplicatedDist distribution works.)
//
// This example visits each replicand. The behavior is the same
// regardless of the second index into MyLocales below.

for locId1 in 0..#nl1 do on MyLocales[locId1, 0] {

  forall drba in DRBA do
    drba = here.id;

  writeln("Dimensional2D(Replicated,BlockCyclic) Array Index Map",
          " from ", here);

  // Technicality: 'writeln(DRBA)' would read DRBA always on Locale 0.
  // Since we want to see what DRBA contains on the current locale,
  // we use 'Helper' that is mapped using the default distribution.
  // 'Helper = DRBA' captures the view of DRBA on the current locale,
  // which we then print out.

  const Helper: [Space] int = DRBA;
  writeln(Helper);
  writeln();

}

//
// Distributions Primer
//
// This primer demonstrates uses of some of Chapel's standard
// distributions.  To use these distributions in a Chapel program,
// the respective module must be used:
//
use BlockDist, CyclicDist, BlockCycDist, ReplicatedDist;
use DimensionalDist2D, ReplicatedDim, BlockCycDim;

//
// For each distribution, we'll create a distributed domain and array
// and then initialize it just to give a brief flavor of how the
// distribution maps across locales.  Running this example on 6
// locales does a nice job of illustrating the distribution
// characteristics.
//
// All of these distributions support options to map to a different
// virtual locale grid than the one used by default (a
// multidimensional factoring of the built-in Locales array), as well
// as to control the amount of parallelism used in data parallel
// loops.  See the Standard Distributions chapter of the language spec
// for more details.
//

//
// Make the program size configurable from the command line.
//
config const n = 8;

//
// Declare a 2-dimensional domain Space that we will later use to
// initialize the distributed domains.
//
const Space = {1..n, 1..n};

//
// The Block distribution distributes a bounding box from
// n-dimensional space across the target locale array viewed as an
// n-dimensional virtual locale grid.  The bounding box is blocked
// into roughly equal portions across the locales.  Note that domains
// declared over a Block distribution can also store indices outside
// of the bounding box; the bounding box is merely used to compute
// the blocking of space.
//
// In this example, we declare a 2-dimensional Block-distributed
// domain BlockSpace and a Block-distributed array BA declared over
// the domain.
//
const BlockSpace = Space dmapped Block(boundingBox=Space);
var BA: [BlockSpace] int;

//
// To illustrate how the index set is distributed across locales,
// we'll use a forall loop to initialize each array element to the
// locale ID that stores that index/element/iteration.
//
forall ba in BA do
  ba = here.id;

//
// Output the Block-distributed array to visually see how the elements
// are partitioned across the locales.
//
writeln("Block Array Index Map");
writeln(BA);
writeln();

//
// Most of Chapel's standard distributions support an optional
// targetLocales argument that permits you to pass in your own
// array of locales to be targeted.  In general, the targetLocales
// argument should match the rank of the distribution.  So for
// example, to map a Block to a [numLocales x 1] view of the
// locale set, one could do something like this:

//
// We start by creating our own array of the locale values.  Here
// we use the standard array reshape function for convenience,
// but more generally, this array could be accessed/assigned like any
// other.
//

var MyLocaleView = {0..#numLocales, 1..1};
var MyLocales: [MyLocaleView] locale = reshape(Locales, MyLocaleView);

//
// Then we'll declare a distributed domain/array that targets
// this view of the locales:
//

const BlockSpace2 = Space dmapped Block(boundingBox=Space,
                                        targetLocales=MyLocales);
var BA2: [BlockSpace2] int;

//
// Then we'll do a similar computation as before to verify where
// everything ended up:
//
forall ba in BA2 do
  ba = here.id;

writeln("Block Array Index Map");
writeln(BA2);
writeln();

//
// We can use the 'targetLocales' method available on an array to get the
// locales array used as targets.
//
for (L, ML) in zip(BA2.targetLocales(), MyLocales) do
  if L != ML then
    halt("Error: BA2.targetLocales() should equal MyLocales");

//
// Next, we'll perform a similar computation for the Cyclic distribution.
// Cyclic distributions start at a designated n-dimensional index and
// distribute the n-dimensional space across an n-dimensional array
// of locales in a round-robin fashion (in each dimension).  As with
// the Block distribution, domains may be declared using the
// distribution who have lower indices that the starting index; that
// value should just be considered a parameterization of how the
// distribution is defined.
//
const CyclicSpace = Space dmapped Cyclic(startIdx=Space.low);
var CA: [CyclicSpace] int;

forall ca in CA do
  ca = here.id;

writeln("Cyclic Array Index Map");
writeln(CA);
writeln();

//
// Next, we'll declare a Block-Cyclic distribution.  These
// distributions also deal out indices in a round-robin fashion,
// but rather than dealing out singleton indices, they deal out blocks
// of indices.  Thus, the BlockCyclic distribution is parameterized
// by a starting index (as with Cyclic) and a block size (per
// dimension) specifying how large the chunks to be dealt out are.
//
const BlkCycSpace = Space dmapped BlockCyclic(startIdx=Space.low,
                                              blocksize=(2, 3));
var BCA: [BlkCycSpace] int;

forall bca in BCA do
  bca = here.id;

writeln("Block-Cyclic Array Index Map");
writeln(BCA);
writeln();

//
// The ReplicatedDist distribution is different: each of the
// original domain's indices - and the corresponding array elements -
// is replicated onto each locale. (Note: consistency among these
// array replicands is NOT maintained automatically.)
//
// This replication is observable in some cases but not others,
// as shown below. Note: this behavior may change in the future.
//
const ReplicatedSpace = Space dmapped ReplicatedDist();
var RA: [ReplicatedSpace] int;

// The replication is observable - this visits each replicand.
forall ra in RA do
  ra = here.id;

writeln("Replicated Array Index Map, ", RA.numElements, " elements total");
writeln(RA);
writeln();

//
// The replication is observable when the replicated array is
// on the left-hand side. If the right-hand side is not replicated,
// it is copied into each replicand.
// We illustrate this using a non-distributed array.
//
var A: [Space] int = [(i,j) in Space] i*100 + j;
RA = A;
writeln("Replicated Array after being array-assigned into");
writeln(RA);
writeln();

//
// Analogously, each replicand will be visited and
// other participated expressions will be computed on each locale
// (a) when the replicated array is assigned a scalar:
//       RA = 5;
// (b) when it appears first in a zippered forall loop:
//       forall (ra, a) in zip(RA, A) do ...;
// (c) when it appears in a for loop:
//       for ra in RA do ...;
//
// Zippering (RA,A) or (A,RA) in a 'for' loop will generate
// an error due to their different number of elements.

// Let RA store the Index Map again, for the examples below.
forall ra in RA do
  ra = here.id;

//
// Only the local replicand is accessed - replication is NOT observable
// and consistency is NOT maintained - when:
// (a) the replicated array is indexed - an individual element is read...
//
on Locales(0) do
  writeln("on ", here, ": ", RA(Space.low));
on Locales(LocaleSpace.high) do
  writeln("on ", here, ": ", RA(Space.low));
writeln();

// ...or an individual element is written;
on Locales(LocaleSpace.high) do
  RA(Space.low) = 7777;

writeln("Replicated Array after being indexed into");
writeln(RA);
writeln();

//
// (b) the replicated array is on the right-hand side of an assignment...
//
on Locales(LocaleSpace.high) do
  A = RA + 4;
writeln("Non-Replicated Array after assignment from Replicated Array + 4");
writeln(A);
writeln();

//
// (c) ...or, generally, the replicated array or domain participates
//     in a zippered forall loop, but not in the first position.
//     The loop could look like:
//
//       forall (a, (i,j), ra) in (A, ReplicatedSpace, RA) do ...;
//

//
// The DimensionalDist2D distribution lets us build a 2D distribution
// as a composition of specifiers for individual dimensions.
// Under such a "dimensional" distribution each dimension is handled
// independently of the other.
//
// The dimension specifiers are similar to the corresponding multi-dimensional
// distributions in constructor arguments and index-to-locale mapping rules.
// However, instead of an array of locales, a specifier constructor
// accepts just the number of locales that the indices in the corresponding
// dimension will be distributed across.
//
// The DimensionalDist2D constructor requires:
// * an [0..nl1-1, 0..nl2-1] array of locales, where
//   nl1 and nl2 are the number of locales in each dimension, and
// * two dimension specifiers, created for nl1 and nl2 locale counts, resp.
//
// Presently, the following dimension specifiers are available
// (shown here with their constructor arguments):
//
// * ReplicatedDim(numLocales)
// * BlockDim(numLocales, boundingBoxLow, boundingBoxHigh)
// * BlockCyclicDim(lowIdx, blockSize, numLocales)
//

//
// The following example creates a dimensional distribution that
// replicates over 2 locales (when available) in the first dimemsion
// and distributes using block-cyclic distribution in the second dimension.
// The example computes nl1 and nl2 and reshapes MyLocales correspondingly.
//

var (nl1, nl2) = if numLocales == 1 then (1, 1) else (2, numLocales/2);
MyLocaleView = {0..#nl1, 0..#nl2};
MyLocales = reshape(Locales[0..#nl1*nl2], MyLocaleView);

const DimReplicatedBlockcyclicSpace = Space
  dmapped DimensionalDist2D(MyLocales,
                            new ReplicatedDim(numLocales = nl1),
                            new BlockCyclicDim(numLocales = nl2,
                                               lowIdx = 1, blockSize = 2));

var DRBA: [DimReplicatedBlockcyclicSpace] int;

// The ReplicatedDim specifier always accesses the local replicand.
// (This differs from how the ReplicatedDist distribution works.)
//
// This example visits each replicand. The behavior is the same
// regardless of the second index into MyLocales below.

for locId1 in 0..#nl1 do on MyLocales[locId1, 0] {

  forall drba in DRBA do
    drba = here.id;

  writeln("Dimensional2D(Replicated,BlockCyclic) Array Index Map",
          " from ", here);

  // Technicality: 'writeln(DRBA)' would read DRBA always on Locale 0.
  // Since we want to see what DRBA contains on the current locale,
  // we use 'Helper' that is mapped using the default distribution.
  // 'Helper = DRBA' captures the view of DRBA on the current locale,
  // which we then print out.

  const Helper: [Space] int = DRBA;
  writeln(Helper);
  writeln();

}


Minor tweaks to modules/standard/AdvancedIters.chpl.

[reviewed by Brad and trivial]

* renamed 'verbose' to 'debugAdvancedIters'

* removed compilerWarning("Working with serial XXX Iterator")

  Since at present the serial iterator is resolved whenever the
  corresponding parallel iterator is resolved, the above message
  simply means "using the XXX iterator". This is not very interesting
  and we usually do not have this kind of diagnostics in our modules.

  If resolving the parallel iterator did not result in resolving
  the serial iterator, such a compilerWarning could be useful
  to warn the user about a potentially-inadvertent performance bug.

* partially indented and removed end-of-line spaces

  I paid my attention only to the code surrounding 'verbose'.
  No consistency was intended.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@21709 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config param debugAdvancedIters:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/chunkSize < nTasks then {
    writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    //yield (c.translate(-c.low),) ;
    yield (totalRange,);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#nTasks do {
     while moreWork do {
      // There is local work in remain
      splitLock$; // Perform the splitting in a critical section
      const current:rType=splitChunk(remain, chunkSize, moreWork);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType= densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=nTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#nTasks do {
     while undone do {
      // There is local work in remain(tid)
      splitLock$; // Perform the splitting in a critical section
      const current:rType=adaptSplit(remain, factor, undone);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType = densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    halt("methodStealing value must be between 0 and 2");

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield (totalRange,);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#nTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if debugAdvancedIters then
        writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
        // There is local work
        const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
        if zeroBasedIters.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
          yield (zeroBasedIters,);
        }
      }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
      if debugAdvancedIters then
        writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

      // Perform the spliting at the victim remaining range

      select methodStealing {
        when 0 do { // Steals from the victim until it exhausts its range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                //after splitting from a victim range
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }

        when 1 do { // Steals once from victim. If it has sucess on the stealing,
                    // it chooses another victim and tries to steal again.

          if moreLocalWork[victim] then {
            // There is work in victim
            const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                 //after splitting from a victim range
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          } else {
            stealFailed=true;
          }
        }
        otherwise { // Tt steals from the victim, but now splitting
                    //from the tail of the victim's range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
                                // after splitting from a victim range:
                               // now we set splitTatil to true (second parameter in the call)
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }
      }
      // If here, then it can have failed the stealing intent at the victim (method 1),
      // or we have exhausted the victim range (methods 0, 2)

      if (methodStealing==0 || methodStealing==2 || (methodStealing==1 && stealFailed)) then {
        nVisitedVictims += 1; // Signal that there is no more work in victim
        if debugAdvancedIters then
          writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        stealFailed=false;
      }
      // Check if there is no more work
      if nVisitedVictims >= nTasks-1 then
        moreWork=false; // Signal that there is no more work in any victim
      else {  // There can be still work in other victim
        victim=(victim+1) % nTasks; // New victim to steal
        if methodStealing==1 then
          if victim==tid then victim=(victim+1) % nTasks;
      }
    }
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/nTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==nTasks-1 then
      rLocal=c#(size*(nTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==nTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.numCores;
      else dnTasks=dataParTasksPerLocale;
  }
  return dnTasks;
}

proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}

config param verbose:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if verbose then
    compilerWarning("Working with serial dynamic Iterator");
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/chunkSize < nTasks then {
    writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    //yield (c.translate(-c.low),) ;
    yield (totalRange,);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#nTasks do {
      while moreWork do {
      // There is local work in remain
      splitLock$; // Perform the splitting in a critical section
      const current:rType=splitChunk(remain, chunkSize, moreWork);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType= densify(current,c);
        if verbose then
          writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

        yield (zeroBasedIters,);}
    }
    }
  }
}

// Follower
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if (verbose) then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int=0) {

  if verbose then
    compilerWarning("Working with serial guided Iterator");
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=nTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#nTasks do {
      while undone do {
      // There is local work in remain(tid)
      splitLock$; // Perform the splitting in a critical section
      const current:rType=adaptSplit(remain, factor, undone);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType = densify(current,c);
        if verbose then
          writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

        yield (zeroBasedIters,);}
    }
    }
  }
}

// Follower
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if verbose then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int=0) {

  if verbose then
    compilerWarning("Working with serial adaptive Iterator");
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    halt("methodStealing value must be between 0 and 2");

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield (totalRange,);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#nTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if verbose then
    writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
      // There is local work
      const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
      if zeroBasedIters.length !=0 then {
        if verbose then
          writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);}
    }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
      if verbose then
        writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

      // Perform the spliting at the victim remaining range

      select methodStealing {
        when 0 do { // Steals from the victim until it exhausts its range

          while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                //after splitting from a victim range
          if zeroBasedIters2.length !=0 then {
            if verbose then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);}
        }
        }

        when 1 do { // Steals once from victim. If it has sucess on the stealing,
                    // it chooses another victim and tries to steal again.

          if moreLocalWork[victim] then {
        // There is work in victim
        const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                           //after splitting from a victim range
        if zeroBasedIters2.length !=0 then {
          if verbose then
            writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
          yield (zeroBasedIters2,);}
          }
          else {
        stealFailed=true;
          }
        }
        otherwise { // Tt steals from the victim, but now splitting
                    //from the tail of the victim's range

          while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
                                // after splitting from a victim range:
                               // now we set splitTatil to true (second parameter in the call)
          if zeroBasedIters2.length !=0 then {
            if verbose then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);}
        }
        }
        }
      // If here, then it can have failed the stealing intent at the victim (method 1),
      // or we have exhausted the victim range (methods 0, 2)

      if (methodStealing==0 || methodStealing==2 || (methodStealing==1 && stealFailed)) then {
        nVisitedVictims += 1; // Signal that there is no more work in victim
        if verbose then
          writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        stealFailed=false;
      }
      // Check if there is no more work
      if nVisitedVictims >= nTasks-1 then
        moreWork=false; // Signal that there is no more work in any victim
      else {  // There can be still work in other victim
        victim=(victim+1) % nTasks; // New victim to steal
        if methodStealing==1 then
          if victim==tid then victim=(victim+1) % nTasks;
      }
    }
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/nTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==nTasks-1 then
      rLocal=c#(size*(nTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==nTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if (verbose) then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.numCores;
      else dnTasks=dataParTasksPerLocale;
  }
  return dnTasks;
}

proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}