Remove repeated line...

Remove repeated line...

/*
Example usage of the FFTW module in Chapel. This particular file demonstrates the single-threaded version of
the code.

To compile this code :
   chpl testFFTW.chpl
If FFTW is not on your include/library paths, you can do
   chpl testFFTW.chpl -I$FFTW_DIR/include -L$FFTW_DIR/lib
where $FFTW_DIR points to your FFTW installation.

Note that you do not need to explicitly include the header or library files; the module uses an experimental
feature of the Chapel compiler to automatically include these.

The FFTW module uses the FFTW3 API, and currently just implements the basic interface.
We will assume that the reader is familiar with using FFTW; more
details are at http://www.fftw.org.

The code computes a series of 1,2 and 3D transforms, exercising the complex <-> complex, and real<->complex
transforms (both in and out of place). The output of the code should be a series of small numbers ( <= 10^-13);
see testFFTW.good for example values (it is possible that your values might differ).

The data for these tests is in arr{1,2,3}d.dat. The format of these files is documented below.
*/
use FFTW;

/* Utility function to print the maximum absolute deviation between values computed by this code,
   and "truth". The true values are computed using Mathematica v10 */
proc printcmp(x, y) {
  var err = max reduce abs(x-y);
  writeln(err);
}

/* This is main test code, parametrized on the number of dimensions ndim.
  fn is the file that contains the test data.
*/
proc runtest(param ndim : int, fn : string) {
  var dims : ndim*int(32);

  /* We define a number of different domains below, corresponding to complex input/output arrays,
     real (input) -> complex (output) out-of-place arrays, and real <-> complex in-place arrays.

     The domains are :
     * D : for complex <-> complex transforms. This domain is also used for the real array in
           a real->complex out-of-place transform.
     * cD : for the complex array in a real <-> complex out-of-place transform.
     * rD : for the real array in a real <-> complex in-place transform. This includes the padding needed
            by the in-place transform. D is a sub-domain of this, and can be used to extract the real array,
            without padding.
     * reD, imD : Utility domains that access the real/complex parts of the complex array in a real<->complex in-place transform.
  */
  var D : domain(ndim);
  var rD,cD,reD,imD : domain(ndim,int,true);

  /* Read in the arrays from the file below. A,B are the arrays that will be used in the FFTW calls,
     goodA and goodB store the true values.

     Note that goodA is all real, allowing us to reuse the same array for the real<->complex tests.

     The file format is as follows (all little-endian):
        [ndim] int32 : dimensions of the array
        [narr] real64 : the real components of A (imaginary components are all zero). narr = N_1*N_2*..*N_ndim (product of dimensions)
        [narr] complex128 : the components of B (stored as real-imaginary pairs).

        The arrays are all stored in C (row-major) order.
  */
  var A,B,goodA,goodB : [D] complex;
  {
    var f = open(fn,iomode.r).reader(kind=iokind.little);
    // Read in dimensions
    for ii in 1..ndim {
      f.read(dims(ii));
    }
    /* The code below sets the domain D. Note that the if statements are folded at compile
       time. This would be more compact with a select statement, but that isn't folded on compilation.
    */
    if (ndim == 1) {
      D = 0.. #dims(ndim);
    }
    if (ndim == 2) {
      D = {0.. #dims(1),0.. #dims(ndim)};
    }
    if (ndim == 3) {
      D = {0.. #dims(1),0.. #dims(2), 0.. #dims(ndim)};
    }

    // Read in the arrays
    for val in goodA {
      f.read(val.re);
      val.im = 0;
    }
    for val in goodB {
      f.read(val);
    }
    f.close();
    writeln("Data read...");
  }

  /* Now set the remaining domains. As above, the if statements are folded on compilation.

     The reader is referred to the FFTW documentation on the storage order for the in-place
     transforms (Sec 2.4 and 4.3.4).
  */
  if (ndim==1) {
    var ldim = dims(1)/2 + 1;
    // Domains for real FFT
    rD = 0.. #(2*ldim); // Padding to do in place transforms
    cD = 0.. #ldim;
    // Define domains to extract the real and imaginary parts for inplace transforms
    reD = rD[0..(2*ldim-1) by 2];
    imD = rD[1..(2*ldim-1) by 2];
  }
  if (ndim==2) {
    // Domains for real FFT
    var ldim = dims(2)/2+1;
    rD = {0.. #dims(1),0.. #(2*ldim)}; // Padding to do in place transforms
    cD = {0.. #dims(1),0.. #ldim};
    // Define domains to extract the real and imaginary parts for in place transforms
    reD = rD[..,0..(2*ldim-1) by 2];
    imD = rD[..,1..(2*ldim-1) by 2];
  }
  if (ndim==3) {
    // Domains for real FFT
    var ldim = dims(3)/2+1;
    rD = {0.. #dims(1),0.. #dims(2),0.. #(2*ldim)}; // Padding to do in place transforms
    cD = {0.. #dims(1),0.. #dims(2),0.. #ldim};
    // Define domains to extract the real and imaginary parts for in place transforms
    reD = rD[..,..,0..(2*ldim-1) by 2];
    imD = rD[..,..,1..(2*ldim-1) by 2];
  }

  // FFTW does not normalize inverse transform, so just compute the normalization constant.
  var norm = * reduce dims;

  /* We start the FFT tests below. The structure is the same :
      * Define plans for forward and reverse transforms.
      * Execute forward transform A -> B.
      * Compare with goodB.
      * Execute reverse transform B -> A and normalize.
      * Compare with goodA.
      * Cleanup plans
  */

  /* Complex <-> complex out-of-place transform.

     Unlike the basic FFTW interface, we do not have specific 1,2,3d planner routines.
     For the complex <-> complex case, the dimensions of the array are inferred automatically.

  */
	var fwd = plan_dft(A, B, FFTW_FORWARD, FFTW_ESTIMATE);
	var rev = plan_dft(B, A, FFTW_BACKWARD, FFTW_ESTIMATE);
	// Test forward and reverse transform
	A = goodA;
	execute(fwd);
	printcmp(B,goodB);
	execute(rev);
	A /= norm;
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Complex <-> complex in-place transform.

     This is the same calling sequence as above, with the input/output arrays the same.
  */
	fwd = plan_dft(A, A, FFTW_FORWARD, FFTW_ESTIMATE);
	rev = plan_dft(A, A, FFTW_BACKWARD, FFTW_ESTIMATE);
	// Test forward and reverse transform
	A = goodA;
	execute(fwd);
	printcmp(A,goodB);
	execute(rev);
	A /= norm; // FFTW does an unnormalized transform
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Real <-> complex out-of-place transform

     As with FFTW, these use an r2c and c2r suffix to define the direction of the transform.

     plan_dft_r2c/plan_dft_c2r are overloaded; for the out-of-place transforms, they infer the
     dimensions from the sizes of the arrays passed in.
  */
  var rA : [D] real(64); // No padding for an out-of place transform
  var cB : [cD] complex;
  fwd = plan_dft_r2c(rA,cB,FFTW_ESTIMATE);
  rev = plan_dft_c2r(cB,rA,FFTW_ESTIMATE);
  rA[D] = goodA.re;
  execute(fwd);
  printcmp(cB,goodB[cD]);
  execute(rev);
  rA /= norm;
  printcmp(rA[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

  /* Real <-> complex in-place transform

     In this case, the first argument to the planning routines is the domain of the *real*
     array WITHOUT padding (in both r2c and c2r cases). This breaks the ambiguity of whether the leading
     dimension of the real array is even or odd.

     This design decision was motivated by the fact that the user has likely already defined a
     domain to extract the unpadded real array from the fully padded array (or that such a domain
     is intrinsically useful).

     For both the r2c and c2r transforms, a real array is passed in.

  */
  var rA2 : [rD] real(64);
  fwd = plan_dft_r2c(D,rA2,FFTW_ESTIMATE);
  rev = plan_dft_c2r(D,rA2,FFTW_ESTIMATE);
  rA2[D] = goodA.re;
  execute(fwd);
  printcmp(rA2[reD],goodB[cD].re); // Check the real and complex parts separately.
  printcmp(rA2[imD],goodB[cD].im);
  execute(rev);
  rA2 /= norm;
  printcmp(rA2[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

}

// A helper function that calls all the tests.
proc testAllDims() {
  writeln("1D");
  runtest(1, "arr1d.dat");
  writeln("2D");
  runtest(2, "arr2d.dat");
  writeln("3D");
  runtest(3, "arr3d.dat");
}

// Do the tests and then cleanup.
proc main() {
  testAllDims();
  cleanup();
}

/*
Example usage of the FFTW module in Chapel. This particular file demonstrates the single-threaded version of
the code.

To compile this code :
   chpl testFFTW.chpl
If FFTW is not on your include/library paths, you can do
   chpl testFFTW.chpl -I$FFTW_DIR/include -L$FFTW_DIR/lib
where $FFTW_DIR points to your FFTW installation.

Note that you do not need to explicitly include the header or library files; the module uses an experimental
feature of the Chapel compiler to automatically include these.

The FFTW module uses the FFTW3 API, and currently just implements the basic interface.
We will assume that the reader is familiar with using FFTW; more
details are at http://www.fftw.org.

The code computes a series of 1,2 and 3D transforms, exercising the complex <-> complex, and real<->complex
transforms (both in and out of place). The output of the code should be a series of small numbers ( <= 10^-13);
see testFFTW.good for example values (it is possible that your values might differ).

The data for these tests is in arr{1,2,3}d.dat. The format of these files is documented below.
*/
use FFTW;

/* Utility function to print the maximum absolute deviation between values computed by this code,
   and "truth". The true values are computed using Mathematica v10 */
proc printcmp(x, y) {
  var err = max reduce abs(x-y);
  writeln(err);
}

/* This is main test code, parametrized on the number of dimensions ndim.
  fn is the file that contains the test data.
*/
proc runtest(param ndim : int, fn : string) {
  var dims : ndim*int(32);

  /* We define a number of different domains below, corresponding to complex input/output arrays,
     real (input) -> complex (output) out-of-place arrays, and real <-> complex in-place arrays.

     The domains are :
     * D : for complex <-> complex transforms. This domain is also used for the real array in
           a real->complex out-of-place transform.
     * cD : for the complex array in a real <-> complex out-of-place transform.
     * rD : for the real array in a real <-> complex in-place transform. This includes the padding needed
            by the in-place transform. D is a sub-domain of this, and can be used to extract the real array,
            without padding.
     * reD, imD : Utility domains that access the real/complex parts of the complex array in a real<->complex in-place transform.
  */
  var D : domain(ndim);
  var rD,cD,reD,imD : domain(ndim,int,true);

  /* Read in the arrays from the file below. A,B are the arrays that will be used in the FFTW calls,
     goodA and goodB store the true values.

     Note that goodA is all real, allowing us to reuse the same array for the real<->complex tests.

     The file format is as follows (all little-endian):
        [ndim] int32 : dimensions of the array
        [narr] real64 : the real components of A (imaginary components are all zero). narr = N_1*N_2*..*N_ndim (product of dimensions)
        [narr] complex128 : the components of B (stored as real-imaginary pairs).

        The arrays are all stored in C (row-major) order.
  */
  var A,B,goodA,goodB : [D] complex;
  {
    var f = open(fn,iomode.r).reader(kind=iokind.little);
    // Read in dimensions
    for ii in 1..ndim {
      f.read(dims(ii));
    }
    /* The code below sets the domain D. Note that the if statements are folded at compile
       time. This would be more compact with a select statement, but that isn't folded on compilation.
    */
    if (ndim == 1) {
      D = 0.. #dims(ndim);
    }
    if (ndim == 2) {
      D = {0.. #dims(1),0.. #dims(ndim)};
    }
    if (ndim == 3) {
      D = {0.. #dims(1),0.. #dims(2), 0.. #dims(ndim)};
    }

    // Read in the arrays
    for val in goodA {
      f.read(val.re);
      val.im = 0;
    }
    for val in goodB {
      f.read(val);
    }
    f.close();
    writeln("Data read...");
  }

  /* Now set the remaining domains. As above, the if statements are folded on compilation.

     The reader is referred to the FFTW documentation on the storage order for the in-place
     transforms (Sec 2.4 and 4.3.4).
  */
  if (ndim==1) {
    var ldim = dims(1)/2 + 1;
    // Domains for real FFT
    rD = 0.. #(2*ldim); // Padding to do in place transforms
    cD = 0.. #ldim;
    // Define domains to extract the real and imaginary parts for inplace transforms
    reD = rD[0..(2*ldim-1) by 2];
    imD = rD[1..(2*ldim-1) by 2];
  }
  if (ndim==2) {
    // Domains for real FFT
    var ldim = dims(2)/2+1;
    rD = {0.. #dims(1),0.. #(2*ldim)}; // Padding to do in place transforms
    cD = {0.. #dims(1),0.. #ldim};
    // Define domains to extract the real and imaginary parts for in place transforms
    reD = rD[..,0..(2*ldim-1) by 2];
    imD = rD[..,1..(2*ldim-1) by 2];
  }
  if (ndim==3) {
    // Domains for real FFT
    var ldim = dims(3)/2+1;
    rD = {0.. #dims(1),0.. #dims(2),0.. #(2*ldim)}; // Padding to do in place transforms
    cD = {0.. #dims(1),0.. #dims(2),0.. #ldim};
    // Define domains to extract the real and imaginary parts for in place transforms
    reD = rD[..,..,0..(2*ldim-1) by 2];
    imD = rD[..,..,1..(2*ldim-1) by 2];
  }

  // FFTW does not normalize inverse transform, so just compute the normalization constant.
  var norm = * reduce dims;

  /* We start the FFT tests below. The structure is the same :
      * Define plans for forward and reverse transforms.
      * Execute forward transform A -> B.
      * Compare with goodB.
      * Execute reverse transform B -> A and normalize.
      * Compare with goodA.
      * Cleanup plans
  */

  /* Complex <-> complex out-of-place transform.

     Unlike the basic FFTW interface, we do not have specific 1,2,3d planner routines.
     For the complex <-> complex case, the dimensions of the array are inferred automatically.

  */
	var fwd = plan_dft(A, B, FFTW_FORWARD, FFTW_ESTIMATE);
	var rev = plan_dft(B, A, FFTW_BACKWARD, FFTW_ESTIMATE);
	// Test forward and reverse transform
	A = goodA;
	execute(fwd);
	printcmp(B,goodB);
	execute(rev);
	A /= norm;
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Complex <-> complex in-place transform.

     This is the same calling sequence as above, with the input/output arrays the same.
  */
	fwd = plan_dft(A, A, FFTW_FORWARD, FFTW_ESTIMATE);
	rev = plan_dft(A, A, FFTW_BACKWARD, FFTW_ESTIMATE);
	A = goodA;
	// Test forward and reverse transform
	A = goodA;
	execute(fwd);
	printcmp(A,goodB);
	execute(rev);
	A /= norm; // FFTW does an unnormalized transform
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Real <-> complex out-of-place transform

     As with FFTW, these use an r2c and c2r suffix to define the direction of the transform.

     plan_dft_r2c/plan_dft_c2r are overloaded; for the out-of-place transforms, they infer the
     dimensions from the sizes of the arrays passed in.
  */
  var rA : [D] real(64); // No padding for an out-of place transform
  var cB : [cD] complex;
  fwd = plan_dft_r2c(rA,cB,FFTW_ESTIMATE);
  rev = plan_dft_c2r(cB,rA,FFTW_ESTIMATE);
  rA[D] = goodA.re;
  execute(fwd);
  printcmp(cB,goodB[cD]);
  execute(rev);
  rA /= norm;
  printcmp(rA[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

  /* Real <-> complex in-place transform

     In this case, the first argument to the planning routines is the domain of the *real*
     array WITHOUT padding (in both r2c and c2r cases). This breaks the ambiguity of whether the leading
     dimension of the real array is even or odd.

     This design decision was motivated by the fact that the user has likely already defined a
     domain to extract the unpadded real array from the fully padded array (or that such a domain
     is intrinsically useful).

     For both the r2c and c2r transforms, a real array is passed in.

  */
  var rA2 : [rD] real(64);
  fwd = plan_dft_r2c(D,rA2,FFTW_ESTIMATE);
  rev = plan_dft_c2r(D,rA2,FFTW_ESTIMATE);
  rA2[D] = goodA.re;
  execute(fwd);
  printcmp(rA2[reD],goodB[cD].re); // Check the real and complex parts separately.
  printcmp(rA2[imD],goodB[cD].im);
  execute(rev);
  rA2 /= norm;
  printcmp(rA2[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

}

// A helper function that calls all the tests.
proc testAllDims() {
  writeln("1D");
  runtest(1, "arr1d.dat");
  writeln("2D");
  runtest(2, "arr2d.dat");
  writeln("3D");
  runtest(3, "arr3d.dat");
}

// Do the tests and then cleanup.
proc main() {
  testAllDims();
  cleanup();
}


Bug fix and minor tweaks to modules/standard/AdvancedIters.chpl.

[committing on behalf of Ben, plus minor tweaks]

* The dynamic iterator used to have all the work done in the (single)
current task when the number of chunks is less than the number of tasks.

This commit makes the current task handle all work when it all fits in
one chunk. Otherwise, the number of tasks created is min(tasks to
satisfy work, number of user specified tasks).

Note that having the current task handle all work has (and has had)
the benefit of avoiding the creation and use of a sync variable,
splitLock$, and other overhead.

** TODO: apply the same to guided and adaptive iterators?

* Do not report "serial execution because there is not enough work"
when there is only a single task. This is for consistency with
other leader iterators that stick to a single task when there is little
work but do not print the warning.

** The above report is enabled, though, when debugAdvancedIters is enabled.

* Converted a halt() to a compilerError().

* Asserted that the 'chunkSize' argument is positive and
halted unless 'numTasks' is positive or 0.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@21813 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config param debugAdvancedIters:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  assert(chunkSize > 0); // caller's responsibility

  use UtilMath;
  // # of tasks the range can fill. (fast) ceil so all work is represented
  const chunkTasks = divceilpos(c.length, chunkSize): int;

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks = min(chunkTasks, defaultNumTasks(numTasks));
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#nTasks do {
     while moreWork do {
      // There is local work in remain
      splitLock$; // Perform the splitting in a critical section
      const current:rType=splitChunk(remain, chunkSize, moreWork);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType= densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=nTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#nTasks do {
     while undone do {
      // There is local work in remain(tid)
      splitLock$; // Perform the splitting in a critical section
      const current:rType=adaptSplit(remain, factor, undone);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType = densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    compilerError("methodStealing value must be between 0 and 2");

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield (totalRange,);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#nTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if debugAdvancedIters then
        writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
        // There is local work
        const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
        if zeroBasedIters.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
          yield (zeroBasedIters,);
        }
      }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
      if debugAdvancedIters then
        writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

      // Perform the spliting at the victim remaining range

      select methodStealing {
        when 0 do { // Steals from the victim until it exhausts its range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                //after splitting from a victim range
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }

        when 1 do { // Steals once from victim. If it has sucess on the stealing,
                    // it chooses another victim and tries to steal again.

          if moreLocalWork[victim] then {
            // There is work in victim
            const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                 //after splitting from a victim range
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          } else {
            stealFailed=true;
          }
        }
        otherwise { // Tt steals from the victim, but now splitting
                    //from the tail of the victim's range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
                                // after splitting from a victim range:
                               // now we set splitTatil to true (second parameter in the call)
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }
      }
      // If here, then it can have failed the stealing intent at the victim (method 1),
      // or we have exhausted the victim range (methods 0, 2)

      if (methodStealing==0 || methodStealing==2 || (methodStealing==1 && stealFailed)) then {
        nVisitedVictims += 1; // Signal that there is no more work in victim
        if debugAdvancedIters then
          writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        stealFailed=false;
      }
      // Check if there is no more work
      if nVisitedVictims >= nTasks-1 then
        moreWork=false; // Signal that there is no more work in any victim
      else {  // There can be still work in other victim
        victim=(victim+1) % nTasks; // New victim to steal
        if methodStealing==1 then
          if victim==tid then victim=(victim+1) % nTasks;
      }
    }
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/nTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==nTasks-1 then
      rLocal=c#(size*(nTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==nTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.numCores;
      else dnTasks=dataParTasksPerLocale;
  } else if nTasks<0 then {
    halt("'numTasks' is negative");
  }
  return dnTasks;
}

proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}

config param debugAdvancedIters:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/chunkSize < nTasks then {
    writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    //yield (c.translate(-c.low),) ;
    yield (totalRange,);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#nTasks do {
     while moreWork do {
      // There is local work in remain
      splitLock$; // Perform the splitting in a critical section
      const current:rType=splitChunk(remain, chunkSize, moreWork);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType= densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=nTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#nTasks do {
     while undone do {
      // There is local work in remain(tid)
      splitLock$; // Perform the splitting in a critical section
      const current:rType=adaptSplit(remain, factor, undone);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType = densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    halt("methodStealing value must be between 0 and 2");

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield (totalRange,);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#nTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if debugAdvancedIters then
        writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
        // There is local work
        const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
        if zeroBasedIters.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
          yield (zeroBasedIters,);
        }
      }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
      if debugAdvancedIters then
        writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

      // Perform the spliting at the victim remaining range

      select methodStealing {
        when 0 do { // Steals from the victim until it exhausts its range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                //after splitting from a victim range
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }

        when 1 do { // Steals once from victim. If it has sucess on the stealing,
                    // it chooses another victim and tries to steal again.

          if moreLocalWork[victim] then {
            // There is work in victim
            const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                 //after splitting from a victim range
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          } else {
            stealFailed=true;
          }
        }
        otherwise { // Tt steals from the victim, but now splitting
                    //from the tail of the victim's range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
                                // after splitting from a victim range:
                               // now we set splitTatil to true (second parameter in the call)
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }
      }
      // If here, then it can have failed the stealing intent at the victim (method 1),
      // or we have exhausted the victim range (methods 0, 2)

      if (methodStealing==0 || methodStealing==2 || (methodStealing==1 && stealFailed)) then {
        nVisitedVictims += 1; // Signal that there is no more work in victim
        if debugAdvancedIters then
          writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        stealFailed=false;
      }
      // Check if there is no more work
      if nVisitedVictims >= nTasks-1 then
        moreWork=false; // Signal that there is no more work in any victim
      else {  // There can be still work in other victim
        victim=(victim+1) % nTasks; // New victim to steal
        if methodStealing==1 then
          if victim==tid then victim=(victim+1) % nTasks;
      }
    }
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/nTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==nTasks-1 then
      rLocal=c#(size*(nTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==nTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.numCores;
      else dnTasks=dataParTasksPerLocale;
  }
  return dnTasks;
}

proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}


Updating comments in leader-follower primer...

...based on the new naming scheme that Vass implemented for me (thanks
Vass!), a series of comments Vass sent me awhile back, and my own
thoughts.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@19280 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// This example shows how to use leader-follower iterators in Chapel.
// leader-follower iterators are used to implement zippered (and
// currently, non-zippered) forall loops in Chapel over data
// structures or standalone iterators.  We expect the leader-follower
// interface to change and improve over time, so this should be
// considered a snapshot of their use in the current implementation.
//
// For a more thorough introduction to leader-follower iterators,
// refer to our PGAS 2011 paper, "User-Defined Parallel Zippered
// Iterators in Chapel".
//

//
// Any zippered forall loop in Chapel will be implemented using
// leader-follower iterators.  Generally speaking, a forall loop
// of the following form:
//
//   forall (a, b, c) in (A, B, C) do
//     ...loop body...
//
// is semantically defined such that the first thing being iterated
// over -- in this case, A -- is designated the 'leader.'  All things
// being iterated over are 'followers' (so for this loop, A, B, and C
// would be).
//
// Given such a loop, the compiler will roughly translate it into
// the following loop structure:
//
//   for work in A.lead() do   // implemented by inlining the leader
//     for (a, b, c) in (A.follow(work), B.follow(work), C.follow(work)) do
//       ...loop body...
//
// where .lead() and .follow() represent the leader-follower iterators
// using a simplified naming scheme.
//
// Note that since Chapel's implicitly parallel semantics are defined
// in terms of zippered iteration, such cases are also implemented using
// leader-follower iterators.  For example:
//
//   A = B + C;
//
// and:
//
//   foo(A, B, C)
//
// (where foo() is defined to take scalar arguments and is promoted
// in this function call) will be converted to their equivalent
// zippered parallel loops and then to the leader-follower idiom shown
// above.

//
// At a high level, the role of a leader iterator is to:
//
//   (a) create the parallel tasks used to implement the forall loop,
//
//   (b) associate the parallel tasks with specific locales as
//       required/desired.
//
//   (c) assign work (e.g., iterations) to each parallel task
//

// It typically creates the parallelism using normal task parallel
// features like coforall loops; and it associates tasks with locales
// using normal locality features like on-clauses.  The leader
// specifies work for tasks by having each task it creates yield some
// representation of the work it owns.
//
// The role of the follower iterator is to take as an input argument
// a chunk of work (as yielded by a leader) and to serially iterate
// over and yield the elements/values corresponding to those
// iterations in order.
//

// Let's go to some code:

//
// For this example, we're going to create a simple iterator named
// 'count' that will be able to be invoked in 'for' or 'forall' loops.
// Count will be defined to take an argument 'n' as input and an
// optional argument 'low' (set to 1 by default), and it will yield n
// integers starting with low.
//

//
// In this example, we'll use the following config const 'numTasks' to
// indicate the degree of parallelism to use in the leader to
// implement forall loops.  By default we've set it to the number of
// cores on the current locale, but it can be overridden on the
// executable command-line using the --numTasks=<n> option.
//
config const numTasks = here.numCores;

//
// numTasks must be at least 1 for this to make sense
//
if (numTasks < 1) then
  halt("numTasks must be a positive integer");

//
// If compiled with verbose set to true, the leader and follower will
// print out some messages indicating what they're doing under the
// covers.
//
config param verbose = false;

//
// Declare a problem size for this test.  By default we use a small
// problem size to make the output readable.  To use the parallelism
// effectively, you'd want to use a much larger problem size of
// course (override on the execution command-line using the
// --probSize=<n> option).
//
config const probSize = 15;

//
// declare an array to work with
//
var A: [1..probSize] real;

//
// when defining a leader-follower iterator pair, our current
// implementation requires that you also define a serial iterator
// of the same name and that it yields the same type as the follower
// iterator does.  In this case, the serial iterator is simple:  We
// simply yield the integers low..low+n-1 (computed using the count
// operator, #):
//
iter count(n: int, low: int=1) {
  for i in low..#n do
    yield i;
}

//
// Here are some simple loops using this iterator to demonstrate
// it.  First we iterate over all indices in our problem size to
// initialize A:
//

for i in count(probSize) do
  A[i] = i:real;

writeln("After serial initialization, A is:");
writeln(A);
writeln();

//
// Then we override the default value of low in order to negate
// the "middle" elements of A:
//
for i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A:");
writeln(A);
writeln();

//
// For serial zippered iteration, nothing is required other than
// this single iterator:
//

for (i, a) in (count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using zippering:");
writeln(A);
writeln();

//
// The leader and follower iterators are defined as overloads of the
// serial version of the iterator, distinguished by an initial
// param argument of the built-in enumerated type 'iterKind'.  To
// invoke the leader iterator and differentiate it from the other
// overloads, the compiler will pass in the value 'iterKind.leader' to
// this argument.  The author of the leader iterator should use a
// 'where' clause to distinguish this overload from the others.  After
// this 'tag' argument, the rest of the argument list should match
// that of the serial iterator exactly.  For our example, this means
// providing the same n and low arguments as before.
//
// The implementation of this leader iterator is relatively simple and
// static.  It uses a coforall loop to create a number of tasks equal
// to the number specified by our 'numTasks' config const and then has
// each yield a subset of the total work to perform.
//
// We compute the work that a task should yield by calling into the
// computeChunk() helper function (defined at the bottom of this file)
// to compute its subrange of the range low..#n owned by the task,
// storing it in a variable called 'myIters.'
//
// To be a legal leader iterator, we could simply yield this range as
// a representation of the work we want the follower to perform.
// However, to support zippering our leader with follower iterators
// written by others, we typically take the convention of having
// iterators over 1D or dense rectangular index spaces yield tuples
// of ranges shifted to a 0-based coordinate system.  In this way, the
// leader-follower iterators have a common representation for the
// work even though each may use its own indexing system.  This
// permits, for example, arrays of the same size/shape to be zippered
// together even if they have different domains.
//
// To this end, rather than yielding subranges of low..#n, we'll yield
// subranges of 0..n-1 and rely on the follower to shift it back to
// the original coordinate system.  For this reason, we translate the
// range by -low to shift it from low-based coordinates to 0-based
// coordinates; and then we wrap it in the tuple() helper function
// to make a 1-tuple out of it.
//
// Note the debugging output inserted into this iterator.  While
// learning about leader-follower iterators, it's useful to turn
// this debugging output on by compiling  with -sverbose=true
//
iter count(param tag: iterKind, n: int, low: int=1)
       where tag == iterKind.leader {

  if (verbose) then
    writeln("In count() leader, creating ", numTasks, " tasks");

  coforall tid in 0..#numTasks {
    const myIters = computeChunk(low..#n, tid, numTasks);
    const zeroBasedIters = myIters.translate(-low);

    if (verbose) then
      writeln("task ", tid, " owns ", myIters, " yielded as: ", zeroBasedIters);

    yield tuple(zeroBasedIters);
  }
}
//
// As mentioned at the outset, this leader is fairly static and
// simple.  More generally, a leader can introduce tasks more
// dynamically, partition work between the tasks more dynamically,
// etc.  See $CHPL_HOME/modules/standard/AdvancedIters.chpl for
// some more interesting examples of leader iterators, including
// those that use dynamic partitioning.
//

//
// The follower is another overload of the same iterator name, this
// time taking the iterKind.follower param enumeration as its first
// argument.  The next arguments should match the leader and serial
// iterators exactly again (so, n and low for our example).  The
// final argument must be called 'followThis' which represents the data
// yielded by the leader (in our case, the 1-tuple of ranges).
//
// The goal of the follower is to do the iteration specified by the
// 'followThis' argument, serially yielding the elements corresponding
// to those iterations.  In our case, this involves plucking the
// range back out of the tuple of ranges, and shifting it back to
// our low-based coordinate system.  We then use a standard for loop
// to iterate over that range and yield the corresponding indices.
// Followers, as the name suggests, tend not to be very sophisticated,
// and simply do what the leader tells them to.
//
// As with the leader, this follower has been authored to support
// debugging output when compiled with -sverbose=true.
//
iter count(param tag: iterKind, n: int, low: int=1, followThis)
       where tag == iterKind.follower && followThis.size == 1 {
  const lowBasedIters = followThis(1).translate(low);

  if (verbose) then
    writeln("Follower received ", followThis, " as work chunk; shifting to ",
            lowBasedIters);

  for i in lowBasedIters do
    yield i;
}

//
// Now that we've defined leader-follower iterators, we can execute
// the same loops we did before, only this time using forall loops
// to make the execution parallel.  We start with some simple
// invocations as before.  In these invocations, the count() iterator
// serves as the only leader and follower iterator being invoked
// since it is the only thing being iterated over (A is being randomly
// accessed within the loop.
//
forall i in count(probSize) do
  A[i] = i:real;

writeln("After parallel initialization, A is:");
writeln(A);
writeln();

//
// Invoking it again with a different low value:
//
forall i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A in parallel:");
writeln(A);
writeln();

//
// Zippered iteration is now a bit more interesting.  In this
// first loop, count() serves as the leader and follower while
// the A array is a follower.  This works because A is a
// rectangular array whose follower iterator accepts tuples of
// 0-based ranges like the ones count()'s leader is yielding.
// If we were to have count() yield something else (like a raw
// subrange of low..#n), then the two things could not be
// zippered correctly because they wouldn't be speaking the
// same language -- either in terms of the type of work being
// yielded (range vs. 1-tuple of range), nor the description of
// the work (low-based indices vs. 0-based indices).
//
forall (i, a) in (count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using parallel zippering:");
writeln(A);
writeln();

//
// We can also zipper in the opposite order, making A the leader,
// in which case count() no longer controls the degree of parallelism
// and work assignment since it is no longer the leader.  Instead,
// A's leader iterator (defined as part of its domain map) is invoked.
// For standard Chapel arrays and domain maps, these leader-follower
// iterators are controlled by the dataPar* configuration constants
// as described in doc/README.executing.
//
forall (a, i) in (A, count(probSize)) do
  a = i/100.0;

writeln("After reassigning A using parallel zippering and A as the leader:");
writeln(A);
writeln();

//
// Finally, as mentioned at the outset, operations that are equivalent
// to zippering also use leader-follower iterators, so for example
// the following whole-array assignment will use A's leader and
// count()'s follower:
//
A = count(probSize, low=100);

writeln("After reassigning A using whole-array assignment:");
writeln(A);
writeln();

//
// Some closing notes:
//
// * Chapel data types like records and classes can support iteration
//   by defining iterator methods (invoked by name) or these() iterators
//   which support iterating over variables of that type directly.  Such
//   iterator methods can be overloaded to support leader-follower
//   versions as well to permit parallel iteration over the variable.
//
// * As mentioned at the outset, our leader-follower scheme has a number
//   of improvements planned for it, including the ability to specify
//   a standalone parallel iterator for non-zippered forall loops to
//   avoid the overhead of using a leader-follower pair and better error
//   checking.  We'll update this primer as we improve these features.
//

//
// This is a poor-man's partitioning algorithm.  It gives
// floor(numElements/NumChunks) work items to the first numChunks-1
// chunks and the remainder to the last chunk.  For simplicity it only
// works for non-strided, default index type ranges.  More work would
// be required to generalize it for strided or unbounded ranges.
//
proc computeChunk(r: range, myChunk, numChunks) where r.stridable == false {
  const numElems = r.length;
  const elemsPerChunk = numElems/numChunks;
  const mylow = r.low + elemsPerChunk*myChunk;
  if (myChunk != numChunks - 1) {
    return mylow..#elemsPerChunk;
  } else {
    return mylow..r.high;
  }
}

//
// This example shows how to use leader/follower iterators in Chapel.
// Leader/follower iterators are used to implement zippered (and
// currently, non-zippered) forall loops in Chapel over data
// structures or standalone iterators.  We expect the leader/follower
// interface to change and improve over time, so this should be
// considered a snapshot of their use in the current implementation.
//

//
// Any zippered forall loop in Chapel will be implemented using
// leader/follower iterators.  Generally speaking, a forall loop
// of the following form:
//
//   forall (a, b, c) in (A, B, C) do
//     ...loop body...
//
// is semantically defined such that the first thing being iterated
// over -- in this case, A -- is the leader.  All things being
// iterated over are followers (so for this loop, A, B, and C would
// be).
//
// Given such a loop, the compiler will roughly translate it into
// the following loop structure:
//
//   for work in A.lead() do
//     for (a, b, c) in (A.follow(work), B.follow(work), C.follow(work)) do
//       ...loop body...
//
// where .lead() and .follow() are conceptually intended to represent
// the leader/follower iterators (using a simplified naming scheme).
//
// Note that since Chapel's implicitly parallel semantics are defined
// in terms of zippered iteration, such cases are also implemented using
// leader/follower iterators.  For example:
//
//   A = B + C;
//
// and:
//
//   foo(A, B, C)
//
// (where foo() is defined to take scalar arguments and is promoted
// in this function call) will be converted to the equivalent zippered
// parallel loops and then to the leader/follower idiom shown above.

//
// At a high level, the role of a leader iterator is to:
//
//   (a) create the parallel tasks used to implement the forall loop,
//
//   (b) associate the parallel tasks with specific locales as
//       required/desired.
//
//   (c) assign work (e.g., iterations) to each parallel task
//
// The leader specifies work for tasks by having each task it creates
// yield some representation of what the work is.
//
// The role of the follower iterator is to take as an input argument
// a chunk of work (as yielded by its leader) and to serially iterate
// over and yield the corresponding elements/values corresponding to
// those iterations.
//

// Let's go to some code:

//
// For this example, we're going to create a simple iterator named
// 'count' that will be able to be invoked in 'for' or 'forall' loops.
// Count will be defined to take an argument 'n' as input and an
// optional argument 'low' (set to 1 by default) and will yield n
// integers starting with low.
//

//
// In this example, we'll use this config const 'numTasks' to indicate
// the degree of parallelism to use in the leader to implement forall
// loops.  By default we've set it to the number of cores on the
// current locale, but it can be overridden on the executable
// command-line using the --numTasks=<n> option.
//
config const numTasks = here.numCores;

//
// numTasks must be at least 1 for this to make sense
//
if (numTasks < 1) then
  halt("numTasks must be a positive integer");

//
// If compiled with verbose set to true, the leader and follower will
// print out some messages indicating what they're doing under the
// covers.
//
config param verbose = false;

//
// Declare a problem size for this test.  By default we use a small
// problem size to make the output readable.  To use the parallelism
// effectively, you'd want to use a much larger problem size of
// course (override on the execution command-line using the
// --probSize=<n> option.
//
config const probSize = 15;

//
// declare an array to work with
//
var A: [1..probSize] real;

//
// when defining a leader/follower iterator, our current
// implementation requires that you also define a serial iterator
// of the same name and that it yields the same type as the follower
// iterator does.  In this case, the serial iterator is simple:  We
// simply yield the integers 1..n:
//
iter count(n: int, low: int=1) {
  for i in low..#n do
    yield i;
}

//
// Here are some simple iterations over this iterator to demonstrate
// it.  First we iterate over all indices in our problem size to
// initialize A:
//

for i in count(probSize) do
  A[i] = i:real;

writeln("After serial initialization, A is:");
writeln(A);
writeln();

//
// Then we override the default value of low in order to negate
// the "middle" elements of A:
//
for i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A:");
writeln(A);
writeln();

//
// For serial zippered iteration, nothing is required other than
// this single iterator:
//

for (i, a) in (count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using zippering:");
writeln(A);
writeln();

//
// The leader and follower iterators are defined as overloads of the
// serial version of the iterator, distinguished by an initial
// param argument of the (poorly named) built-in enumerated type
// 'iterator'.  To invoke the leader iterator and differentiate it
// from the other overloads, the compiler will pass in the value
// 'iterator.leader' to this argument.  The author of the leader
// iterator should use a where clause to distinguish this overload
// from the others.  After this 'tag' argument, the rest of the
// argument list should match that of the serial iterator exactly.
// For our example, this means providing the same n and low arguments
// as before.
//
// The implementation of this leader iterator is relatively simple and
// static.  It uses a coforall loop to create a number of tasks equal
// to the number specified by our 'numTasks' config const and then has
// each yield a subset of the total work to perform.
//
// We compute the work that a task should yield by calling into the
// computeChunk() helper function (defined at the bottom of this file)
// to compute its subrange of the range low..#n owned by the task,
// storing it in a variable called 'myIters.'
//
// To be a legal leader iterator, we could simply yield this range as
// a representation of the work we want the follower to perform.
// However, to support zippering with follower iterators other than
// our own, dense rectangular iterators yield tuples of ranges shifted
// to 0-based coordinates by convention.  In this way, our iterator
// will be able to be zippered with iterators for dense rectangular
// domains and arrays, as well as other iterators that follow this
// convention.
//
// To this end, rather than yielding subranges of low..#n, we'll yield
// subranges of 0..n-1 and rely on the follower to shift it back to
// the original coordinate system.  For this reason, we translate the
// range by -low to shift it from low-based coordinates to 0-based
// coordinates; and then we wrap it in the tuple() helper function
// to make a 1-tuple out of it.
//
// Note the debugging output inserted into this iterator.  While
// learning about leader/follower iterators, it's useful to turn
// this debugging output on by compiling  with -sverbose=true
//
iter count(param tag: iterKind, n: int, low: int=1)
       where tag == iterKind.leader {

  if (verbose) then
    writeln("In count() leader, creating ", numTasks, " tasks");

  coforall tid in 0..#numTasks {
    const myIters = computeChunk(low..#n, tid, numTasks);
    const zeroBasedIters = myIters.translate(-low);

    if (verbose) then
      writeln("task ", tid, " owns ", myIters, " yielded as: ", zeroBasedIters);

    yield tuple(zeroBasedIters);
  }
}
//
// As mentioned at the outset, this leader is fairly static and
// simple.  More generally, a leader can introduce tasks more
// dynamically, partition work between the tasks more dynamically,
// etc.
//

//
// The follower is another overload of the same iterator name, this
// time taking the iterator.follower param enumeration as its first
// argument.  The next arguments should match the leader and serial
// iterators exactly again (so, n and low for our example).  The
// final argument is called 'followThis' (another name that could
// stand to be improved) which represents the data yielded by the
// leader (in our case, the 1-tuple of ranges).
//
// The goal of the follower is to do the iteration specified by the
// 'followThis' argument, serially yielding the elements corresponding
// to those iterations.  In our case, this involves plucking the
// range back out of the tuple of ranges, and shifting it back to
// our low-based coordinate system.  We then use a standard for loop
// to iterate over that range and yield the corresponding indices.
// Followers, as the name suggests, tend not to be very sophisticated,
// and simply do what the leader tells them to.
//
// As with the leader, this follower has been authored to support
// debugging output when compiled with -sverbose=true.
//
iter count(param tag: iterKind, n: int, low: int=1, followThis)
       where tag == iterKind.follower && followThis.size == 1 {
  const lowBasedIters = followThis(1).translate(low);

  if (verbose) then
    writeln("Follower received ", followThis, " as work chunk; shifting to ",
            lowBasedIters);

  for i in lowBasedIters do
    yield i;
}

//
// Now that we've defined leader/follower iterators, we can execute
// the same loops we did before, only this time using forall loops
// to make the execution parallel.  We start with some simple
// invocations as before.  In these invocations, the count() iterator
// serves as the only leader and follower iterator being invoked
// since it is the only thing being iterated over (A is being randomly
// accessed within the loop.
//
forall i in count(probSize) do
  A[i] = i:real;

writeln("After parallel initialization, A is:");
writeln(A);
writeln();

//
// Invoking it again with a different low value:
//
forall i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A in parallel:");
writeln(A);
writeln();

//
// Zippered iteration is now a bit more interesting.  In this
// first loop, count() serves as the leader and follower while
// the A array is a follower.  This works because A is a
// rectangular array whose follower iterator accepts tuples of
// 0-based ranges like the ones count()'s leader is yielding.
// If we were to have count() yield something else (like a raw
// subrange of low..#n), then the two things could not be
// zippered correctly because they wouldn't be speaking the
// same language -- either in terms of the type of work being
// yielded (range vs. 1-tuple of range), nor the description of
// the work (low-based indices vs. 0-based indices).
//
forall (i, a) in (count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using parallel zippering:");
writeln(A);
writeln();

//
// We can also zipper in the opposite order, making A the leader,
// in which case count() no longer controls the degree of parallelism
// and work assignment since it is no longer the leader.  Instead,
// A's leader iterator (defined as part of its domain map) is invoked.
// For standard Chapel arrays and domain maps, these leader/follower
// iterators are controlled by the dataPar* configuration constants
// as described in doc/README.executing.
//
forall (a, i) in (A, count(probSize)) do
  a = i/100.0;

writeln("After reassigning A using parallel zippering and A as the leader:");
writeln(A);
writeln();

//
// Finally, as mentioned at the outset, operations that are equivalent
// to zippering also use leader/follower iterators, so for example
// the following whole-array assignment will use A's leader and
// count()'s follower:
//
A = count(probSize, low=100);

writeln("After reassigning A using whole-array assignment:");
writeln(A);
writeln();

//
// Some closing notes:
//
// * Chapel data types like records and classes can support iteration
//   by defining iterator methods (invoked by name) or these() iterators
//   which support iterating over variables of that type directly.  Such
//   iterator methods can be overloaded to support leader/follower
//   versions as well to permit parallel iteration over the variable.
//
// * As mentioned at the outset, our leader/follower scheme has a number
//   of improvements planned for it, including the ability to specify
//   a standalone parallel iterator for non-zippered forall loops to
//   avoid the overhead of using a leader/follower pair and better error
//   checking.  We'll update this primer as we improve these features.
//

//
// This is a poor-man's partitioning algorithm.  It gives
// floor(numElements/NumChunks) work items to the first numChunks-1
// chunks and the remainder to the last chunk.  For simplicity it only
// works for non-strided ranges.
//
proc computeChunk(r: range, myChunk, numChunks) where r.stridable == false {
  const numElems = r.length;
  const elemsPerChunk = numElems/numChunks;
  const mylow = r.low + elemsPerChunk*myChunk;
  if (myChunk != numChunks - 1) {
    return mylow..#elemsPerChunk;
  } else {
    return mylow..r.high;
  }
}


merging...

merging...

/** Git Intro!
 *
 * The Chapel team's intro to git version control. The core team members should
 * update their name tuple in the array below.
 */

// To play Chapel adlibs, fill out your tuple with a programming language, a
// feeling, and a verb.
//
// name
//         programming language
//                 feeling
//                         verb
const TheTeam = [
  ("Ben", "", "", ""),
  ("Brad", "", "", ""),
  ("David", "", "", ""),
  ("Greg", "", "", ""),
  ("Kyle", "C", "well", "like"),
  ("Lydia", "", "", ""),
  ("Michael", "", "", ""),
  ("Elliot", "", "", ""),
  ("Simon", "", "", ""),
  ("Sung", "", "", ""),
  ("Thomas", "Python", "ecstatic", "<3"),
  ("Tim", "", "", ""),
  ("TMac", "", "", ""),
  ("Tom", "", "", ""),
  ("Vass", "", "", "")
];

// Prologue
writeln("Chapel Adlibs!");
writeln("--------------");
writeln();

// Print the adlibs!
for (name, lang, feeling, verb) in TheTeam {
  writef("My name is %s. I used to use %s for all my problems.\nNow, I use Chapel and feel %s! I %s Chapel.\n",
         name, lang, feeling, verb);
  writeln();
}

/** Git Intro!
 *
 * The Chapel team's intro to git version control. The core team members should
 * update their name tuple in the array below.
 */

// To play Chapel adlibs, fill out your tuple with a programming language, a
// feeling, and a verb.
//
// name
//         programming language
//                 feeling
//                         verb
const TheTeam = [
  ("Ben", "", "", ""),
  ("Brad", "", "", ""),
  ("David", "", "", ""),
  ("Greg", "", "", ""),
  ("Kyle", "", "", ""),
  ("Lydia", "", "", ""),
  ("Michael", "", "", ""),
  ("Elliot", "", "", ""),
  ("Simon", "", "", ""),
  ("Sung", "", "", ""),
  ("Thomas", "Python", "ecstatic", "<3"),
  ("Tim", "", "", ""),
  ("TMac", "", "", ""),
  ("Tom", "", "", ""),
  ("Vass", "", "", "")
];

// Prologue
writeln("Chapel Adlibs!");
writeln("--------------");
writeln();

// Print the adlibs!
for (name, lang, feeling, verb) in TheTeam {
  writef("My name is %s. I used to use %s for all my problems.\nNow, I use Chapel and feel %s! I %s Chapel.\n",
         name, lang, feeling, verb);
  writeln();
}