More Filerator edits

More Filerator edits

Most of these are comment cleanups/clarifications.  Thomas also
caught that I wasn't passing one of the arguments through on
my recursive call, which my tests weren't complex enough to catch.

/*
 * Copyright 2004-2014 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial.
//

/* iter listdir(path: string, dotfiles=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * dotfiles: should files/directories that start with '.' be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, dotfiles=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;
  extern proc chpl_rt_isDir(pathname: c_string, followLinks: bool): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (dotfiles || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          const fullpath = path + "/" + filename;
          //
          // chpl_rt_isDir() returns 'true' if 'fullpath' is a
          // directory.  If we are listing symbolic links
          // (listlinks==true), it will also return symbolic links to
          // directories.
          //
          if (chpl_rt_isDir(fullpath:c_string, listlinks)) {
            if (dirs) then
              yield filename;
          } else {
            if (files) then
              yield filename;
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 dotfiles=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.
     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * dotfiles: indicates whether to enter directories with dotfile names
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), dotfiles=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, dotfiles=dotfiles, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, dotfiles,
                             followlinks, sort) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter wordexp(pattern="*")

   wordexp() gives a glob-like capability, implemented with C's wordexp()
   (which is, itself a routine that provides a glob-like capability :)
     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

iter wordexp(pattern="*") {
  extern type wordexp_t;
  extern proc wordexp(pattern:c_string, ref ret_glob: wordexp_t,
                      flags_c_int): c_int;
  extern proc chpl_wordexp_num(x:wordexp_t): size_t;
  extern proc chpl_wordexp_index(x:wordexp_t, idx:size_t): c_string;
  extern proc wordfree(ref glb:wordexp_t);

  var glb : wordexp_t;

  const err = wordexp(pattern:c_string, glb, 0);

  for i in 0..chpl_wordexp_num(glb)-1 {
    yield chpl_wordexp_index(glb, i): string;
  }

  wordfree(glb);
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()
     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

iter glob(pattern="*") {
  extern type glob_t;
  extern proc glob(pattern:c_string, flags: c_int, errfunc:c_void_ptr,
                   ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);

  var glb : glob_t;

  const err = glob(pattern:c_string, 0, c_nil, glb);

  for i in 0..chpl_glob_num(glb)-1 do
    yield chpl_glob_index(glb, i): string;

  globfree(glb);
}

/* iter findfiles(startdir = ".", recur=false, dotfiles=false)

   findfiles() is a simple find-like utility implemented using the
   above routines
     * startdir: where to start when looking for files
     * recur: tells whether or not to descend recurisvely
     * dotfiles: tells whether or not to yield dotfiles
*/

iter findfiles(startdir = ".", recur=false, dotfiles=false) {
  if (recur) then
    for subdir in walkdirs(startdir, dotfiles=dotfiles) do
      for file in listdir(subdir, dotfiles=dotfiles, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, dotfiles=dotfiles, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

/*
 * Copyright 2004-2014 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial.
//

/* iter listdir(path: string, dotfiles=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * dotfiles: should files/directories that start with '.' be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, dotfiles=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;
  extern proc chpl_rt_isDir(pathname: c_string, followLinks: bool): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (dotfiles || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          const fullpath = path + "/" + filename;
          //
          // chpl_rt_isDir() returns 'true' if 'fullpath' is a
          // directory.  If we are listing symbolic links
          // (listlinks==true), it will also return symbolic links to
          // directories.
          //
          if (chpl_rt_isDir(fullpath:c_string, listlinks)) {
            if (dirs) then
              yield filename;
          } else {
            if (files) then
              yield filename;
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 dotfiles=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.
     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * dotfiles: indicates whether to enter directories with dotfile names
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), dotfiles=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, dotfiles=dotfiles, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, dotfiles, followlinks) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter wordexp(pattern="*")

   wordexp() gives a glob-like capability, implemented with C's wordexp()
     * pattern: the pattern to match against

   By default, it will list all files/directories in the directory
*/

iter wordexp(pattern="*") {
  extern type wordexp_t;
  extern proc wordexp(pattern:c_string, ref ret_glob: wordexp_t,
                      flags_c_int): c_int;
  extern proc chpl_wordexp_num(x:wordexp_t): size_t;
  extern proc chpl_wordexp_index(x:wordexp_t, idx:size_t): c_string;
  extern proc wordfree(ref glb:wordexp_t);

  var glb : wordexp_t;

  const err = wordexp(pattern:c_string, glb, 0);

  for i in 0..chpl_wordexp_num(glb)-1 {
    yield chpl_wordexp_index(glb, i): string;
  }

  wordfree(glb);
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()
     * pattern: the pattern to match against

   By default, it will list all files/directories in the directory
*/

iter glob(pattern="*") {
  extern type glob_t;
  extern proc glob(pattern:c_string, flags: c_int, errfunc:c_void_ptr,
                   ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);

  var glb : glob_t;

  const err = glob(pattern:c_string, 0, c_nil, glb);

  for i in 0..chpl_glob_num(glb) - 1 do
    yield chpl_glob_index(glb, i): string;

  globfree(glb);
}

/* iter findfiles(startdir = ".", recur=false, dotfiles=false)

   findfiles() is a simple find-like utility implemented using the
   above routines
     * startdir: where to start when looking for files
     * recur: tells whether or not to descend recurisvely
     * dotfiles: tells whether or not to yield dotfiles
*/

iter findfiles(startdir = ".", recur=false, dotfiles=false) {
  if (recur) then
    for subdir in walkdirs(startdir, dotfiles=dotfiles) do
      for file in listdir(subdir, dotfiles=dotfiles, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, dotfiles=dotfiles, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}


Converted dos2unix

Converted dos2unix

/*
   NPB Integer Sort
   Chapel parallel version: multi-task, multi-locale
     based on bucket sort (same as NPB reference version)

   Originally written by Margaret Cahir (Cray, Inc.)

   NOTES on the Block version:
     Limitations:
       - works with 32-bit integers only
       - tested on 1, 2 and 4 locales, Classes S, A, B and C (not D)

   NOTES on the BlockCyclic version:
     Still need to
     - adjust to solve for Class D (means adjusting for int64)
     - can gen keys be made a little cleaner? (see note)
     - can a backwards index be made to work? see note in fullVerify
  	var mype: int = i / nkeys_per_pe;  //  can this be i >> log2nkeys_per_pe?
     - loop2: can we use scan in line below

   Modifications [sungeun@cray.com]: 12/2011

   - Combined Block and BlockCyclic versions

*/

// Problem size parameters
enum  classVals {S=0, W=1, A=2, B=3, C=4};
const Classes: domain(classVals);
const ClassNkeys:  [Classes] int = ( 16, 20, 23, 25, 27 ),
      ClassRanges: [Classes] int = ( 11, 16, 19, 21, 23 );

// Set of config constants that control size of problem and output options
config const probClass = classVals.S;	 // default to Sample class
config const maxIterations = 10;	 // can be dialed down for devel runs
config const DEBUG:        bool = false; // shows progress and summary info
config param printLoopTimings: bool = false; // prints timing info
config param printArrays:  bool = false; // prints out arrays (can be long)
config const printTime    = false;	 // turn off for test verification

// Standard modules
use Time, Random, BlockDist, BlockCycDist;
enum ISDistType { block, blockcyclic };
config param distType = ISDistType.block;

// Comparison numbers for partial verification (these are given)
var testD: domain(2) =  {0..4, 0..4};	// # of Classes, # of keys to verify
var testIndexArray: [testD] int = (	( 48427,17148,23627,62548,4431),
					( 357773,934767,875723,898999,404505),
					( 2112377,662041,5336171,3642833,4250760),
					( 41869,812306,5102857,18232239,26860214),
					( 44172927,72999161,74326391,129606274,21736814) );
const testRankArray: [testD]  int = (   ( 0,18,346,64917,65463),
					( 1249,11698,1039987,1043896,1048018),
					( 104,17523,123928,8288932,8388264),
					( 33422937,10244,59149,33135281,99),
					( 61147,882988,266290,133997595,133525895) );

var passedVerifications = 0;		// verification counter

// Set problem sizes and storage requirements
const log2range = ClassRanges(probClass);
const log2nkeys = ClassNkeys (probClass);
var range:    int = 1 << log2range;	// left shift for power of 2 values
var nkeys:    int = 1 << log2nkeys;
var npes:     int = numLocales*(if dataParTasksPerLocale==0
                                then here.maxTaskPar
                                  else dataParTasksPerLocale);  // npes is the total number of tasks
var nbuckets: int = if distType==ISDistType.block then npes else 4*npes;

var keybuff_pe: real = 3*nkeys/npes;
var keybuffsz:  int  = keybuff_pe:int;

// Domains for main arrays
const keySpace:    domain(1)= {0..#nkeys};

// mcahir: Alas, it seems a block cyclic dist array has to be 2D so ended up making countSpace a 2D domain
const countSpace = if distType==ISDistType.block then {0..#range}
                                                 else {0..#range, 0..0};
const bucketSpace: domain(2)= {0..#npes,0..#nbuckets};
const buffSpace:   domain(2)= {0..#nbuckets,0..#keybuffsz};

// ... block a 2D space into horizontal slabs
const MyLocaleView = {0..#numLocales, 1..1};
const MyLocales: [MyLocaleView] locale = reshape(Locales, MyLocaleView);
const bucketDom = bucketSpace dmapped Block(boundingBox=bucketSpace, targetLocales=MyLocales);
const buffDom = if distType==ISDistType.block
  then buffSpace dmapped Block(boundingBox=buffSpace, targetLocales=MyLocales)
  else buffSpace dmapped BlockCyclic (startIdx=buffSpace.low,
                                      blocksize=(dataParTasksPerLocale,
                                                 keybuffsz),
                                      targetLocales=MyLocales);

// Map domains to locales
const keyDom   = keySpace dmapped Block(boundingBox=keySpace);
const countDom = if distType==ISDistType.block
  then countSpace dmapped Block(boundingBox=countSpace)
  else countSpace dmapped BlockCyclic(startIdx=countSpace.low,
                                      blocksize=(dataParTasksPerLocale,1),
                                      targetLocales=MyLocales);

// Now declare the main distributed arrays
var key:      [keyDom] int;	// random numbers between 0 and range -1
var rank:     [keyDom] int;	// store ranks here
var sortkey:  [keyDom] int;	// used for verification
var keybuff:  [buffDom]int;	// used for remote storage
var keycount: [countDom] int = 0;	// store #occurrences of values
var keycountbuf: [countDom] int = 0;	// buffer for pre-fix scan

// these counter arrays are also distributed
var bucket_cnts: [bucketDom] int;	// there is a row here for each pe (rank, locale)
var bucket_ptrs: [bucketDom] int;  // similarly, a row for each pe, a column for each bucket

// Timers
var tsetup, tsort, tverif: Timer;
var tloops: [1..5] Timer;

// Compute log of the number of pes and num keys per pe
var log2npes: int = 0;
while (1<<log2npes < npes) do {log2npes += 1; }
var nkeys_per_pe: int = nkeys >> log2npes;
var log2nbuckets: int = 0;
while (1<<log2nbuckets < nbuckets) do {log2nbuckets += 1; }

proc main () {

  tsetup.start();

  writef ("NAS Parallel Benchmarks 2.4 -- IS Benchmark\n" );
  writef (" Size:                       #########  (class %t)\n", nkeys, probClass);
  writef (" Iterations:                 #########\n",maxIterations);
  writef (" Number of locales:          #########\n",numLocales);
  writef (" Number of tasks per locale: #########\n",dataParTasksPerLocale);
  writef (" \n");

  // initialize key values
  gen_keys();

  tsetup.stop();
  if (printTime)
    {writeln ("Elapsed time for setup    =",tsetup.elapsed()," seconds");}

  if (DEBUG) then writeln("npes =",npes," log2npes=",log2npes);
  // ," log2nbuckets=", if ISDistType==block then "N/A" else log2nbuckets);

  tsort.start();				// start the timed section
						// main iteration loop
  for iteration in 1..maxIterations {

    if (probClass != classVals.S) then writeln (iteration," Iteration ");

    key[iteration] = iteration;
    key[iteration+maxIterations] = range - iteration;

    rank_keys ( iteration );

    partialVerification(iteration, keycountbuf);

  }						// end main iteration loop

  tsort.stop();					// end the timed section
  if (printTime) {
    writeln ("Elapsed time for key sort =", tsort.elapsed()," seconds");}
  if (printTime) {
    writeln ("Time for individual loops");
    for i in 1..5 do writeln("  Loop ",i,"  ",tloops(i).elapsed()," seconds"); }

  tverif.start();  				// full verification (full sort)
  keycount = keycountbuf;
  fullVerify();
  tverif.stop();
  if (printTime)
    {writeln ("Elapsed time for full verify =",tverif.elapsed()," seconds");}

  // Now complete output

  writef("\n\n IS Benchmark Completed\n");
  writef(" Class           =                         %t\n",probClass);
  writef(" Size            = #########################\n",nkeys);
  writef(" Iterations      = #########################\n",maxIterations);
  if (printTime) {
      writef(" Time in seconds = ######################.##\n",
        tsort.elapsed(TimeUnits.seconds));
      writef(" Mop/s total     = ######################.##\n",
        (maxIterations*nkeys)/tsort.elapsed(TimeUnits.seconds)/1000000 );
  }
  writeln(" Operation type  = ","              keys ranked");
  if (passedVerifications ==  (maxIterations*5+1) ) then
    writeln(" Verification    =                SUCCESSFUL");
  else
    writeln(" Verification    =              UNSUCCESSFUL");

} // end of main program

proc rank_keys ( iteration ) {

  bucket_cnts = 0;			// initialize bucket_cnts and bucket_ptrs
  bucket_ptrs = 0;			//   for each iteration
  keycount    = 0;

  // count the number of occurrences of each key value (each pe can do their share independently)

  tloops(1).start();
  forall (k,i) in zip(key, keyDom) do {
    on k do {
      var ibucket: int = k >> (log2range-log2nbuckets);  // this is equivalent to (key(i)/range)*nbuckets
      var mype: int = i / nkeys_per_pe;
      if false {
        writeln("mype=",mype," i=",i," key=",k," ibucket=",ibucket);
      }
      bucket_cnts[mype, ibucket] += 1;
    }
  }

  tloops(1).stop();
  if (printLoopTimings) then writeln("time for first loop =",tloops(1).elapsed());
  if (DEBUG) {
    var totbucket: [0..nbuckets-1] int;
    for ibucket in 0..nbuckets-1 {
      totbucket[ibucket] = + reduce (bucket_cnts[0..npes-1,ibucket]);
    }
    writeln ("total # keys by bucket= ",totbucket);
  }
  if (printArrays) { writeln("bucket_cnts"); writeln (bucket_cnts); }

  // need to synchronize here - co/forall construct above takes care of that for us

  // figure out pointers for keybuff -- even though these are remote references,
  // each pe can pre-fix scan a column independently

  var hi0 = if distType==ISDistType.block then 0..npes-1
    else 0..nbuckets-1;
  var hi1 = if distType==ISDistType.block then 1..nbuckets-1
    else 1..npes-1;
  tloops(2).start();
  forall b in hi0 do {
    for i in hi1 {
      bucket_ptrs[i, b] = bucket_ptrs[i-1, b] + bucket_cnts[i-1, b];
      if false then writeln("bucket_ptrs", (i,b), "=", bucket_ptrs[i, b]);
    }
  }

  tloops(2).stop();
  if (printLoopTimings) then writeln("time for second loop =",tloops(2).elapsed());
  if (printArrays) { writeln ("bucket_ptrs"); writeln (bucket_ptrs); }

  // now send off the keys -- there are remote references here, but there are no race conditions,
  //                          that is, each pe can work independently
  tloops(3).start();
  forall (k,i) in zip(key, keyDom) do {
    on k do {
      var mype: int = i / nkeys_per_pe;   // change this to a shift?
      var ibucket: int;
      ibucket = k >> (log2range-log2nbuckets);
      if false {writeln("2 mype=",mype," i=",i," key=",k," ibucket=",ibucket);}
      if false then writeln("2 bucket_ptrs", (mype,ibucket), "=", bucket_ptrs[mype,ibucket]);
      keybuff(ibucket, bucket_ptrs[mype,ibucket]) = k;
      bucket_ptrs[mype, ibucket] += 1;
    }
  }
  tloops(3).stop();
  if (printLoopTimings) then writeln("time for 3rd loop =",tloops(3).elapsed());
  if (printArrays) { writeln ("keybuff");     writeln (keybuff); }
  if (printArrays) { writeln ("bucket_ptrs"); writeln (bucket_ptrs); }

  // need to sychronize here - implicit sync of coforall takes care of it for us

  // now think of operating in an alternate domain
  // each pe does this bit
  tloops(4).start();
  forall b in 0..nbuckets-1 do {
    on keybuff[b,0] do     {			// is this necessary?
      var mykeys: int = bucket_ptrs[npes-1, b];
      for i in 0..mykeys-1 {
        var bindx = if distType==ISDistType.block then keybuff[b,i]
          else (keybuff[b,i], 0);
        keycount[bindx] += 1;
      }
    }
  }
  tloops(4).stop();
  if (printLoopTimings) then writeln("time for 4th loop =",tloops(4).elapsed());
  if (printArrays) {writeln ("keycount"); writeln (keycount);}

  // prefix scan the keycount array - this is a global operation, everyone participates
  tloops(5).start();
  keycountbuf = + scan (keycount);
  /*
  if distType==ISDistType.block {
    for i in 1..range-1 {
      keycount[i] += keycount[i-1];
    }
    keycountbuf = keycount;
  } else {
  }
  // keycountbuf = + scan (keycount);
  */
  tloops(5).stop();
  if printLoopTimings then writeln("time for 5th loop =",tloops(5).elapsed());
  if printArrays { writeln ("scanned keycountbuf"); writeln (keycountbuf); }

}

proc partialVerification(iteration: int, keyBuff1 /*:[countDom]int*/ ) {

  //  Performs a partial verification after each iteration.
  //  This code is pretty much a straight translation from the C reference
  //  version except that we use enums to access the comparison values directly.

  for i in 0..4 {
    var k = key(testIndexArray(probClass,i));
    var kbidx = if distType==ISDistType.block then k-1 else (k-1,0);
    select probClass {
      when classVals.S do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i)+iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i)-iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.W do {
        if (i < 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + (iteration-2)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.A do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.B do {
        if (i == 1 || i == 2 || i == 4) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.C do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
    }
  }
  if (DEBUG) {writeln("After partial verification, passedVerifications=",passedVerifications); }
}	// end partial verification

proc fullVerify () {

  // Perform a full verification after all iterations are complete.
  // Since the keys were never actually sorted, we need to do that first.

  for mype in 0..nbuckets-1 do {
    var mykeys: int = bucket_ptrs[npes-1, mype];
    var val: int;
    for i in 0..mykeys-1 {
      var val = if distType==ISDistType.block then keybuff[mype,i]
        else (keybuff[mype,i],0);
      keycount[val] -= 1;
      keybuff[mype,i] = keycount[val];  // keycount is now the row, reuse keybuff for local storage
    }
  }
  if (printArrays) {writeln ("keycount"); writeln (keycount);}
  if (printArrays) {writeln ("keybuff "); writeln (keybuff);}

  // store the ranks in the rank array -- think of being back on the orginal domain
  // first, adjust the bucketptr array

  forall mype in 0..npes-1 do {
    var first: int = mype*nkeys_per_pe;
    var last:  int = first+nkeys_per_pe-1;
    var ibucket: int;
    // var r = last..first by -1;
    var i: int = last;
    for ii in first..last {			// going backwards
      ibucket = key(i) >> (log2range-log2nbuckets);
      bucket_ptrs[mype,ibucket] -= 1;
      if (DEBUG) {writeln(" indx=",i," key=",key(i)," ibucket=",ibucket,
                          " bucket_ptr=", bucket_ptrs[mype,ibucket],
                          " keybuff =",  keybuff[ibucket, bucket_ptrs[mype,ibucket]]); }
      rank[i] = keybuff[ibucket, bucket_ptrs[mype,ibucket]];
      i = i -1;
    }                         // end for
  }                           // end coforall

  // Finally, sort the keys
  for i in 0..nkeys-1 { sortkey[rank[i]] = key[i]; }

  if (printArrays) {writeln ("Rank "); writeln (rank);}
  if (printArrays) {writeln ("Sortd="); writeln(sortkey);}

  // Verify that the keys are sorted correctly
  if ( && reduce ( sortkey[0..nkeys-2] <= sortkey[1..nkeys-1] ) ) then {passedVerifications += 1;}
  if (DEBUG) {writeln ("full Verify: passedVerifications=",passedVerifications);}

}

proc gen_keys () {
  // initialize key values - do this on each rank or locale such that the key values
  // are the same no matter how many locales or ranks are used
  // -- come back later and see if this can be simplified w/ a local range

  coforall loc in Locales do {
    on loc do {
      var tmpreals: [1..4] real;
      var seed: int(64) = 314159265;

      var mype:  int = here.id;
      var first: int = mype*(nkeys/numLocales);
      var last : int = first+(nkeys/numLocales) - 1;

      var rs = new RandomStream(seed);
      rs.skipToNth(first*4+1);
      for i in first..last {
        rs.fillRandom(tmpreals);
        key(i) = ( (range>>2)*(+ reduce tmpreals ) ): int;
      }
      delete rs;
    }
  }
}


Simplifying the check "if there is enough work to do splitting" in the leader
iterator of the guided strategy.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@19058 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config param verbose:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int) {

  if verbose then
    compilerWarning("Working with serial dynamic Iterator");
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int= if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader
{
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/chunkSize < numTasks then {
    writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    //yield tuple(c.translate(-c.low)) ;
    yield tuple(totalRange);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#numTasks do {
      while moreWork do {
	  // There is local work in remain
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=splitChunk(remain, chunkSize, moreWork);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType= densify(current,c);
	    //const zeroBasedIters= current.translate(-c.low);
	    if verbose then
	      writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  //follower(1).translate(c.low);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int) {

  if verbose then
    compilerWarning("Working with serial guided Iterator");
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterator, c:range(?), numTasks:int = if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader
{
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < numTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield tuple(totalRange);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=numTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#numTasks do {
      while undone do {
	  // There is local work in remain(tid)
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=adaptSplit(remain, factor, undone);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType = densify(current,c);
	    if verbose then
	      writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter guided(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower

{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  if verbose then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int) {

  if verbose then
    compilerWarning("Working with serial adaptive Iterator");
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterator, c:range(?), numTasks:int = if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    halt("methodStealing value must be between 0 and 2");

  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < numTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield tuple(totalRange);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#numTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination
    var nVisitedVictims: int=0;

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#numTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if verbose then
	writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
	  // There is local work
	  const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
	  if zeroBasedIters.length !=0 then {
	    if verbose then
	      writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
	    yield tuple(zeroBasedIters);}
	}

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var victim=(tid+1) % numTasks;
      var stealFailed:bool=false;

      while moreWork do {
	  if verbose then
	    writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

	  // Perform the spliting at the victim remaining range

	  select methodStealing {
	    when 0 do { // Steals from the victim until it exhausts its range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                        //after splitting from a victim range
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }

	    when 1 do { // Steals once from victim. If it has sucess on the stealing,
	                // it chooses another victim and tries to steal again.

	      if moreLocalWork[victim] then {
		// There is work in victim
		const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                   //after splitting from a victim range
		if zeroBasedIters2.length !=0 then {
		  if verbose then
		    writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		  yield tuple(zeroBasedIters2);}
	      }
	      else {
		stealFailed=true;
	      }
	    }
	    otherwise { // Tt steals from the victim, but now splitting
	                //from the tail of the victim's range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
		                        // after splitting from a victim range:
		                       // now we set splitTatil to true (second parameter in the call)
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }
	    }
	  // If here, then it can have failed the stealing intent at the victim (method 1),
	  // or we have exhausted the victim range (methods 0, 2)

	  if (methodStealing==0 || methodStealing==2 || methodStealing==1 && stealFailed) then {
	    nVisitedVictims += 1; // Signal that there is no more work in victim
	    if verbose then
	      writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
	    stealFailed=false;
	  }
	  // Check if there is no more work
	  if nVisitedVictims >= numTasks-1 then
	    moreWork=false; // Signal that there is no more work in any victim
	  else {  // There can be still work in other victim
	    victim=(victim+1) % numTasks; // New victim to steal
	    if methodStealing==1 then
	      if victim==tid then victim=(victim+1) % numTasks;
	  }
	}
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/numTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==numTasks-1 then
      rLocal=c#(size*(numTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==numTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    //var current:rType;
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      //const zeroBasedCurrent = densify(current,c);
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  var current:rType=unDensify(follower(1),c);
  //follower(1).translate(c.low);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}

config param verbose:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int) {

  compilerWarning("Working with serial dynamic Iterator");
  if verbose then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int= if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader
{
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/chunkSize < numTasks then {
    writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    //yield tuple(c.translate(-c.low)) ;
    yield tuple(totalRange);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#numTasks do {
      while moreWork do {
	  // There is local work in remain
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=splitChunk(remain, chunkSize, moreWork);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType= densify(current,c);
	    //const zeroBasedIters= current.translate(-c.low);
	    if verbose then
	      writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  //follower(1).translate(c.low);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int) {

  compilerWarning("Working with serial guided Iterator");
  if verbose then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterator, c:range(?), numTasks:int = if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader
{
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/numTasks < numTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield tuple(totalRange);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=numTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#numTasks do {
      while undone do {
	  // There is local work in remain(tid)
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=adaptSplit(remain, factor, undone);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType = densify(current,c);
	    if verbose then
	      writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter guided(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower

{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  if verbose then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int) {

  compilerWarning("Working with serial adaptive Iterator");
  if verbose then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterator, c:range(?), numTasks:int = if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    halt("methodStealing value must be between 0 and 2");

  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < numTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    //    compilerWarning("First yield type:", typeToString(totalRange.type));
    yield tuple(totalRange);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#numTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination
    var nVisitedVictims: int=0;

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#numTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if verbose then
	writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
	  // There is local work
	  const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
	  if zeroBasedIters.length !=0 then {
	    if verbose then
	      writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
	    yield tuple(zeroBasedIters);}
	}

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var victim=(tid+1) % numTasks;
      var stealFailed:bool=false;

      while moreWork do {
	  if verbose then
	    writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

	  // Perform the spliting at the victim remaining range

	  select methodStealing {
	    when 0 do { // Steals from the victim until it exhausts its range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                        //after splitting from a victim range
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }

	    when 1 do { // Steals once from victim. If it has sucess on the stealing,
	                // it chooses another victim and tries to steal again.

	      if moreLocalWork[victim] then {
		// There is work in victim
		const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                   //after splitting from a victim range
		if zeroBasedIters2.length !=0 then {
		  if verbose then
		    writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		  yield tuple(zeroBasedIters2);}
	      }
	      else {
		stealFailed=true;
	      }
	    }
	    otherwise { // Tt steals from the victim, but now splitting
	                //from the tail of the victim's range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
		                        // after splitting from a victim range:
		                       // now we set splitTatil to true (second parameter in the call)
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }
	    }
	  // If here, then it can have failed the stealing intent at the victim (method 1),
	  // or we have exhausted the victim range (methods 0, 2)

	  if (methodStealing==0 || methodStealing==2 || methodStealing==1 && stealFailed) then {
	    nVisitedVictims += 1; // Signal that there is no more work in victim
	    if verbose then
	      writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
	    stealFailed=false;
	  }
	  // Check if there is no more work
	  if nVisitedVictims >= numTasks-1 then
	    moreWork=false; // Signal that there is no more work in any victim
	  else {  // There can be still work in other victim
	    victim=(victim+1) % numTasks; // New victim to steal
	    if methodStealing==1 then
	      if victim==tid then victim=(victim+1) % numTasks;
	  }
	}
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/numTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==numTasks-1 then
      rLocal=c#(size*(numTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==numTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    //var current:rType;
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      //const zeroBasedCurrent = densify(current,c);
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  var current:rType=unDensify(follower(1),c);
  //follower(1).translate(c.low);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}


Realized that I was using forall loops in places that shouldn't be executed in
parallel.  Changed them to regular for loops.  Also, inserted an atomic
statement in a forall loop where a value is decremented then used as an array
index.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@9755 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use Random;
use Time;

enum classVals {S, W, A, B, C};

config const probClass = S;
config const seed = 314159265;
config const printTime = false;

const Class: domain(probClass);

const probSizes: [Class] int = ( 2**16, 2**20, 2**23, 2**25, 2**27 ),
      Bmaxs:     [Class] int = ( 2**11, 2**16, 2**19, 2**21, 2**23 ),
      Imaxs:     [Class] int = ( 10, 10, 10, 10, 10 );

config const probSize = probSizes(probClass);
config const Bmax = Bmaxs(probClass);
config const Imax = Imaxs(probClass);

const D: domain(1) = [0..probSize);
const E: domain(1) = [0..Bmax);

// The following should be replaced by two enumerated domain arrays of arrays
const S_testIndexArray: [0..4] int = ( 48427,17148,23627,62548,4431 );
const S_testRankArray: [0..4] int = ( 0,18,346,64917,65463 );
const W_testIndexArray: [0..4] int = ( 357773,934767,875723,898999,404505 );
const W_testRankArray: [0..4] int = ( 1249,11698,1039987,1043896,1048018 );
const A_testIndexArray: [0..4] int = ( 2112377,662041,5336171,3642833,4250760 );
const A_testRankArray: [0..4] int = ( 104,17523,123928,8288932,8388264 );
const B_testIndexArray: [0..4] int = ( 41869,812306,5102857,18232239,26860214 );
const B_testRankArray: [0..4] int = ( 33422937,10244,59149,33135281,99 );
const C_testIndexArray: [0..4] int = ( 44172927,72999161,74326391,129606274,21736814 );
const C_testRankArray: [0..4] int = ( 61147,882988,266290,133997595,133525895 );

var testIndexArray: [0..4] int;
var testRankArray: [0..4] int;

var keyArray: [D] int;
var ranks: [E] int;
var buffer:  [D] int;

var passedVerifications = 0;

def main() {
  var time = Timer();
  var randomStream = RandomStream(seed);
  var tempreals: [1..4] real;
  var max = Bmax / 4;

  setupPartialVerify();

  writeln(" NAS Parallel Benchmarks (Chapel) - IS Benchmark\n");
  writeln(" Size:  ", probSize, "  (class ", probClass, ")");
  writeln(" Iterations:   ", Imax, "\n");

  // Generate Keys
  for i in D {
    randomStream.fillRandom(tempreals);
    keyArray(i) = (max * (+ reduce tempreals)):int;
  }

  // Do one interation for free (i.e., untimed) to guarantee initialization of
  // all data and code pages and respective tables
  rank(1);

  if (probClass != S) then writeln("   iteration");

  time.start();
  for i in 1..Imax {
    if (probClass != S) then writeln("        ", i);
    rank(i);
  }
  time.stop();
  fullVerify();

  writeln("\n\n IS Benchmark Completed");
  writeln(" Class           = ", probClass);
  writeln(" Size            = ", probSize);
  writeln(" Iterations      = ", Imax);
  if printTime {
    writeln(" Time in seconds = ", time.elapsed(seconds));
    writeln(" Mop/s total     = ",
            (Imax*probSize)/time.elapsed(seconds)/1000000);
  }
  if (passedVerifications == (Imax+1) * 5 + 1) then
    writeln(" Verification    = SUCCESSFUL");
  else
    writeln(" Verification    = FAILED ", passedVerifications);
}

def setupPartialVerify() {
  // Would like to use an array over an enumerated domain of arrays instead
  select probClass {
    when S do {
      testIndexArray = S_testIndexArray;
      testRankArray  = S_testRankArray;
    }
    when W do {
      testIndexArray = W_testIndexArray;
      testRankArray  = W_testRankArray;
    }
    when A do {
      testIndexArray = A_testIndexArray;
      testRankArray  = A_testRankArray;
    }
    when B do {
      testIndexArray = B_testIndexArray;
      testRankArray  = B_testRankArray;
    }
    when C do {
      testIndexArray = C_testIndexArray;
      testRankArray  = C_testRankArray;
    }
  }
}

def rank(iteration: int) {
  keyArray(iteration) = iteration;
  keyArray(iteration+Imax) = Bmax - iteration;

  ranks = 0;
  ranks(keyArray) += 1;
  for (i,j) in (0..Bmax-2, 1..Bmax-1) do ranks(j) += ranks(i);

  partialVerification(iteration);
}

def partialVerification(iteration: int) {
  for i in 0..4 {
    var k = keyArray(testIndexArray(i));
    select probClass {
      when S do {
        if (i <= 2) {
          if (ranks(k-1) != testRankArray(i)+iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i)-iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when W do {
        if (i < 2) {
          if (ranks(k-1) != testRankArray(i) + (iteration-2)) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when A do {
        if (i <= 2) {
          if (ranks(k-1) != testRankArray(i) + (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when B do {
        if (i == 1 || i == 2 || i == 4) {
          if (ranks(k-1) != testRankArray(i) + iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when C do {
        if (i <= 2) {
          if (ranks(k-1) != testRankArray(i) + iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
    }
  }
}

def fullVerify() {
  var failures = 0;
  buffer = keyArray;
  [i in D] {
    atomic {
      ranks(buffer(i)) -= 1;
      keyArray(ranks(buffer(i))) = buffer(i);
    }
  }

  [i in 0..D.numIndices-2]
    if (keyArray(i) > keyArray(i+1)) then failures += 1;

  if (failures != 0) then
    halt("Full verification: number of keys out of sort ", failures);
  else
    passedVerifications += 1;
}

use Random;
use Time;

enum classVals {S, W, A, B, C};

config const probClass = S;
config const seed = 314159265;
config const printTime = false;

const Class: domain(probClass);

const probSizes: [Class] int = ( 2**16, 2**20, 2**23, 2**25, 2**27 ),
      Bmaxs:     [Class] int = ( 2**11, 2**16, 2**19, 2**21, 2**23 ),
      Imaxs:     [Class] int = ( 10, 10, 10, 10, 10 );

config const probSize = probSizes(probClass);
config const Bmax = Bmaxs(probClass);
config const Imax = Imaxs(probClass);

const D: domain(1) = [0..probSize);
const E: domain(1) = [0..Bmax);

// The following should be replaced by two enumerated domain arrays of arrays
const S_testIndexArray: [0..4] int = ( 48427,17148,23627,62548,4431 );
const S_testRankArray: [0..4] int = ( 0,18,346,64917,65463 );
const W_testIndexArray: [0..4] int = ( 357773,934767,875723,898999,404505 );
const W_testRankArray: [0..4] int = ( 1249,11698,1039987,1043896,1048018 );
const A_testIndexArray: [0..4] int = ( 2112377,662041,5336171,3642833,4250760 );
const A_testRankArray: [0..4] int = ( 104,17523,123928,8288932,8388264 );
const B_testIndexArray: [0..4] int = ( 41869,812306,5102857,18232239,26860214 );
const B_testRankArray: [0..4] int = ( 33422937,10244,59149,33135281,99 );
const C_testIndexArray: [0..4] int = ( 44172927,72999161,74326391,129606274,21736814 );
const C_testRankArray: [0..4] int = ( 61147,882988,266290,133997595,133525895 );

var testIndexArray: [0..4] int;
var testRankArray: [0..4] int;

var keyArray: [D] int;
var ranks: [E] int;
var buffer:  [D] int;

var passedVerifications = 0;

def main() {
  var time = Timer();
  var randomStream = RandomStream(seed);
  var tempreals: [1..4] real;
  var max = Bmax / 4;

  setupPartialVerify();

  writeln(" NAS Parallel Benchmarks (Chapel) - IS Benchmark\n");
  writeln(" Size:  ", probSize, "  (class ", probClass, ")");
  writeln(" Iterations:   ", Imax, "\n");

  // Generate Keys
  [i in D] {
    randomStream.fillRandom(tempreals);
    keyArray(i) = (max * (+ reduce tempreals)):int;
  }

  // Do one interation for free (i.e., untimed) to guarantee initialization of
  // all data and code pages and respective tables
  rank(1);

  if (probClass != S) then writeln("   iteration");

  time.start();
  [i in 1..Imax] {
    if (probClass != S) then writeln("        ", i);
    rank(i);
  }
  time.stop();
  fullVerify();

  writeln("\n\n IS Benchmark Completed");
  writeln(" Class           = ", probClass);
  writeln(" Size            = ", probSize);
  writeln(" Iterations      = ", Imax);
  if printTime {
    writeln(" Time in seconds = ", time.elapsed(seconds));
    writeln(" Mop/s total     = ",
            (Imax*probSize)/time.elapsed(seconds)/1000000);
  }
  if (passedVerifications == (Imax+1) * 5 + 1) then
    writeln(" Verification    = SUCCESSFUL");
  else
    writeln(" Verification    = FAILED ", passedVerifications);
}

def setupPartialVerify() {
  // Would like to use an array over an enumerated domain of arrays instead
  select probClass {
    when S do {
      testIndexArray = S_testIndexArray;
      testRankArray  = S_testRankArray;
    }
    when W do {
      testIndexArray = W_testIndexArray;
      testRankArray  = W_testRankArray;
    }
    when A do {
      testIndexArray = A_testIndexArray;
      testRankArray  = A_testRankArray;
    }
    when B do {
      testIndexArray = B_testIndexArray;
      testRankArray  = B_testRankArray;
    }
    when C do {
      testIndexArray = C_testIndexArray;
      testRankArray  = C_testRankArray;
    }
  }
}

def rank(iteration: int) {
  keyArray(iteration) = iteration;
  keyArray(iteration+Imax) = Bmax - iteration;
  ranks = 0;
  [i in D] ranks(keyArray(i)) += 1;
  [i in 0..Bmax-2] ranks(i+1) += ranks(i);

  partialVerification(iteration);
}

def partialVerification(iteration: int) {
  for i in 0..4 {
    var k = keyArray(testIndexArray(i));
    select probClass {
      when S do {
        if (i <= 2) {
          if (ranks(k-1) != testRankArray(i)+iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i)-iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when W do {
        if (i < 2) {
          if (ranks(k-1) != testRankArray(i) + (iteration-2)) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when A do {
        if (i <= 2) {
          if (ranks(k-1) != testRankArray(i) + (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when B do {
        if (i == 1 || i == 2 || i == 4) {
          if (ranks(k-1) != testRankArray(i) + iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when C do {
        if (i <= 2) {
          if (ranks(k-1) != testRankArray(i) + iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (ranks(k-1) != testRankArray(i) - iteration) {
            writeln("Failed partial verification: iteration ",
                 iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
    }
  }
}

def fullVerify() {
  var failures = 0;
  [i in D] buffer(i) = keyArray(i);
  [i in D] {
    ranks(buffer(i)) -= 1;
    keyArray(ranks(buffer(i))) = buffer(i);
  }

  [i in 0..D.numIndices-2]
    if (keyArray(i) > keyArray(i+1)) then failures += 1;

  if (failures != 0) then
    halt("Full verification: number of keys out of sort ", failures);
  else
    passedVerifications += 1;
}