removed formal arguments from verifyResults; passing these arrays as arguments to verification slowed verification down so that it did not scale

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14955 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for Block distributions and Timing routines
//
use BlockDist, Time;

//
// Use the user modules for computing HPCC problem sizes and for
// defining RA's random stream of values
//
use HPCCProblemSize, RARandomStream;

//
// The number of tables as well as the element and index types of
// that table
//
const numTables = 1;
type elemType = randType,
     indexType = randType;

//
// Configuration constants defining log2(problem size) -- n -- and
// the number of updates -- N_U
//
config const n = computeProblemSize(numTables, elemType,
                                    returnLog2=true, retType=indexType),
             N_U = 2**(n+2);

//
// Constants defining the problem size (m) and a bit mask for table
// indexing
//
const m = 2**n,
      indexMask = m-1;

//
// Configuration constant defining the number of errors to allow (as a
// fraction of the number of updates, N_U)
//
config const errorTolerance = 1e-2;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// TableDist is a 1D block distribution for domains storing indices
// of type "indexType", and it is computed by blocking the bounding
// box 0..m-1 across the set of locales.  UpdateDist is a similar
// distribution that is computed by blocking the indices 0..N_U-1
// across the locales.
//
const TableDist = new Block1D(indexType, bbox=[0..m-1]),
      UpdateDist = new Block1D(indexType, bbox=[0..N_U-1]);

//
// TableSpace describes the index set for the table.  It is a 1D
// domain storing indices of type indexType, it is distributed
// according to TableDist, and it contains the indices 0..m-1.
// Updates is an index set describing the set of updates to be made.
// It is distributed according to UpdateDist and contains the
// indices 0..N_U-1.
//
const TableSpace: domain(1, indexType) distributed TableDist = [0..m-1],
      Updates: domain(1, indexType) distributed UpdateDist = [0..N_U-1];

//
// T is the distributed table itself, storing a variable of type
// elemType for each index in TableSpace.
//
var T: [TableSpace] elemType;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // In parallel, initialize the table such that each position
  // contains its index.  "[i in TableSpace]" is shorthand for "forall
  // i in TableSpace"
  //
  [i in TableSpace] T(i) = i;

  const startTime = getCurrentTime();              // capture the start time

  //
  // The main computation: Iterate over the set of updates and the
  // stream of random values in a parallel, zippered manner, dropping
  // the update index on the ground ("_") and storing the random value
  // in r.  Use an on-clause to force the table update to be executed on
  // the locale which owns the table element in question to minimize
  // communications.  Compute the update using r both to compute the
  // index and as the update value.
  //
  forall (_, r) in (Updates, RAStream()) do
    on T.domain.dist.ind2loc(r & indexMask) do
      T(r & indexMask) ^= r;

  const execTime = getCurrentTime() - startTime;   // capture the elapsed time

  const validAnswer = verifyResults();             // verify the updates
  printResults(validAnswer, execTime);             // print the results
}

//
// Print the problem size and number of updates
//
def printConfiguration() {
  if (printParams) {
    printProblemSize(elemType, numTables, m);
    writeln("Number of updates = ", N_U, "\n");
  }
}

//
// Verify that the computation is correct
//
def verifyResults() {
  //
  // Print the table, if requested
  //
  if (printArrays) then writeln("After updates, T is: ", T, "\n");

  //
  // Reverse the updates by recomputing them, this time using an
  // atomic statement to ensure no conflicting updates
  //
  forall (_, r) in (Updates, RAStream()) do
    on T.domain.dist.ind2loc(r & indexMask) do
      atomic T(r & indexMask) ^= r;

  //
  // Print the table again after the updates have been reversed
  //
  if (printArrays) then writeln("After verification, T is: ", T, "\n");

  //
  // Compute the number of table positions that weren't reverted
  // correctly.  This is an indication of the number of conflicting
  // updates.
  //
  const numErrors = + reduce [i in TableSpace] (T(i) != i);
  if (printStats) then writeln("Number of errors is: ", numErrors, "\n");

  //
  // Return whether or not the number of errors was within the benchmark's
  // tolerance.
  //
  return numErrors <= (errorTolerance * N_U);
}

//
// Print out success/failure, the execution time, and the GUPS value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (GUPS) = ", (N_U / execTime) * 1e-9);
  }
}

//
// Use standard modules for Block distributions and Timing routines
//
use BlockDist, Time;

//
// Use the user modules for computing HPCC problem sizes and for
// defining RA's random stream of values
//
use HPCCProblemSize, RARandomStream;

//
// The number of tables as well as the element and index types of
// that table
//
const numTables = 1;
type elemType = randType,
     indexType = randType;

//
// Configuration constants defining log2(problem size) -- n -- and
// the number of updates -- N_U
//
config const n = computeProblemSize(numTables, elemType,
                                    returnLog2=true, retType=indexType),
             N_U = 2**(n+2);

//
// Constants defining the problem size (m) and a bit mask for table
// indexing
//
const m = 2**n,
      indexMask = m-1;

//
// Configuration constant defining the number of errors to allow (as a
// fraction of the number of updates, N_U)
//
config const errorTolerance = 1e-2;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// TableDist is a 1D block distribution for domains storing indices
// of type "indexType", and it is computed by blocking the bounding
// box 0..m-1 across the set of locales.  UpdateDist is a similar
// distribution that is computed by blocking the indices 0..N_U-1
// across the locales.
//
const TableDist = new Block1D(indexType, bbox=[0..m-1]),
      UpdateDist = new Block1D(indexType, bbox=[0..N_U-1]);

//
// TableSpace describes the index set for the table.  It is a 1D
// domain storing indices of type indexType, it is distributed
// according to TableDist, and it contains the indices 0..m-1.
// Updates is an index set describing the set of updates to be made.
// It is distributed according to UpdateDist and contains the
// indices 0..N_U-1.
//
const TableSpace: domain(1, indexType) distributed TableDist = [0..m-1],
      Updates: domain(1, indexType) distributed UpdateDist = [0..N_U-1];

//
// T is the distributed table itself, storing a variable of type
// elemType for each index in TableSpace.
//
var T: [TableSpace] elemType;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // In parallel, initialize the table such that each position
  // contains its index.  "[i in TableSpace]" is shorthand for "forall
  // i in TableSpace"
  //
  [i in TableSpace] T(i) = i;

  const startTime = getCurrentTime();              // capture the start time

  //
  // The main computation: Iterate over the set of updates and the
  // stream of random values in a parallel, zippered manner, dropping
  // the update index on the ground ("_") and storing the random value
  // in r.  Use an on-clause to force the table update to be executed on
  // the locale which owns the table element in question to minimize
  // communications.  Compute the update using r both to compute the
  // index and as the update value.
  //
  forall (_, r) in (Updates, RAStream()) do
    on T.domain.dist.ind2loc(r & indexMask) do
      T(r & indexMask) ^= r;

  const execTime = getCurrentTime() - startTime;   // capture the elapsed time

  const validAnswer = verifyResults(T, Updates);   // verify the updates
  printResults(validAnswer, execTime);             // print the results
}

//
// Print the problem size and number of updates
//
def printConfiguration() {
  if (printParams) {
    printProblemSize(elemType, numTables, m);
    writeln("Number of updates = ", N_U, "\n");
  }
}

//
// Verify that the computation is correct
//
def verifyResults(T: [?TDom], Updates) {
  //
  // Print the table, if requested
  //
  if (printArrays) then writeln("After updates, T is: ", T, "\n");

  //
  // Reverse the updates by recomputing them, this time using an
  // atomic statement to ensure no conflicting updates
  //
  forall (_, r) in (Updates, RAStream()) do
    on T.domain.dist.ind2loc(r & indexMask) do
      atomic T(r & indexMask) ^= r;

  //
  // Print the table again after the updates have been reversed
  //
  if (printArrays) then writeln("After verification, T is: ", T, "\n");

  //
  // Compute the number of table positions that weren't reverted
  // correctly.  This is an indication of the number of conflicting
  // updates.
  //
  const numErrors = + reduce [i in TDom] (T(i) != i);
  if (printStats) then writeln("Number of errors is: ", numErrors, "\n");

  //
  // Return whether or not the number of errors was within the benchmark's
  // tolerance.
  //
  return numErrors <= (errorTolerance * N_U);
}

//
// Print out success/failure, the execution time, and the GUPS value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (GUPS) = ", (N_U / execTime) * 1e-9);
  }
}


Oops, I made this test better in my workaround but
then forgot to update the test itself.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13147 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

class C {
  var arr: [1..3] sync int;
}

var myC = C();

writeln("Made it past constructor");

for i in 1..3 {
  writeln("myC.arr(", i, ") = ", myC.arr(i).readXX());
  writeln("myC.arr(", i, ").isFull = ", myC.arr(i).isFull);
}

class C {
  var arr: [1..3] sync int;
}

var myC = C();

writeln("Made it past constructor");

for i in 1..3 do
  writeln("myC.arr(", i, ") = ", myC.arr(i).readXX());


Use a better name for module used to hide C interfaces

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial and single-locale, though the ultimate intention
// is to also support parallel and multi-locale options.
//
// These interfaces should be considered experimental and subject to
// change for the 1.10 release.  For that reason, send in feedback if
// you would like to see changes or additional arguments/features.
// And, as always, send in bug reports if you think we got something
// wrong.
//

/* iter listdir(path: string, hidden=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * hidden: should hidden files/directories be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, hidden=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (hidden || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          //
          // use FileSystem;  // Doesn't work, see comment below
          //
          const fullpath = path + "/" + filename;
          {
            //
            // The use of this compound statement to restrict the
            // impact of the 'use' of FileSystem is unfortunate
            // (compared to placing it in the more logical place
            // above), yet seemingly required at present; otherwise
            // the 'path' argument gets shadowed by a
            // (compiler-introduced?) method coming from one of the
            // standard or internal modules.  See
            // test/modules/bradc/useFileSystemShadowsPath.chpl for
            // a smaller standalone test exhibiting the issue (or
            // uncomment the 'use' above to see it here).
            //
            use FileSystem;

            if (listlinks || !isLink(fullpath)) {
              if (dirs && isDir(fullpath)) then
                yield filename;
              else if (files && isFile(fullpath)) then
                yield filename;
            }
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 hidden=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.

     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * hidden: indicates whether to enter hidden directories
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden,
                             followlinks, sort) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

//
// Here's a parallel version
//
iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false, param tag: iterKind): string
       where tag == iterKind.standalone {

  if (sort) then
    warning("sorting has no effect for parallel invocations of walkdirs()");

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    forall subdir in subdirs {
      const fullpath = path + "/" + subdir;
      //
      // Call standalone walkdirs() iterator recursively; set sort=false since it is
      // not useful and we've already printed the warning
      //
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden, followlinks, sort=false, iterKind.standalone) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()

     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

module chpl_glob_c_interface {
  extern type glob_t;

  extern const GLOB_NOMATCH: c_int;

  extern proc chpl_glob(pattern:c_string, flags: c_int,
                        ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);
}

iter glob(pattern="*") {
  var glb : chpl_glob_c_interface.glob_t;

  const err = chpl_glob_c_interface.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != chpl_glob_c_interface.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  const num = chpl_glob_c_interface.chpl_glob_num(glb);
  if (num) then
    for i in 0..num-1 do
      yield chpl_glob_c_interface.chpl_glob_index(glb, i): string;

  chpl_glob_c_interface.globfree(glb);
}

iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.standalone {
  var glb : chpl_glob_c_interface.glob_t;

  const err = chpl_glob_c_interface.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != chpl_glob_c_interface.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  const num = chpl_glob_c_interface.chpl_glob_num(glb);
  if (num) then
    forall i in 0..num-1 do
      yield chpl_glob_c_interface.chpl_glob_index(glb, i): string;

  chpl_glob_c_interface.globfree(glb);
}

//
// TODO: The following leader/follower iterator assumes that
// the file system state won't change between calls because
// it calls the glob routines twice and assumes the same
// result will come back.  In leader-follower 2.0, we would
// like to store such state between calls in which case this
// should be rewritten to do so (and would require freeing
// the state at the end of the call).
//
iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.leader {
  var glb : chpl_glob_c_interface.glob_t;

  const err = chpl_glob_c_interface.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != chpl_glob_c_interface.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  //
  // cast is used here to ensure we create an int-based leader
  //
  const num = chpl_glob_c_interface.chpl_glob_num(glb).safeCast(int);
  if (num) {
    chpl_glob_c_interface.globfree(glb);

    //
    // Forward to the range type's leader
    //
    for followThis in (0..num-1).these(tag) do
      yield followThis;
  }
}

iter glob(pattern:string="*", followThis, param tag: iterKind)
       where tag == iterKind.follower {
  var glb : chpl_glob_c_interface.glob_t;
  if (followThis.size != 1) then
    compilerError("glob() iterator can only be zipped with 1D iterators");
  var r = followThis(1);

  const err = chpl_glob_c_interface.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != chpl_glob_c_interface.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  const num = chpl_glob_c_interface.chpl_glob_num(glb);
  if (r.high > num.safeCast(int)) then
    halt("glob() iterator zipped with something too big");
  if (num) then
    for i in r do
      //
      // safe cast is used here to turn an int into a size_t
      //
      yield chpl_glob_c_interface.chpl_glob_index(glb, i.safeCast(size_t)): string;

  chpl_glob_c_interface.globfree(glb);
}

/* iter findfiles(startdir = ".", recursive=false, hidden=false)

   findfiles() is a simple find-like utility implemented using the
   above routines

     * startdir: where to start when looking for files
     * recursive: tells whether or not to descend recurisvely
     * hidden: tells whether or not to yield hidden
*/

iter findfiles(startdir = ".", recursive=false, hidden=false) {
  if (recursive) then
    for subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

iter findfiles(startdir = ".", recursive=false, hidden=false, param tag: iterKind) where tag == iterKind.standalone {
  if (recursive) then
    forall subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial and single-locale, though the ultimate intention
// is to also support parallel and multi-locale options.
//
// These interfaces should be considered experimental and subject to
// change for the 1.10 release.  For that reason, send in feedback if
// you would like to see changes or additional arguments/features.
// And, as always, send in bug reports if you think we got something
// wrong.
//

/* iter listdir(path: string, hidden=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * hidden: should hidden files/directories be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, hidden=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (hidden || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          //
          // use FileSystem;  // Doesn't work, see comment below
          //
          const fullpath = path + "/" + filename;
          {
            //
            // The use of this compound statement to restrict the
            // impact of the 'use' of FileSystem is unfortunate
            // (compared to placing it in the more logical place
            // above), yet seemingly required at present; otherwise
            // the 'path' argument gets shadowed by a
            // (compiler-introduced?) method coming from one of the
            // standard or internal modules.  See
            // test/modules/bradc/useFileSystemShadowsPath.chpl for
            // a smaller standalone test exhibiting the issue (or
            // uncomment the 'use' above to see it here).
            //
            use FileSystem;

            if (listlinks || !isLink(fullpath)) {
              if (dirs && isDir(fullpath)) then
                yield filename;
              else if (files && isFile(fullpath)) then
                yield filename;
            }
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 hidden=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.

     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * hidden: indicates whether to enter hidden directories
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden,
                             followlinks, sort) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

//
// Here's a parallel version
//
iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false, param tag: iterKind): string
       where tag == iterKind.standalone {

  if (sort) then
    warning("sorting has no effect for parallel invocations of walkdirs()");

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    forall subdir in subdirs {
      const fullpath = path + "/" + subdir;
      //
      // Call standalone walkdirs() iterator recursively; set sort=false since it is
      // not useful and we've already printed the warning
      //
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden, followlinks, sort=false, iterKind.standalone) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()

     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

module Chpl_rt_glob {
  extern type glob_t;

  extern const GLOB_NOMATCH: c_int;

  extern proc chpl_glob(pattern:c_string, flags: c_int,
                        ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);
}

iter glob(pattern="*") {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != Chpl_rt_glob.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    for i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.standalone {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != Chpl_rt_glob.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    forall i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: The following leader/follower iterator assumes that
// the file system state won't change between calls because
// it calls the glob routines twice and assumes the same
// result will come back.  In leader-follower 2.0, we would
// like to store such state between calls in which case this
// should be rewritten to do so (and would require freeing
// the state at the end of the call).
//
iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.leader {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != Chpl_rt_glob.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  //
  // cast is used here to ensure we create an int-based leader
  //
  const num = Chpl_rt_glob.chpl_glob_num(glb).safeCast(int);
  if (num) {
    Chpl_rt_glob.globfree(glb);

    //
    // Forward to the range type's leader
    //
    for followThis in (0..num-1).these(tag) do
      yield followThis;
  }
}

iter glob(pattern:string="*", followThis, param tag: iterKind)
       where tag == iterKind.follower {
  var glb : Chpl_rt_glob.glob_t;
  if (followThis.size != 1) then
    compilerError("glob() iterator can only be zipped with 1D iterators");
  var r = followThis(1);

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  // TODO: Handle error cases better
  if (err != 0 && err != Chpl_rt_glob.GLOB_NOMATCH) then
    __primitive("chpl_error", "unhandled error in glob()");
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (r.high > num.safeCast(int)) then
    halt("glob() iterator zipped with something too big");
  if (num) then
    for i in r do
      //
      // safe cast is used here to turn an int into a size_t
      //
      yield Chpl_rt_glob.chpl_glob_index(glb, i.safeCast(size_t)): string;

  Chpl_rt_glob.globfree(glb);
}

/* iter findfiles(startdir = ".", recursive=false, hidden=false)

   findfiles() is a simple find-like utility implemented using the
   above routines

     * startdir: where to start when looking for files
     * recursive: tells whether or not to descend recurisvely
     * hidden: tells whether or not to yield hidden
*/

iter findfiles(startdir = ".", recursive=false, hidden=false) {
  if (recursive) then
    for subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

iter findfiles(startdir = ".", recursive=false, hidden=false, param tag: iterKind) where tag == iterKind.standalone {
  if (recursive) then
    forall subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}


Corrected typo in halt message in norm example.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@8750 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module Norm{
  enum normType {norm1, norm2, normInf, normFrob};

  def norm(x: [], p: normType) where x.rank == 1 {
    select (p) {
      when norm1 do return + reduce abs(x);
      when norm2 do return sqrt(+ reduce (abs(x)*abs(x)));
      when normInf do return max reduce abs(x);
      when normFrob do halt("Frobenius norm not defined for 1D arrays");
      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x: [?D], p: normType) where x.rank == 2 {
    select (p) {
      when norm1 do
        return max reduce [j in D(2)] (+ reduce abs(x[D(1), j]));

      when norm2 do
        halt("Haven't implemented 2-norm for 2D arrays yet");

      when normInf do
        return max reduce [i in D(1)] (+ reduce abs(x[i, D(2)]));

      when normFrob do return sqrt(+ reduce (abs(x)*abs(x)));

      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x: [], p: normType) where x.rank > 2 {
    compilerError("Norms not implemented for array ranks > 2D");
  }

  def norm(x: []) {
    select (x.rank) {
      when 1 do return norm(x, norm2);
      when 2 do return norm(x, normFrob);
      otherwise compilerError("Norms not implemented for array ranks > 2D");
    }
  }
}

module TestNorm {
  use Norm;

  def testNorm(arr: []) {
    var testType = if (arr.rank == 1) then "vector" else "matrix";
    writeln("Test of ", testType, " norms.  Array = ");
    writeln(arr);
    writeln("1-norm = ", norm(arr, norm1));
    if (arr.rank == 1) then
      writeln("2-norm = " , norm(arr, norm2));
    writeln("infinity norm = ", norm(arr, normInf));
    if (arr.rank == 2) then
      writeln("frobenius norm = ", norm(arr, normFrob));
    writeln("default norm = ", norm(arr));
    writeln();
  }

  def main() {
    const D1 = [1..4];
    var a:[D1] real;
    a = 2.0;
    testNorm(a);

    const D2 = [1..2,1..2];
    var b:[D2] real;
    b = 2.0;
    testNorm(b);

    const D3 = [1..3,1..2];
    var c:[D3] real;
    c = 2.0;
    testNorm(c);

    var d:[D2] int;
    d = 1;
    testNorm(d);

    var e:[D2] complex;
    e = 1.0 + 1.0i;
    testNorm(e);

    var f: [1..2, 1..3, 1..4] real = 0.0;
    //    norm(f, norm1);
    //    norm(f);
  }
}

module Norm{
  enum normType {norm1, norm2, normInf, normFrob};

  def norm(x: [], p: normType) where x.rank == 1 {
    select (p) {
      when norm1 do return + reduce abs(x);
      when norm2 do return sqrt(+ reduce (abs(x)*abs(x)));
      when normInf do return max reduce abs(x);
      when normFrob do halt("From-norm not defined for 1D arrays");
      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x: [?D], p: normType) where x.rank == 2 {
    select (p) {
      when norm1 do
        return max reduce [j in D(2)] (+ reduce abs(x[D(1), j]));

      when norm2 do
        halt("Haven't implemented 2-norm for 2D arrays yet");

      when normInf do
        return max reduce [i in D(1)] (+ reduce abs(x[i, D(2)]));

      when normFrob do return sqrt(+ reduce (abs(x)*abs(x)));

      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x: [], p: normType) where x.rank > 2 {
    compilerError("Norms not implemented for array ranks > 2D");
  }

  def norm(x: []) {
    select (x.rank) {
      when 1 do return norm(x, norm2);
      when 2 do return norm(x, normFrob);
      otherwise compilerError("Norms not implemented for array ranks > 2D");
    }
  }
}

module TestNorm {
  use Norm;

  def testNorm(arr: []) {
    var testType = if (arr.rank == 1) then "vector" else "matrix";
    writeln("Test of ", testType, " norms.  Array = ");
    writeln(arr);
    writeln("1-norm = ", norm(arr, norm1));
    if (arr.rank == 1) then
      writeln("2-norm = " , norm(arr, norm2));
    writeln("infinity norm = ", norm(arr, normInf));
    if (arr.rank == 2) then
      writeln("frobenius norm = ", norm(arr, normFrob));
    writeln("default norm = ", norm(arr));
    writeln();
  }

  def main() {
    const D1 = [1..4];
    var a:[D1] real;
    a = 2.0;
    testNorm(a);

    const D2 = [1..2,1..2];
    var b:[D2] real;
    b = 2.0;
    testNorm(b);

    const D3 = [1..3,1..2];
    var c:[D3] real;
    c = 2.0;
    testNorm(c);

    var d:[D2] int;
    d = 1;
    testNorm(d);

    var e:[D2] complex;
    e = 1.0 + 1.0i;
    testNorm(e);

    var f: [1..2, 1..3, 1..4] real = 0.0;
    //    norm(f, norm1);
    //    norm(f);
  }
}


updating subgraph routine to correctly adjust vertex ids

/*
 A simple Graph object.  This code was originally in the Chapel source, which can be found here:
 https://github.com/chapel-lang/chapel/blob/master/test/studies/labelprop/Graph.chpl

 */

 module GraphUtils {
   use Core;

     //
     // VertexData: stores the neighbor list of a vertex.
     //
     /* private */
     record VertexData {
       type nodeIdType;
       param weighted; //hardcoded for now.
       type edgeWeightType;

       param initialFirstAvail;
       param initialLastAvail;

       //
       // We will represent the neighbor list as an array of nleType.
       // nle = Neighbor List Element.
       //

       type nleType = (nodeIdType, edgeWeightType);
       //type nleType = (nodeIdType,);
       var ndom = {initialFirstAvail..initialLastAvail};
       var neighborList: [ndom] nleType;
       var last = 0;

       var nid: int;
       //param nid = 1;  // TODO: Assign a vertex id
       var weight: real;

       proc numNeighbors()  return ndom.numIndices;
 /*
       var firstAvailNeighbor$: [vertex_domain] sync int = initialFirstAvail;

       // Both the vertex and firstAvail must be passed by reference.
       // TODO: possibly compute how many neighbors the vertex has, first.
       // Then allocate that big of a neighbor list right away.
       // That way there will be no need for a sync, just an atomic.
       G.Row[u].addEdgeOnVertex(u, v, w, firstAvailNeighbor$[u]);
 */
       // firstAvail$ must be passed by reference
       proc addEdgeOnVertex(to:nodeIdType, weight:edgeWeightType, firstAvail$: sync int) {
         on this do {
           // todo: the compiler should make these values local automatically!
           const /*u = from,*/ v = to, w = weight;
             // Lock and unlock should be within 'local', but currently
             // need to pull them out due to implementation.
             // lock the vertex
             const edgePos = firstAvail$;

           local {
             const prevNdomLen = ndom.high;
             if edgePos > prevNdomLen {
               // grow our arrays, by 2x
               // statistics: growCount += 1;
               ndom = {1..prevNdomLen * 2};
               // bounds checking below will ensure (edgePos <= ndom.high)
             }
             // store the edge
             if weighted {
               neighborList[edgePos] = (v, w);
             } else {
               neighborList[edgePos] = (v,);
             }
           }

             // release the lock
             firstAvail$ = edgePos + 1;
         } // on
       }

       // not parallel-safe
       proc tidyNeighbors(firstAvail$: sync int) {
         local {
           // no synchronization here
           var edgeCount = firstAvail$.readXX() - 1;
           RemoveDuplicates(1, edgeCount);
           // TODO: ideally if we don't save much memory, do not resize
           if edgeCount != ndom.numIndices {
             // statistics: shrinkCount += 1;
             ndom = 1..edgeCount;
           }
           // writeln("stats ", growCount, " ", shrinkCount, ".");
         }
       }
     } // record VertexData

     use BlockDist;

     // ------------------------------------------------------------------------
     // The data structures below are chosen to implement an irregular (sparse)
     // graph using rectangular domains and arrays.
     // Each node in the graph has a list of neighbors and a corresponding list
     // of (integer) weights for the implicit edges.
     // ------------------------------------------------------------------------

     /* store a graph
      */
     class Graph {
       type nodeIdType = int(64);
       param weighted = false;
       type edgeWeightType = int(64);

       const vertices; // generic type - domain of vertices
       //var vertices: domain(1); // generic type - domain of vertices

       param initialFirstAvail = 1;
       param initialLastAvail = 1;

       var   Row      : [vertices] VertexData(nodeIdType, weighted,
                                      edgeWeightType,
                                      initialFirstAvail, initialLastAvail);
       var num_edges = -1;

       /*
       proc init(vertices: domain) {
         writeln("  GOOD ONE, INIT?");
         this.vertices = vertices;
       }
        */

       /* iterate over all neighbor (ID, weight) pairs
          (actually returns an iterable rather than being
           an iterator).
        */
       proc NeighborPairs( v : index (vertices) ) {
         return Row (v).neighborList;
       }

       /* iterate over all neighbor IDs
        */
       iter Neighbors( v : index (vertices) ) {
         for nlElm in Row(v).neighborList do
           yield nlElm(1); // todo -- use nid
       }

       /* iterate over all neighbor IDs
        */
       iter Neighbors( v : index (vertices), param tag: iterKind)
       where tag == iterKind.leader {
         for block in Row(v).neighborList._value.these(tag) do
           yield block;
       }

       /* iterate over all neighbor IDs
        */
       iter Neighbors( v : index (vertices), param tag: iterKind, followThis)
       where tag == iterKind.follower {
         for nlElm in Row(v).neighborList._value.these(tag, followThis) do
           yield nElm(1);
       }

       /* iterate over all neighbor weights
        */
       iter edge_weight( v : index (vertices) ) {
         for nlElm in Row(v).neighborList do
           yield nlElm(2); // todo -- use VertexData.weight
       }

       /* iterate over all neighbor weights
        */
       iter edge_weight( v : index (vertices), param tag: iterKind)
       where tag == iterKind.leader {
         for block in Row(v).neighborList._value.these(tag) do
           yield block;
       }

       /* iterate over all neighbor weights
        */
       iter edge_weight( v : index (vertices), param tag: iterKind, followThis)
       where tag == iterKind.follower {
         for nlElm in Row(v).neighborList._value.these(tag, followThis) do
           yield nlElm(2); // todo -- use VertexData.weight
       }

       /* return the number of neighbors
        */
       proc   n_Neighbors (v : index (vertices) )
       {return Row (v).numNeighbors();}

       /* Returns and array of the degree for each vertex.
        */
       proc degree() {
         var r: [Row.domain] real;
         for i in Row.domain {
           //writeln(" vid: ", i);
           //r[i] = n_Neighbors(i);
           r[i] = degree(i);
         }
         return r;
       }

       /*
        Degree of a single vertex id
        As in http://igraph.org/python/doc/igraph.Vertex-class.html#degree
        */
       proc degree(v: int) {
         return n_Neighbors(v);
       }

       /* Returns a subgraph from the selected vertices
        modeled after igraph: http://igraph.org/python/doc/igraph.Graph-class.html#subgraph
        */
       proc subgraph(vset: domain) {
         var verts: [vset] int,
             newVerts = {1..vset.size};

         var newVertMap: [vset] int = for v in newVerts do v;
         //writeln(" newVertMap: ", newVertMap);
         var subG = new Graph(nodeIdType = int.type,
                           edgeWeightType = this.edgeWeightType,
                           vertices = newVerts,
                           initialLastAvail=0);

         //writeln("subG.vertices: ", subG.vertices);
         forall v in vset {
           var nList: [{1..0}] Row(v).nleType;
           for n in this.Row(v).neighborList {
             // Based on old ID
             if vset.member(n(1)) {
               // Set to new ID
               nList.push_back((newVertMap(n(1)), n(2)));
             }
           }
           subG.Row[newVertMap(v)].ndom = nList.domain;
           subG.Row[newVertMap(v)].neighborList = nList;
           subG.Row[newVertMap(v)].nid = newVertMap(v);
         }
         return (subG, newVertMap);
       }

       /* Return the list of vertex ids as a domain
        */
       proc vs() {
         return vertices;
       }

     } // class Associative_Graph

     /* how to use Graph: e.g.
     const vertex_domain =
       if DISTRIBUTION_TYPE == "BLOCK" then
         {1..N_VERTICES} dmapped Block ( {1..N_VERTICES} )
       else
     {1..N_VERTICES} ;

     writeln("allocating Associative_Graph");
     var G = new Associative_Graph (vertex_domain);
     */

     /* Helps to construct a graph from row, column, value
        format.
     */
     proc buildUndirectedGraph(triples, param weighted:bool, vertices) where
       isRecordType(triples.eltType)
     {

       // sync version, one-pass, but leaves 0s in graph
       /*
       var r: triples.eltType;
       var G = new Graph(nodeIdType = r.to.type,
                         edgeWeightType = r.weight.type,
                         vertices = vertices);
       var firstAvailNeighbor$: [vertices] sync int = G.initialFirstAvail;
       forall trip in triples {
         var u = trip.from;
         var v = trip.to;
         var w = trip.weight;
         // Both the vertex and firstAvail must be passed by reference.
         // TODO: possibly compute how many neighbors the vertex has, first.
         // Then allocate that big of a neighbor list right away.
         // That way there will be no need for a sync, just an atomic.
         G.Row[u].addEdgeOnVertex(v, w, firstAvailNeighbor$[u]);
         G.Row[v].addEdgeOnVertex(u, w, firstAvailNeighbor$[v]);
       }*/

       // atomic version, tidier
       var r: triples.eltType;
       var G = new Graph(nodeIdType = r.to.type,
                         edgeWeightType = r.weight.type,
                         vertices = vertices,
                         initialLastAvail=0);
       var next$: [vertices] atomic int;

       forall x in next$ {
         next$.write(G.initialFirstAvail);
       }

       // Pass 1: count.
       forall trip in triples {
         var u = trip.from;
         var v = trip.to;
         var w = trip.weight;
         // edge from u to v will be represented in both u and v's edge
         // lists
         next$[u].add(1, memory_order_relaxed);
         next$[v].add(1, memory_order_relaxed);
       }
       // resize the edge lists
       forall v in vertices {
         var min = G.initialFirstAvail;
         var max = next$[v].read(memory_order_relaxed) - 1;
         G.Row[v].ndom = {min..max};
       }
       // reset all of the counters.
       forall x in next$ {
         next$.write(G.initialFirstAvail, memory_order_relaxed);
       }
       // Pass 2: populate.
       forall trip in triples {
         var u = trip.from;
         var v = trip.to;
         var w = trip.weight;
         // edge from u to v will be represented in both u and v's edge
         // lists
         var uslot = next$[u].fetchAdd(1, memory_order_relaxed);
         var vslot = next$[v].fetchAdd(1, memory_order_relaxed);
         G.Row[u].neighborList[uslot] = (v,);
         G.Row[v].neighborList[vslot] = (u,);
       }

       return G;
     }

     /*
      Take a general sparse matrix and turn it into a Graph. Mixed w/ this SO
        https://stackoverflow.com/questions/45846989/how-to-iterate-non-zeroes-in-a-sparse-matrix-in-chapel/46248469#46248469
      */
     proc buildFromSparseMatrix(A: [], param weighted:bool, param directed:bool) {
       //const n = max reduce A.shape;
       const n = A.shape[1];

       const vertices: domain(1) = {1..n};
       var G = new Graph(nodeIdType = int.type,
                         edgeWeightType = A.eltType,
                         vertices = vertices,
                         initialLastAvail=0);
       var next$: [vertices] atomic int;

       // Set number of edges per node, copying logic above but with a matrix
       if !directed {
         forall v in vertices {
           next$[v].write(G.initialFirstAvail);
           G.Row[v].nid = v;
         }
         // Increase the domain size at the right nodes
         for (u,v) in A.domain {
           var w = A[u,v];
           // edge from u to v will be represented in both u and v's edge
           // lists
           next$[u].add(1, memory_order_relaxed);
           next$[v].add(1, memory_order_relaxed);
         }
         // resize the edge lists
         forall v in vertices {
           var min = G.initialFirstAvail;
           var max = next$[v].read(memory_order_relaxed) - 1;
           G.Row[v].ndom = {min..max};
         }
         // reset all of the counters.
         forall x in next$ {
           next$.write(G.initialFirstAvail, memory_order_relaxed);
         }
         // Pass 2: populate.
         //forall trip in triples {
         forall (u,v) in A.domain {
           var w = A[u,v];
           // edge from u to v will be represented in both u and v's edge
           // lists
           var uslot = next$[u].fetchAdd(1, memory_order_relaxed);
           var vslot = next$[v].fetchAdd(1, memory_order_relaxed);
           G.Row[u].neighborList[uslot] = (v,w);
           G.Row[v].neighborList[vslot] = (u,w);
         }
       }

       // In this case just run along the rows, since no symmetry
       if directed {
         forall v in vertices {
           next$[v].write(G.initialFirstAvail);
           G.Row[v].nid = v;
           // There should be a way to bulk add, but I don't want to mess with it just yet.
           var t = 0;
           for c in A.domain.dimIter(2,v) do {
             t = t+1;
             next$[v].add(1, memory_order_relaxed);
             G.Row[v].ndom = {G.initialFirstAvail..t};
             G.Row[v].neighborList[t] = (c,A[v,c]);
           }
         }
       }
       return G;
     }

 }

/*
 A simple Graph object.  This code was originally in the Chapel source, which can be found here:
 https://github.com/chapel-lang/chapel/blob/master/test/studies/labelprop/Graph.chpl

 */

 module GraphUtils {
   use Core;

     //
     // VertexData: stores the neighbor list of a vertex.
     //
     /* private */
     record VertexData {
       type nodeIdType;
       param weighted; //hardcoded for now.
       type edgeWeightType;

       param initialFirstAvail;
       param initialLastAvail;

       //
       // We will represent the neighbor list as an array of nleType.
       // nle = Neighbor List Element.
       //

       type nleType = (nodeIdType, edgeWeightType);
       //type nleType = (nodeIdType,);
       var ndom = {initialFirstAvail..initialLastAvail};
       var neighborList: [ndom] nleType;
       var last = 0;

       var nid: int;
       //param nid = 1;  // TODO: Assign a vertex id
       var weight: real;

       proc numNeighbors()  return ndom.numIndices;
 /*
       var firstAvailNeighbor$: [vertex_domain] sync int = initialFirstAvail;

       // Both the vertex and firstAvail must be passed by reference.
       // TODO: possibly compute how many neighbors the vertex has, first.
       // Then allocate that big of a neighbor list right away.
       // That way there will be no need for a sync, just an atomic.
       G.Row[u].addEdgeOnVertex(u, v, w, firstAvailNeighbor$[u]);
 */
       // firstAvail$ must be passed by reference
       proc addEdgeOnVertex(to:nodeIdType, weight:edgeWeightType, firstAvail$: sync int) {
         on this do {
           // todo: the compiler should make these values local automatically!
           const /*u = from,*/ v = to, w = weight;
             // Lock and unlock should be within 'local', but currently
             // need to pull them out due to implementation.
             // lock the vertex
             const edgePos = firstAvail$;

           local {
             const prevNdomLen = ndom.high;
             if edgePos > prevNdomLen {
               // grow our arrays, by 2x
               // statistics: growCount += 1;
               ndom = {1..prevNdomLen * 2};
               // bounds checking below will ensure (edgePos <= ndom.high)
             }
             // store the edge
             if weighted {
               neighborList[edgePos] = (v, w);
             } else {
               neighborList[edgePos] = (v,);
             }
           }

             // release the lock
             firstAvail$ = edgePos + 1;
         } // on
       }

       // not parallel-safe
       proc tidyNeighbors(firstAvail$: sync int) {
         local {
           // no synchronization here
           var edgeCount = firstAvail$.readXX() - 1;
           RemoveDuplicates(1, edgeCount);
           // TODO: ideally if we don't save much memory, do not resize
           if edgeCount != ndom.numIndices {
             // statistics: shrinkCount += 1;
             ndom = 1..edgeCount;
           }
           // writeln("stats ", growCount, " ", shrinkCount, ".");
         }
       }
     } // record VertexData

     use BlockDist;

     // ------------------------------------------------------------------------
     // The data structures below are chosen to implement an irregular (sparse)
     // graph using rectangular domains and arrays.
     // Each node in the graph has a list of neighbors and a corresponding list
     // of (integer) weights for the implicit edges.
     // ------------------------------------------------------------------------

     /* store a graph
      */
     class Graph {
       type nodeIdType = int(64);
       param weighted = false;
       type edgeWeightType = int(64);

       const vertices; // generic type - domain of vertices
       //var vertices: domain(1); // generic type - domain of vertices

       param initialFirstAvail = 1;
       param initialLastAvail = 1;

       var   Row      : [vertices] VertexData(nodeIdType, weighted,
                                      edgeWeightType,
                                      initialFirstAvail, initialLastAvail);
       var num_edges = -1;

       /*
       proc init(vertices: domain) {
         writeln("  GOOD ONE, INIT?");
         this.vertices = vertices;
       }
        */

       /* iterate over all neighbor (ID, weight) pairs
          (actually returns an iterable rather than being
           an iterator).
        */
       proc NeighborPairs( v : index (vertices) ) {
         return Row (v).neighborList;
       }

       /* iterate over all neighbor IDs
        */
       iter Neighbors( v : index (vertices) ) {
         for nlElm in Row(v).neighborList do
           yield nlElm(1); // todo -- use nid
       }

       /* iterate over all neighbor IDs
        */
       iter Neighbors( v : index (vertices), param tag: iterKind)
       where tag == iterKind.leader {
         for block in Row(v).neighborList._value.these(tag) do
           yield block;
       }

       /* iterate over all neighbor IDs
        */
       iter Neighbors( v : index (vertices), param tag: iterKind, followThis)
       where tag == iterKind.follower {
         for nlElm in Row(v).neighborList._value.these(tag, followThis) do
           yield nElm(1);
       }

       /* iterate over all neighbor weights
        */
       iter edge_weight( v : index (vertices) ) {
         for nlElm in Row(v).neighborList do
           yield nlElm(2); // todo -- use VertexData.weight
       }

       /* iterate over all neighbor weights
        */
       iter edge_weight( v : index (vertices), param tag: iterKind)
       where tag == iterKind.leader {
         for block in Row(v).neighborList._value.these(tag) do
           yield block;
       }

       /* iterate over all neighbor weights
        */
       iter edge_weight( v : index (vertices), param tag: iterKind, followThis)
       where tag == iterKind.follower {
         for nlElm in Row(v).neighborList._value.these(tag, followThis) do
           yield nlElm(2); // todo -- use VertexData.weight
       }

       /* return the number of neighbors
        */
       proc   n_Neighbors (v : index (vertices) )
       {return Row (v).numNeighbors();}

       /* Returns and array of the degree for each vertex.
        */
       proc degree() {
         var r: [Row.domain] real;
         for i in Row.domain {
           //writeln(" vid: ", i);
           r[i] = n_Neighbors(i);
         }
         return r;
       }

       /*
        Degree of a single vertex id
        As in http://igraph.org/python/doc/igraph.Vertex-class.html#degree
        */
       proc degree(v: int) {
         return n_Neighbors(v);
       }

       /* Returns a subgraph from the selected vertices
        modeled after igraph: http://igraph.org/python/doc/igraph.Graph-class.html#subgraph
        */
       proc subgraph(vset: domain) {
         var verts: [vset] int,
             newVerts = {1..vset.size};

         var newVertMap: [vset] int = for v in newVerts do v;
         var subG = new Graph(nodeIdType = int.type,
                           edgeWeightType = this.edgeWeightType,
                           vertices = newVerts,
                           initialLastAvail=0);

         forall v in vset {
           var nList = for n in this.Row(v).neighborList do
             if vset.member(n(1)) then n;
           //subG.Row[v].ndom = nList.domain;
           subG.Row[newVertMap(v)].ndom = nList.domain;
           subG.Row[newVertMap(v)].neighborList = nList;
         }
         return subG;
       }

       /* Return the list of vertex ids as a domain
        */
       proc vs() {
         return vertices;
       }

     } // class Associative_Graph

     /* how to use Graph: e.g.
     const vertex_domain =
       if DISTRIBUTION_TYPE == "BLOCK" then
         {1..N_VERTICES} dmapped Block ( {1..N_VERTICES} )
       else
     {1..N_VERTICES} ;

     writeln("allocating Associative_Graph");
     var G = new Associative_Graph (vertex_domain);
     */

     /* Helps to construct a graph from row, column, value
        format.
     */
     proc buildUndirectedGraph(triples, param weighted:bool, vertices) where
       isRecordType(triples.eltType)
     {

       // sync version, one-pass, but leaves 0s in graph
       /*
       var r: triples.eltType;
       var G = new Graph(nodeIdType = r.to.type,
                         edgeWeightType = r.weight.type,
                         vertices = vertices);
       var firstAvailNeighbor$: [vertices] sync int = G.initialFirstAvail;
       forall trip in triples {
         var u = trip.from;
         var v = trip.to;
         var w = trip.weight;
         // Both the vertex and firstAvail must be passed by reference.
         // TODO: possibly compute how many neighbors the vertex has, first.
         // Then allocate that big of a neighbor list right away.
         // That way there will be no need for a sync, just an atomic.
         G.Row[u].addEdgeOnVertex(v, w, firstAvailNeighbor$[u]);
         G.Row[v].addEdgeOnVertex(u, w, firstAvailNeighbor$[v]);
       }*/

       // atomic version, tidier
       var r: triples.eltType;
       var G = new Graph(nodeIdType = r.to.type,
                         edgeWeightType = r.weight.type,
                         vertices = vertices,
                         initialLastAvail=0);
       var next$: [vertices] atomic int;

       forall x in next$ {
         next$.write(G.initialFirstAvail);
       }

       // Pass 1: count.
       forall trip in triples {
         var u = trip.from;
         var v = trip.to;
         var w = trip.weight;
         // edge from u to v will be represented in both u and v's edge
         // lists
         next$[u].add(1, memory_order_relaxed);
         next$[v].add(1, memory_order_relaxed);
       }
       // resize the edge lists
       forall v in vertices {
         var min = G.initialFirstAvail;
         var max = next$[v].read(memory_order_relaxed) - 1;
         G.Row[v].ndom = {min..max};
       }
       // reset all of the counters.
       forall x in next$ {
         next$.write(G.initialFirstAvail, memory_order_relaxed);
       }
       // Pass 2: populate.
       forall trip in triples {
         var u = trip.from;
         var v = trip.to;
         var w = trip.weight;
         // edge from u to v will be represented in both u and v's edge
         // lists
         var uslot = next$[u].fetchAdd(1, memory_order_relaxed);
         var vslot = next$[v].fetchAdd(1, memory_order_relaxed);
         G.Row[u].neighborList[uslot] = (v,);
         G.Row[v].neighborList[vslot] = (u,);
       }

       return G;
     }

     /*
      Take a general sparse matrix and turn it into a Graph. Mixed w/ this SO
        https://stackoverflow.com/questions/45846989/how-to-iterate-non-zeroes-in-a-sparse-matrix-in-chapel/46248469#46248469
      */
     proc buildFromSparseMatrix(A: [], param weighted:bool, param directed:bool) {
       //const n = max reduce A.shape;
       const n = A.shape[1];

       const vertices: domain(1) = {1..n};
       var G = new Graph(nodeIdType = int.type,
                         edgeWeightType = A.eltType,
                         vertices = vertices,
                         initialLastAvail=0);
       var next$: [vertices] atomic int;

       // Set number of edges per node, copying logic above but with a matrix
       if !directed {
         forall v in vertices {
           next$[v].write(G.initialFirstAvail);
           G.Row[v].nid = v;
         }
         // Increase the domain size at the right nodes
         for (u,v) in A.domain {
           var w = A[u,v];
           // edge from u to v will be represented in both u and v's edge
           // lists
           next$[u].add(1, memory_order_relaxed);
           next$[v].add(1, memory_order_relaxed);
         }
         // resize the edge lists
         forall v in vertices {
           var min = G.initialFirstAvail;
           var max = next$[v].read(memory_order_relaxed) - 1;
           G.Row[v].ndom = {min..max};
         }
         // reset all of the counters.
         forall x in next$ {
           next$.write(G.initialFirstAvail, memory_order_relaxed);
         }
         // Pass 2: populate.
         //forall trip in triples {
         forall (u,v) in A.domain {
           var w = A[u,v];
           // edge from u to v will be represented in both u and v's edge
           // lists
           var uslot = next$[u].fetchAdd(1, memory_order_relaxed);
           var vslot = next$[v].fetchAdd(1, memory_order_relaxed);
           G.Row[u].neighborList[uslot] = (v,w);
           G.Row[v].neighborList[vslot] = (u,w);
         }
       }

       // In this case just run along the rows, since no symmetry
       if directed {
         forall v in vertices {
           next$[v].write(G.initialFirstAvail);
           G.Row[v].nid = v;
           // There should be a way to bulk add, but I don't want to mess with it just yet.
           var t = 0;
           for c in A.domain.dimIter(2,v) do {
             t = t+1;
             next$[v].add(1, memory_order_relaxed);
             G.Row[v].ndom = {G.initialFirstAvail..t};
             G.Row[v].neighborList[t] = (c,A[v,c]);
           }
         }
       }
       return G;
     }

 }


Added an extra argument to the call to chpl_malloc

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15485 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

_extern def chpl_malloc(number, size, description, true, lineno, filename);

chpl_malloc(max(int(32)), max(int(32)), "test big malloc", 3, "allocBig.chpl");

_extern def chpl_malloc(number, size, description, lineno, filename);

chpl_malloc(max(int(32)), max(int(32)), "test big malloc", 3, "allocBig.chpl");


Removing the :[] specification on array formal arguments when
the domain is not needed.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@8699 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module Norm{
  enum normType {norm1, norm2, normInf, normFrob};

  def norm(x, p: normType) where x.rank == 1 {
    select (p) {
      when norm1 do return + reduce abs(x);
      when norm2 do return sqrt(+ reduce (abs(x)*abs(x)));
      when normInf do return max reduce abs(x);
      when normFrob do halt("From-norm not defined for 1D arrays");
      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x:[?D], p: normType) where x.rank == 2 {
    select (p) {
      when norm1 do
        return max reduce [j in D(2)] (+ reduce abs(x[D(1), j]));

      when norm2 do
        halt("Haven't implemented 2-norm for 2D arrays yet");

      when normInf do
        return max reduce [i in D(1)] (+ reduce abs(x[i, D(2)]));

      when normFrob do return sqrt(+ reduce (abs(x)*abs(x)));

      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x, p: normType) where x.rank > 2 {
    compilerError("Norms not implemented for array ranks > 2D");
  }

  def norm(x) {
    select (x.rank) {
      when 1 do return norm(x, norm2);
      when 2 do return norm(x, normFrob);
      otherwise compilerError("Norms not implemented for array ranks > 2D");
    }
  }
}

module TestNorm {
  use Norm;

  def testNorm(arr) {
    var testType = if (arr.rank == 1) then "vector" else "matrix";
    writeln("Test of ", testType, " norms.  Array = ");
    writeln(arr);
    writeln("1-norm = ", norm(arr, norm1));
    if (arr.rank == 1) then
      writeln("2-norm = " , norm(arr, norm2));
    writeln("infinity norm = ", norm(arr, normInf));
    if (arr.rank == 2) then
      writeln("frobenius norm = ", norm(arr, normFrob));
    writeln("default norm = ", norm(arr));
    writeln();
  }

  def main() {
    const D1 = [1..4];
    var a:[D1] real;
    a = 2.0;
    testNorm(a);

    const D2 = [1..2,1..2];
    var b:[D2] real;
    b = 2.0;
    testNorm(b);

    const D3 = [1..3,1..2];
    var c:[D3] real;
    c = 2.0;
    testNorm(c);

    var d:[D2] int;
    d = 1;
    testNorm(d);

    var e:[D2] complex;
    e = 1.0 + 1.0i;
    testNorm(e);

    var f: [1..2, 1..3, 1..4] real = 0.0;
    //    norm(f, norm1);
    //    norm(f);
  }
}

module Norm{
  enum normType {norm1, norm2, normInf, normFrob};

  def norm(x: [], p: normType) where x.rank == 1 {
    select (p) {
      when norm1 do return + reduce abs(x);
      when norm2 do return sqrt(+ reduce (abs(x)*abs(x)));
      when normInf do return max reduce abs(x);
      when normFrob do halt("From-norm not defined for 1D arrays");
      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x:[?D], p: normType) where x.rank == 2 {
    select (p) {
      when norm1 do
        return max reduce [j in D(2)] (+ reduce abs(x[D(1), j]));

      when norm2 do
        halt("Haven't implemented 2-norm for 2D arrays yet");

      when normInf do
        return max reduce [i in D(1)] (+ reduce abs(x[i, D(2)]));

      when normFrob do return sqrt(+ reduce (abs(x)*abs(x)));

      otherwise halt("Unexpected norm type");
    }
  }

  def norm(x:[], p: normType) where x.rank > 2 {
    compilerError("Norms not implemented for array ranks > 2D");
  }

  def norm(x: []) {
    select (x.rank) {
      when 1 do return norm(x, norm2);
      when 2 do return norm(x, normFrob);
      otherwise compilerError("Norms not implemented for array ranks > 2D");
    }
  }
}

module TestNorm {
  use Norm;

  def testNorm(arr) {
    var testType = if (arr.rank == 1) then "vector" else "matrix";
    writeln("Test of ", testType, " norms.  Array = ");
    writeln(arr);
    writeln("1-norm = ", norm(arr, norm1));
    if (arr.rank == 1) then
      writeln("2-norm = " , norm(arr, norm2));
    writeln("infinity norm = ", norm(arr, normInf));
    if (arr.rank == 2) then
      writeln("frobenius norm = ", norm(arr, normFrob));
    writeln("default norm = ", norm(arr));
    writeln();
  }

  def main() {
    const D1 = [1..4];
    var a:[D1] real;
    a = 2.0;
    testNorm(a);

    const D2 = [1..2,1..2];
    var b:[D2] real;
    b = 2.0;
    testNorm(b);

    const D3 = [1..3,1..2];
    var c:[D3] real;
    c = 2.0;
    testNorm(c);

    var d:[D2] int;
    d = 1;
    testNorm(d);

    var e:[D2] complex;
    e = 1.0 + 1.0i;
    testNorm(e);

    var f: [1..2, 1..3, 1..4] real = 0.0;
    //    norm(f, norm1);
    //    norm(f);
  }
}


Updated externed chpl_malloc in this test.  I missed it earlier because this
test is only run on 32-bit linux systems.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15586 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

_extern def chpl_malloc(number, size, description, lineno, filename);

chpl_malloc(max(int(32)), max(int(32)), "test big malloc", 3, "allocBig.chpl");

_extern def chpl_malloc(number, size, description, userCode, lineno, filename);

chpl_malloc(max(int(32)), max(int(32)), "test big malloc", true, 3, "allocBig.chpl");