Some test update.

Some test update.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@4367 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

--param del = -1000000;
--var del = -1000000;

class HashTable {
  type T;
  var size : integer;
  var del : T;
  var null : T;

  var table: _fdata(T) = _fdata(T, 20);

  function h1(k : integer) : integer {
    return k mod size;
  }

  function Init() {
    if (null) {
      for i in 1..size do
        table(i-1) = null;
    }
    else {
      --halt
      writeln("You should have set a null value");
    }
  }

  function h2(k : integer) : integer {
    return (1 + k mod (size - 1));
  }

  function h(k : integer, i : integer) : integer {
    return ((h1(k) + i*h2(k)) mod size);
  }

  function Insert(k: integer, v : T) : integer {
    var j : integer = -1;
    var i : integer = 0;
    while (i < size){
      j = h(k, i);
      if (table(j) == null or table(j) == del) {
        table(j) = v;
        return j;
      }
      i += 1;
    }
    writeln("Table overflow!");
    return -1;
  }

  function Remove(k : integer) : HashTable {
    var j : integer = Search(k);
    if (j == -1) {
      writeln("The key you want to delete is not in the table!");
      return this;
    }
    table(j) = del;
    return this;
  }

  function Search (k : integer) : integer {
    var i : integer = 0;
    var j : integer = -1;
    while ((i < size) and (table(j) != null)) {
      j = h(k, i);
      if (j == k){
        return j;
      }
      i += 1;
    }
    writeln("Key not found.");
    return -1;
  }
}

  --var size : integer;
  --writeln("Enter table size");
  --read(size);
  --writeln("Size is: ", size);

  var s: integer = 10;

  var del : integer;
  writeln("Enter a <deleted key> value.");
  read(del);

  var null : integer;
  writeln("Enter a <null key> value.");
  read(null);

  var ht : HashTable(T=integer) = HashTable(T=integer, size=s, del=del, null=null);

  ht.Init();

  var i : integer = 0;
  var v : integer;

  while(i < s) {
    writeln("Enter an element to insert");
    read(v);
    --if (v == ht.del)
	--  halt("You cannot insert a key with the same value as the deleted key.");
    ht.Insert(i, v);
    i += 1;
  }
  --writeln(ht);

  for i in 1..ht.size do {
    writeln(ht.table(i-1));
  }

--param del = -1000000;
--var del = -1000000;

class HashTable {
  type T;
  var size : integer;
  var del : T;
  var null : T;

  var table: _fdata(T) = _fdata(T, 20);

  function h1(k : integer) : integer {
    return k mod size;
  }

  function Init() {
    if (null) {
      for i in 1..size do
        table(i-1) = null;
    }
    else {
      --halt
      writeln("You should have set a null value");
    }
  }

  function h2(k : integer) : integer {
    return (1 + k mod (size - 1));
  }

  function h(k : integer, i : integer) : integer {
    return ((h1(k) + i*h2(k)) mod size);
  }

  function Insert(k: integer, v : T) : integer {
    var j : integer = -1;
    var i : integer = 0;
    while (i < size){
      j = h(k, i);
      if (table(j) == null or table(j) == del) {
        table(j) = v;
        return j;
      }
      i += 1;
    }
    writeln("Table overflow!");
    return -1;
  }

  function Remove(k : integer) : HashTable {
    var j : integer = Search(k);
    if (j == -1) {
      writeln("The key you want to delete is not in the table!");
      return this;
    }
    table(j) = del;
    return this;
  }

  function Search (k : integer) : integer {
    var i : integer = 0;
    var j : integer = -1;
    while ((i < size) or (table(j) == null)) {
      j = h(k, i);
      if (j == k){
        return j;
      }
      i += 1;
    }
    writeln("Key not found.");
    return -1;
  }
}

  --var size : integer;
  --writeln("Enter table size");
  --read(size);
  --writeln("Size is: ", size);

  var s: integer = 10;

  var del : integer;
  writeln("Enter a <deleted key> value.");
  read(del);

  var null : integer;
  writeln("Enter a <null key> value.");
  read(null);

  var ht : HashTable(T=integer) = HashTable(T=integer, size=s, del=del, null=null);

  ht.Init();

  var i : integer = 0;
  var v : integer;

  while(i < s) {
    writeln("Enter an element to insert");
    read(v);
    --if (v == ht.del)
	--  halt("You cannot insert a key with the same value as the deleted key.");
    ht.Insert(i, v);
    i += 1;
  }
  --writeln(ht);

  for i in 1..ht.size do {
    writeln(ht.table(i-1));
  }


Another typo: comment said that after resetting a sync, the state would be full.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@22176 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Sync and Single Variables Primer
//
// This primer illustrates Chapel's sync and single variables.
//
config const n = 7;

// Sync and single are type qualifiers that can be applied to all
// Chapel primitive types except strings, complex, and record.
//

// Sync and single variables have an associated state that is either
// 'full' or 'empty'.  Single variables are sync variables that can
// only be written once.  If a sync or single variable is declared
// with an initializing expression, its state is set to full and its
// value is that of the expression.  Without an initializing
// expression, a sync or single variable's state is set to empty and
// its value is the default value for the base type.
//
var sy$: sync int=1;  // state = full, value = 1
var si$: single bool; // state = empty, value = false

// By convention, sync and single variable names end in a $ to
// add a visual cue indicating that reads and writes are possibly
// expensive operations and can potentially block the task.
//

// For both sync and single variables, the state must be empty before
// the value can be written.  When the write has completed, the state
// is set to full.
//
si$ = true; // state = full, value = true

// For both sync and single variables, the state must be full before the
// value can be read.  When the read has completed, the state of a
// sync variable is set to empty, and the state of a single variable
// remains full.
//
var sy = sy$; // sy$: state = empty
var si = si$; // si$: state = full

// If a sync or single variable is not in the correct state for reading
// (full) or writing (empty), the current task blocks.
//
// The following creates a new task via a begin statement and declares
// a variable sy that is initialized to sy$.  The initialization statement
// will block until sy$ is full.  The last statement in the begin
// block sets the done$ to full.
//
var done$: sync bool;
writeln("Launching new task");
begin {
  var sy = sy$; // This statement will block until sy$ is full
  writeln("New task unblocked, sy=", sy);
  done$ = true;
}

// Recall that execution proceeds immediately following the begin
// statement after task creation.
//
writeln("After launching new task");
// When sy$ is written, its state will be set to full and the blocked
// task above will continue.
//
sy$ = n;
// This next statement blocks until last statement in the above begin
// completes.
//
done$;

//
// Example: simple split-phase barrier for tasks
//
var count$: sync int = n;  // counter which also serves as a lock
var release$: single bool; // barrier release

coforall t in 1..n {
  var myc = count$;   // read the count, grab the lock (state = empty)
  if myc!=1 {         // still waiting for others
    write(".");
    count$ = myc-1;   // update the count, release the lock (state = full)
                      // we could do some work while waiting
    release$;         // wait for everyone
  } else {            // last one here
    release$ = true;  // release everyone first (state = full)
    writeln("done");
  }
}

//
// There are a number of methods defined for sync and single variables.
//

// The reset() method, defined for sync variables, sets the value of
// the variable to the default value for the type and the state to
// empty.
//
sy$.reset();  // value = 0, state = empty

// The isFull method returns true if the sync or single variable is
// in the full state, false otherwise.
//
writeln(sy$.isFull);
writeln(si$.isFull);

// The writeEF() method, defined for sync and single variables,
// blocks until the state is empty and then assigns the value argument
// to the variable and then sets the state to full.  Assignment of
// sync and single variables are performed using writeEF().
//
sy$.writeEF(2*n); // equivalent to: sy$ = 2*n;

// The readFE() method, defined for sync variables, blocks until the
// state is full and then reads the value of the variable, sets the
// state to empty, and then returns the value.  Normal reads of sync
// variables are performed using readFE().
//
var sy2 = sy$.readFE(); // equivalent to: var sy2 = sy$;
writeln(sy2);

// The readFF() method, defined for sync and single variables, blocks
// until the state is full and then reads the value of the variable
// and returns the value.  The state remains full.  Normal reads of
// single variables are performed using readFF().
//
var si2 = si$.readFF();
writeln(si2);

// The writeXF() method, defined for sync variables, assigns the
// value argument to the variable and then sets the state to full.
// This method does not block.
//
sy$.writeXF(3*n);

// The readXX() method, defined for sync and single variables, returns
// the value of the variable regardless of the state.  This method
// does not block and the state is unchanged.
//
var sy3 = sy$.readXX();
var si3 = si$.readXX();
writeln(sy3);
writeln(si3);

// The writeFF() method, defined for sync variables, blocks until the
// state is full and then and then assigns the value argument to the
// variable.  The state is unchanged.
//
sy$.writeFF(4*n);

sy$.reset();
// Sync and single arguments are passed by reference.  As a result,
// the state of the variable does not change.
//
writeln(sy$.isFull);
f_withSyncIntFormal(sy$);
writeln(si$.isFull);
f_withSingleBoolFormal(si$);

sy$ = 4*n;
// When a sync or single variable is passed as an argument to a
// function that expects the base type of the variable, the value is
// read before being passed to the function.  Therefore, the task will
// block until the state of the variable is full.
//
f_withIntFormal(sy$);
f_withBoolFormal(si$);

sy$ = 5*n;
// Similarly, when a sync or single variable is passed as an argument
// to a generic function such as writeln(), the value is read before
// being passed to the function.  The generic formal argument is set
// to be the base type of the actual.
//
writeln(sy$);
writeln(si$);

proc f_withSyncIntFormal(x: sync int) {
  writeln(x.isFull);
}

proc f_withSingleBoolFormal(x: single bool) {
  writeln(x.isFull);
}

proc f_withIntFormal(x: int) {
  writeln(x);
}

proc f_withBoolFormal(x: bool) {
  writeln(x);
}

//
// Sync and Single Variables Primer
//
// This primer illustrates Chapel's sync and single variables.
//
config const n = 7;

// Sync and single are type qualifiers that can be applied to all
// Chapel primitive types except strings, complex, and record.
//

// Sync and single variables have an associated state that is either
// 'full' or 'empty'.  Single variables are sync variables that can
// only be written once.  If a sync or single variable is declared
// with an initializing expression, its state is set to full and its
// value is that of the expression.  Without an initializing
// expression, a sync or single variable's state is set to empty and
// its value is the default value for the base type.
//
var sy$: sync int=1;  // state = full, value = 1
var si$: single bool; // state = empty, value = false

// By convention, sync and single variable names end in a $ to
// add a visual cue indicating that reads and writes are possibly
// expensive operations and can potentially block the task.
//

// For both sync and single variables, the state must be empty before
// the value can be written.  When the write has completed, the state
// is set to full.
//
si$ = true; // state = full, value = true

// For both sync and single variables, the state must be full before the
// value can be read.  When the read has completed, the state of a
// sync variable is set to empty, and the state of a single variable
// remains full.
//
var sy = sy$; // sy$: state = empty
var si = si$; // si$: state = full

// If a sync or single variable is not in the correct state for reading
// (full) or writing (empty), the current task blocks.
//
// The following creates a new task via a begin statement and declares
// a variable sy that is initialized to sy$.  The initialization statement
// will block until sy$ is full.  The last statement in the begin
// block sets the done$ to full.
//
var done$: sync bool;
writeln("Launching new task");
begin {
  var sy = sy$; // This statement will block until sy$ is full
  writeln("New task unblocked, sy=", sy);
  done$ = true;
}

// Recall that execution proceeds immediately following the begin
// statement after task creation.
//
writeln("After launching new task");
// When sy$ is written, its state will be set to full and the blocked
// task above will continue.
//
sy$ = n;
// This next statement blocks until last statement in the above begin
// completes.
//
done$;

//
// Example: simple split-phase barrier for tasks
//
var count$: sync int = n;  // counter which also serves as a lock
var release$: single bool; // barrier release

coforall t in 1..n {
  var myc = count$;   // read the count, grab the lock (state = empty)
  if myc!=1 {         // still waiting for others
    write(".");
    count$ = myc-1;   // update the count, release the lock (state = full)
                      // we could do some work while waiting
    release$;         // wait for everyone
  } else {            // last one here
    release$ = true;  // release everyone first (state = full)
    writeln("done");
  }
}

//
// There are a number of methods defined for sync and single variables.
//

// The reset() method, defined for sync variables, sets the value of
// the variable to the default value for the type and the state to
// empty.
//
sy$.reset();  // value = 0, state = full

// The isFull method returns true if the sync or single variable is
// in the full state, false otherwise.
//
writeln(sy$.isFull);
writeln(si$.isFull);

// The writeEF() method, defined for sync and single variables,
// blocks until the state is empty and then assigns the value argument
// to the variable and then sets the state to full.  Assignment of
// sync and single variables are performed using writeEF().
//
sy$.writeEF(2*n); // equivalent to: sy$ = 2*n;

// The readFE() method, defined for sync variables, blocks until the
// state is full and then reads the value of the variable, sets the
// state to empty, and then returns the value.  Normal reads of sync
// variables are performed using readFE().
//
var sy2 = sy$.readFE(); // equivalent to: var sy2 = sy$;
writeln(sy2);

// The readFF() method, defined for sync and single variables, blocks
// until the state is full and then reads the value of the variable
// and returns the value.  The state remains full.  Normal reads of
// single variables are performed using readFF().
//
var si2 = si$.readFF();
writeln(si2);

// The writeXF() method, defined for sync variables, assigns the
// value argument to the variable and then sets the state to full.
// This method does not block.
//
sy$.writeXF(3*n);

// The readXX() method, defined for sync and single variables, returns
// the value of the variable regardless of the state.  This method
// does not block and the state is unchanged.
//
var sy3 = sy$.readXX();
var si3 = si$.readXX();
writeln(sy3);
writeln(si3);

// The writeFF() method, defined for sync variables, blocks until the
// state is full and then and then assigns the value argument to the
// variable.  The state is unchanged.
//
sy$.writeFF(4*n);

sy$.reset();
// Sync and single arguments are passed by reference.  As a result,
// the state of the variable does not change.
//
writeln(sy$.isFull);
f_withSyncIntFormal(sy$);
writeln(si$.isFull);
f_withSingleBoolFormal(si$);

sy$ = 4*n;
// When a sync or single variable is passed as an argument to a
// function that expects the base type of the variable, the value is
// read before being passed to the function.  Therefore, the task will
// block until the state of the variable is full.
//
f_withIntFormal(sy$);
f_withBoolFormal(si$);

sy$ = 5*n;
// Similarly, when a sync or single variable is passed as an argument
// to a generic function such as writeln(), the value is read before
// being passed to the function.  The generic formal argument is set
// to be the base type of the actual.
//
writeln(sy$);
writeln(si$);

proc f_withSyncIntFormal(x: sync int) {
  writeln(x.isFull);
}

proc f_withSingleBoolFormal(x: single bool) {
  writeln(x.isFull);
}

proc f_withIntFormal(x: int) {
  writeln(x);
}

proc f_withBoolFormal(x: bool) {
  writeln(x);
}


More work on matrix helpers...

More work on matrix helpers...

Found an interesting performance issue.

/* 2D matrix helper functions.
 *
 */

use Assert;

// Transpose inputMatrix and store in outputMatrix.
proc transpose(ref outputMatrix: [?D] real, ref inputMatrix: [?E] real)
  where D.rank == 2 && E.rank == 2
{
  if boundsChecking {
    assert(D.dim(1) == E.dim(2) && D.dim(2) == E.dim(1),
           "Dimensions of outputMatrix are not transpose of inputMatrix.");
  }

  var outCols = inputMatrix.domain.dim(1),
    outRows = inputMatrix.domain.dim(2);
  forall (row, col) in outputMatrix.domain {
    outputMatrix[row, col] = inputMatrix[col, row];
  }
}

// Calculate dot product (matrix multiplication) of A and B and store result in
// C. Specifically, it does C = A * B.
proc dotProduct(ref C: [?DC] real, ref A: [?DA] real, ref B: [?DB] real)
  where DC.rank == 2 && DA.rank == 2 && DB.rank == 2
{
  if boundsChecking {
    assert(DC.dim(1) == DA.dim(1) && DC.dim(2) == DB.dim(2),
           "Dimensions for C, A, or B do not work for dot product.");
    assert(DA.dim(2) == DB.dim(1),
           "Inner array dimensions do not match.");
  }

  // // This is the chapelerific way to write the 2d matrix dot
  // // product. Unfortunately, it is pretty slow (2015-03-15) at present so
  // // implement nested forall loops that are functionally the same thing.
  // var rows = A.domain.dim(1),
  //   cols = B.domain.dim(2);
  // forall (row, col) in C.domain {
  //   C[row, col] = + reduce (A[row, 1..] * B[1.., col]);
  // }

  forall col in DB.dim(2) {
    forall row in DA.dim(1) {
      C[row, col] = 0.0;

      for z in DA.dim(2) {
        C[row, col] += A[row, z] * B[z, col];
      }
    }
  }
}

/* 2D matrix helper functions.
 *
 */

// Transpose inputMatrix and store in outputMatrix.
proc transpose(ref outputMatrix: [?D] real, ref inputMatrix: [?E] real)
  where D.rank == 2 && E.rank == 2
{
  if boundsChecking {
    assert(D.dim(1) == E.dim(2) && D.dim(2) == E.dim(1),
           "Dimensions of outputMatrix are not transpose of inputMatrix.");
  }

  var outCols = inputMatrix.domain.dim(1),
    outRows = inputMatrix.domain.dim(2);
  forall (row, col) in outputMatrix.domain {
    outputMatrix[row, col] = inputMatrix[col, row];
  }
}

// Calculate dot product (matrix multiplication) of A and B and store result in
// C. Specifically, it does C = A * B.
proc dotProduct(ref C: [?DC] real, ref A: [?DA] real, ref B: [?DB] real)
  where DC.rank == 2 && DA.rank == 2 && DB.rank == 2
{
  if boundsChecking {
    assert(DC.dim(1) == DA.dim(1) && DC.dim(2) == DB.dim(2),
           "Dimensions for C, A, or B do not work for dot product.");
  }

  var rows = A.domain.dim(1),
    cols = B.domain.dim(2);
  forall (row, col) in C.domain {
    C[row, col] = + reduce (A[row, 1..] * B[1.., col]);
  }
}


Update to use atomic reals all the time (after r20392 added floating
point processor atomics)

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20403 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  +==========================================================================+

{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      proc maxweight(s) {
	var mymax = 0;
	for w in G.edge_weight[s] do
	  if w > mymax then mymax = w;
	return mymax;
      }

      const heaviest_edge_weight = max reduce [s in G.vertices] maxweight(s);

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in G.NeighborPairs (s) do

	  if w == heaviest_edge_weight then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ", heaviest_edge_weight);
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in G.NeighborPairs (s) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
                                Heavy_Edge_List     : domain,
                                Heavy_Edge_Subgraph : [],
                                in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance             : [vertex_domain] atomic int;
        forall m in min_distance do m.write(-1);

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance ( y ).write(0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do {

              if min_distance(w).compareExchangeStrong(-1, path_length) then {
                Next_Level.add (w);
                Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
	      }

	      if min_distance(w).read() == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  use BlockDist;
  config param useOnClause = CHPL_NETWORK_ATOMICS!="ugni";
  // For task-private temporary variables
  config const defaultNumTPVs = 16;
  config var numTPVs = min(defaultNumTPVs, numLocales);
  // Would be nice to use PriavteDist, but aliasing is not supported (yet)
  const PrivateSpace = {LocaleSpace} dmapped Block(boundingBox={LocaleSpace});

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
                                            Between_Cent : [] real,
                                            out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Considering using a dense 1-d array instead.  This would
      // complicate the Level_Set implementation a little, but would
      // probably be more efficient.
      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] sync real = 0.0;
      var Sum_Min_Dist$ : sync real = 0.0;

      //
      // Throughout kernel 4, we use distributed arrays that are
      // private to each iteration of the outer loop.  Because set-up
      // and clean-up of such variables is very expensive, it is
      // desirable to do it at most only once per task versus once
      // per iteration, as a task executes multiple iterations of a
      // forall loop.
      //
      // To avoid this per-iteration overhead, we have implemented
      // an optimization that make certain variables private to each
      // task.  These variables are reinitialized each iteration of
      // the loop, but not reallocated, reprivatized, etc.
      //
      // Note that the Chapel group and collaborators are currently in
      // the process of bringing the idea of task private variable to
      // the language.
      //

      numTPVs = min(numTPVs, starting_vertices.size);
      const TPVSpace = {0..#numTPVs};
      var TPVLocales: [TPVSpace] locale;

      // Here we set up the locales array for our task private temporary
      // variables.  We use _computChunkStartEnd() to distribute the
      // variables approximately evenly across the 1d locales array.
      for t in TPVSpace {
        TPVLocales[t] = Locales[_computeChunkStartEnd(numLocales,
                                                      numTPVs, t+1)[1]-1];
      }
      const TPVLocaleSpace = {TPVSpace} dmapped Block(boundingBox={TPVSpace},
                                                      targetLocales=TPVLocales);

      // There will be numTPVs copies of the temps, thus throttling the
      // number of starting vertices being considered simultaneously.
      var TPV: [TPVLocaleSpace] taskPrivateData(domain(index(vertex_domain)),
                                                vertex_domain.type);

      // Initialize
      coforall (t, i) in (TPVLocaleSpace, TPVSpace) do on TPVLocaleSpace.dist.idxToLocale(t) {
          TPV[i] = new taskPrivateData(domain(index(vertex_domain)), vertex_domain);
          var tpv = TPV[i];
          forall v in vertex_domain do
            tpv.BCaux[v].children_list.nd = {1..G.n_Neighbors[v]};
          for loc in Locales do on loc {
            tpv.Active_Level[here.id] = new Level_Set (Sparse_Vertex_List);
            tpv.Active_Level[here.id].previous = nil;
            tpv.Active_Level[here.id].next = new Level_Set (Sparse_Vertex_List);;
            tpv.Active_Level[here.id].next.previous = tpv.Active_Level[here.id];
            tpv.Active_Level[here.id].next.next = nil;
          }
      }
      var TPVM: TPVManager(TPV.type) = new TPVManager(TPV);

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      forall s in starting_vertices do on vertex_domain.dist.idxToLocale(s) {

        const shere = here.id;

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

        // Initialize task private variables
        const tid = TPVM.gettid();
        const tpv = TPVM.getTPV(tid);
        var BCaux => tpv.BCaux;
        pragma "dont disable remote value forwarding"
        inline proc f1(BCaux, v) {
          BCaux[v].path_count$.write(0.0);
        }
        forall v in vertex_domain do {
          BCaux[v].depend = 0.0;
          BCaux[v].min_distance.write(-1);
          // BCaux[v].path_count$.writeXF(0.0);
          f1(BCaux, v);
          BCaux[v].children_list.child_count.write(0);
        }
	var Lcl_Sum_Min_Dist: sync real = 0.0;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

        //
        // Used to check termination of the forward pass
        //
        // We could use a task-private reduction variable here.  This
        // is yet another concept that the Chapel group is planning to
        // implement.
        //

        var remaining = true;

        //
        // Each locale will have its own level sets.  A locale's level set
        // will only contain nodes that are physically allocated on that
        // particular locale.
        //
        var Active_Level => tpv.Active_Level;
        pragma "dont disable remote value forwarding"
        inline proc f2(BCaux, s) {
          BCaux[s].path_count$.write(1.0);
        }

        var barrier = new Barrier(numLocales);

        coforall loc in Locales do on loc {
          Active_Level[here.id].Members.clear();
          Active_Level[here.id].next.Members.clear();
          if vertex_domain.dist.idxToLocale(s) == here {
            // Establish the initial level sets for the breadth-first
            // traversal from s
            Active_Level[here.id].Members.add(s);
            BCaux[s].min_distance.write(0);
            // BCaux[s].path_count$.writeXF(1);
            f2(BCaux, s);
          }
          barrier.barrier();

          var current_distance : int = 0;

          while remaining do {
	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;

            // The Chapel compiler is still a bit conservative when it
            // comes to forwarding read-only variables to remote
            // locales, so we make a const copy here to insure it is
            // forwarded to the remote locale in the following
            // coforall loop.
            const current_distance_c = current_distance;
            pragma "dont disable remote value forwarding"
            inline proc f3(BCaux, v, u) {
                BCaux[v].path_count$.add(BCaux[u].path_count$.read());
            }

            forall u in Active_Level[here.id].Members do {
              forall v in G.FilteredNeighbors(u) do
                on if useOnClause then vertex_domain.dist.idxToLocale(v) else here {
                  // --------------------------------------------
                  // add any unmarked neighbors to the next level
                  // --------------------------------------------

                  if  BCaux[v].min_distance.compareExchangeStrong(-1, current_distance_c) {
                    var aloc = if useOnClause then here.id else vertex_domain.dist.idxToLocale(v).id;
                    Active_Level[aloc].next.Members.add (v);
                    if VALIDATE_BC then
                      Lcl_Sum_Min_Dist += current_distance_c;
                  }

                  // ------------------------------------------------
                  // only neighbors of  u  that are in the next level
                  // are on shortest paths from s through v.  Some
                  // task will have set  min_distance (v) by the
                  // time this code is reached, whether  v  lies in
                  // the previous, the current or the next level.
                  // ------------------------------------------------

                  if BCaux[v].min_distance.read() == current_distance_c {
                    // BCaux[v].path_count$ += BCaux[u].path_count$.readFF();
                    f3(BCaux, v, u);
                    BCaux[u].children_list.add_child (v);
                  }

                }
            };

            // This barrier is needed to insure all updates to the next
            // level are completed before updating to use the next level

            // do some work while we wait
            // barrier.notify(); // This is expensive without network atomics
            //  for now, just do a normal barrier

            if Active_Level[here.id].next.next == nil {
              Active_Level[here.id].next.next = new Level_Set (Sparse_Vertex_List);
              Active_Level[here.id].next.next.previous = Active_Level[here.id].next;
              Active_Level[here.id].next.next.next = nil;
            } else {
              Active_Level[here.id].next.next.Members.clear();
            }
            // barrier.wait(); // ditto
            barrier.barrier();
            if here.id==shere {
              remaining = false;
            }

            barrier.barrier();
            Active_Level[here.id] = Active_Level[here.id].next;
            if Active_Level[here.id].Members.numIndices:bool then
              remaining = true;

            barrier.barrier();

          };  // end forward pass

          if here.id==0 {
            if VALIDATE_BC then
              Sum_Min_Dist$ += Lcl_Sum_Min_Dist;
          }

          // -------------------------------------------------------------
          // compute the dependencies recursively, traversing the vertices
          // of the graph in non-increasing order of distance (reverse
          // ordering from the initial traversal)
          // -------------------------------------------------------------

          const graph_diameter = current_distance - 1;

          if here.id==0 {
            if DEBUG_KERNEL4 then
              writeln ( " graph diameter from starting node ", s,
                        "  is ", graph_diameter );
          }

          pragma "dont disable remote value forwarding"
          inline proc f4(BCaux, Between_Cent$, u) {
            BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
              ( BCaux[u].path_count$.read() /
                BCaux[v].path_count$.read() )      *
              ( 1.0 + BCaux[v].depend );
            Between_Cent$ (u) += BCaux[u].depend;
          }

          // back up to last level
          var curr_Level =  Active_Level[here.id].previous;

          for current_distance in 2 .. graph_diameter by -1 {
            curr_Level = curr_Level.previous;

            for u in curr_Level.Members do on vertex_domain.dist.idxToLocale(u) {
                f4(BCaux, Between_Cent$, u);
                /*
              BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
                ( BCaux[u].path_count$.readFF() /
                  BCaux[v].path_count$.readFF() )      *
                ( 1.0 + BCaux[v].depend );
              Between_Cent$ (u) += BCaux[u].depend;
                */
            }

            barrier.barrier();
          }
        }

        TPVM.releaseTPV(tid);

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	const n_edges          = G.num_edges;
	const N_VERTICES       = vertex_domain.numIndices;
	const N_START_VERTICES = if starting_vertices == G.vertices
                                 then N_VERTICES
                                      - + reduce [v in vertex_domain]
                                        (G.n_Neighbors (v) == 0)
                                 else starting_vertices.numIndices;
	const TEPS             = 7.0 * N_VERTICES * N_START_VERTICES / K4_time;
	const Adjusted_TEPS    = n_edges * N_START_VERTICES / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
	Sum_Min_Dist = Sum_Min_Dist$;

      Between_Cent = Between_Cent$;

      if DELETE_KERNEL4_DS {
        coforall t in TPVSpace do on t {
          var tpv = TPV[t];
          var al = tpv.Active_Level;
          coforall loc in Locales do on loc {
            var level = al[here.id];
            while level != nil {
                var l2 = level.next;
                delete level;
                level = l2;
            }
          }
          delete TPV[t];
        }
        delete TPVM;
      }

    } // end of Brandes' betweenness centrality calculation

  //
  // Addition support data structures for kernel 4
  //

  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
    var next : Level_Set (Sparse_Vertex_List);
  }

  //
  // Data structure to save Children lists between the forward
  //  and backwards passes
  //
  record child_struct {
    type vertex;
    var nd: domain(1);
    var Row_Children: [nd] vertex;
    var child_count: atomic int;

    // This function should only be called using unique vertices
    proc add_child ( new_child: vertex ) {
      var c = child_count.fetchAdd(1)+1;
      Row_Children[c] = new_child;
    }
  }

  //
  // Implementation of task-private variables for kernel 4
  //
  record taskPrivateArrayData {
    type vertex;
    var min_distance  : atomic int;
    var path_count$   : atomic real;
    var depend        : real;
    var children_list : child_struct(vertex);
  }

  class taskPrivateData {
    type Sparse_Vertex_List;
    const vertex_domain;
    var used  : atomic bool;
    var BCaux : [vertex_domain] taskPrivateArrayData(index(vertex_domain));
    var Active_Level : [PrivateSpace] Level_Set (Sparse_Vertex_List);
  }

  // This is a simple class that hands out task private variables from the
  // array of task private variables, TPV.
  class TPVManager {
    const TPV;
    var currTPV: atomic int;
    proc gettid() {
      const tid = this.currTPV.fetchAdd(1)%numTPVs;
      on this.TPV[tid] do
        while this.TPV[tid].used.testAndSet() do chpl_task_yield();
      return tid;
    }
    proc getTPV(tid) {
      return this.TPV[tid];
    }
    proc releaseTPV(tid) {
      this.TPV[tid].used.clear();
    }
  }

  //
  // reusable barrier implementation
  //
  record Barrier {
    param reusable = true;
    var n: int;
    var count: atomic_int64;
    var done: atomicflag;

    proc Barrier(_n: int) {
      reset(_n);
    }

    inline proc reset(_n: int) {
      on this {
        n = _n;
        count.write(n);
        done.write(false);
      }
    }

    inline proc barrier() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to barrier()");
          if reusable {
            count.waitFor(n-1);
            count.add(1);
            done.clear();
          }
        } else {
          done.waitFor(true);
          if reusable {
            count.add(1);
            done.waitFor(false);
          }
        }
      }
    }

    inline proc notify() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to notify()");
        }
      }
    }

    inline proc wait() {
      on this {
        done.waitFor(true);
        if reusable {
          const myc = count.fetchAdd(1);
          if myc == n-1 then
            done.clear();
          done.waitFor(false);
        }
      }
    }

    inline proc try() {
      return done.read();
    }
  }

}

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  +==========================================================================+

{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      proc maxweight(s) {
	var mymax = 0;
	for w in G.edge_weight[s] do
	  if w > mymax then mymax = w;
	return mymax;
      }

      const heaviest_edge_weight = max reduce [s in G.vertices] maxweight(s);

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in G.NeighborPairs (s) do

	  if w == heaviest_edge_weight then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ", heaviest_edge_weight);
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in G.NeighborPairs (s) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
                                Heavy_Edge_List     : domain,
                                Heavy_Edge_Subgraph : [],
                                in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance             : [vertex_domain] atomic int;
        forall m in min_distance do m.write(-1);

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance ( y ).write(0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do {

              if min_distance(w).compareExchangeStrong(-1, path_length) then {
                Next_Level.add (w);
                Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
	      }

	      if min_distance(w).read() == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  use BlockDist;
  config param useAtomicReal = CHPL_NETWORK_ATOMICS=="ugni";
  config param useOnClause = CHPL_NETWORK_ATOMICS!="ugni";
  // For task-private temporary variables
  config const defaultNumTPVs = 16;
  config var numTPVs = min(defaultNumTPVs, numLocales);
  // Would be nice to use PriavteDist, but aliasing is not supported (yet)
  const PrivateSpace = {LocaleSpace} dmapped Block(boundingBox={LocaleSpace});

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
                                            Between_Cent : [] real,
                                            out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Considering using a dense 1-d array instead.  This would
      // complicate the Level_Set implementation a little, but would
      // probably be more efficient.
      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] sync real = 0.0;
      var Sum_Min_Dist$ : sync real = 0.0;

      //
      // Throughout kernel 4, we use distributed arrays that are
      // private to each iteration of the outer loop.  Because set-up
      // and clean-up of such variables is very expensive, it is
      // desirable to do it at most only once per task versus once
      // per iteration, as a task executes multiple iterations of a
      // forall loop.
      //
      // To avoid this per-iteration overhead, we have implemented
      // an optimization that make certain variables private to each
      // task.  These variables are reinitialized each iteration of
      // the loop, but not reallocated, reprivatized, etc.
      //
      // Note that the Chapel group and collaborators are currently in
      // the process of bringing the idea of task private variable to
      // the language.
      //

      numTPVs = min(numTPVs, starting_vertices.size);
      const TPVSpace = {0..#numTPVs};
      var TPVLocales: [TPVSpace] locale;

      // Here we set up the locales array for our task private temporary
      // variables.  We use _computChunkStartEnd() to distribute the
      // variables approximately evenly across the 1d locales array.
      for t in TPVSpace {
        TPVLocales[t] = Locales[_computeChunkStartEnd(numLocales,
                                                      numTPVs, t+1)[1]-1];
      }
      const TPVLocaleSpace = {TPVSpace} dmapped Block(boundingBox={TPVSpace},
                                                      targetLocales=TPVLocales);

      // There will be numTPVs copies of the temps, thus throttling the
      // number of starting vertices being considered simultaneously.
      var TPV: [TPVLocaleSpace] taskPrivateData(domain(index(vertex_domain)),
                                                vertex_domain.type);

      // Initialize
      coforall (t, i) in (TPVLocaleSpace, TPVSpace) do on TPVLocaleSpace.dist.idxToLocale(t) {
          TPV[i] = new taskPrivateData(domain(index(vertex_domain)), vertex_domain);
          var tpv = TPV[i];
          forall v in vertex_domain do
            tpv.BCaux[v].children_list.nd = {1..G.n_Neighbors[v]};
          for loc in Locales do on loc {
            tpv.Active_Level[here.id] = new Level_Set (Sparse_Vertex_List);
            tpv.Active_Level[here.id].previous = nil;
            tpv.Active_Level[here.id].next = new Level_Set (Sparse_Vertex_List);;
            tpv.Active_Level[here.id].next.previous = tpv.Active_Level[here.id];
            tpv.Active_Level[here.id].next.next = nil;
          }
      }
      var TPVM: TPVManager(TPV.type) = new TPVManager(TPV);

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      forall s in starting_vertices do on vertex_domain.dist.idxToLocale(s) {

        const shere = here.id;

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

        // Initialize task private variables
        const tid = TPVM.gettid();
        const tpv = TPVM.getTPV(tid);
        var BCaux => tpv.BCaux;
        pragma "dont disable remote value forwarding"
          inline proc f1(BCaux, v) {
	  if useAtomicReal then
            BCaux[v].path_count$.write(0.0);
	  else
            BCaux[v].path_count$.writeXF(0.0);
        }
        forall v in vertex_domain do {
          BCaux[v].depend = 0.0;
          BCaux[v].min_distance.write(-1);
          // BCaux[v].path_count$.writeXF(0.0);
          f1(BCaux, v);
          BCaux[v].children_list.child_count.write(0);
        }
	var Lcl_Sum_Min_Dist: sync real = 0.0;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

        //
        // Used to check termination of the forward pass
        //
        // We could use a task-private reduction variable here.  This
        // is yet another concept that the Chapel group is planning to
        // implement.
        //

        var remaining = true;

        //
        // Each locale will have its own level sets.  A locale's level set
        // will only contain nodes that are physically allocated on that
        // particular locale.
        //
        var Active_Level => tpv.Active_Level;
        pragma "dont disable remote value forwarding"
          inline proc f2(BCaux, s) {
	  if useAtomicReal then
            BCaux[s].path_count$.write(1.0);
	  else
            BCaux[s].path_count$.writeXF(1.0);
        }

        var barrier = new Barrier(numLocales);

        coforall loc in Locales do on loc {
          Active_Level[here.id].Members.clear();
          Active_Level[here.id].next.Members.clear();
          if vertex_domain.dist.idxToLocale(s) == here {
            // Establish the initial level sets for the breadth-first
            // traversal from s
            Active_Level[here.id].Members.add(s);
            BCaux[s].min_distance.write(0);
            // BCaux[s].path_count$.writeXF(1);
            f2(BCaux, s);
          }
          barrier.barrier();

          var current_distance : int = 0;

          while remaining do {
	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;

            // The Chapel compiler is still a bit conservative when it
            // comes to forwarding read-only variables to remote
            // locales, so we make a const copy here to insure it is
            // forwarded to the remote locale in the following
            // coforall loop.
            const current_distance_c = current_distance;
            pragma "dont disable remote value forwarding"
              inline proc f3(BCaux, v, u) {
	      if useAtomicReal then
                BCaux[v].path_count$.add(BCaux[u].path_count$.read());
              else
                BCaux[v].path_count$ += BCaux[u].path_count$.readFF();
            }

            forall u in Active_Level[here.id].Members do {
              forall v in G.FilteredNeighbors(u) do
                on if useOnClause then vertex_domain.dist.idxToLocale(v) else here {
                  // --------------------------------------------
                  // add any unmarked neighbors to the next level
                  // --------------------------------------------

                  if  BCaux[v].min_distance.compareExchangeStrong(-1, current_distance_c) {
                    var aloc = if useOnClause then here.id else vertex_domain.dist.idxToLocale(v).id;
                    Active_Level[aloc].next.Members.add (v);
                    if VALIDATE_BC then
                      Lcl_Sum_Min_Dist += current_distance_c;
                  }

                  // ------------------------------------------------
                  // only neighbors of  u  that are in the next level
                  // are on shortest paths from s through v.  Some
                  // task will have set  min_distance (v) by the
                  // time this code is reached, whether  v  lies in
                  // the previous, the current or the next level.
                  // ------------------------------------------------

                  if BCaux[v].min_distance.read() == current_distance_c {
                    // BCaux[v].path_count$ += BCaux[u].path_count$.readFF();
                    f3(BCaux, v, u);
                    BCaux[u].children_list.add_child (v);
                  }

                }
            };

            // This barrier is needed to insure all updates to the next
            // level are completed before updating to use the next level

            // do some work while we wait
            // barrier.notify(); // This is expensive without network atomics
            //  for now, just do a normal barrier

            if Active_Level[here.id].next.next == nil {
              Active_Level[here.id].next.next = new Level_Set (Sparse_Vertex_List);
              Active_Level[here.id].next.next.previous = Active_Level[here.id].next;
              Active_Level[here.id].next.next.next = nil;
            } else {
              Active_Level[here.id].next.next.Members.clear();
            }
            // barrier.wait(); // ditto
            barrier.barrier();
            if here.id==shere {
              remaining = false;
            }

            barrier.barrier();
            Active_Level[here.id] = Active_Level[here.id].next;
            if Active_Level[here.id].Members.numIndices:bool then
              remaining = true;

            barrier.barrier();

          };  // end forward pass

          if here.id==0 {
            if VALIDATE_BC then
              Sum_Min_Dist$ += Lcl_Sum_Min_Dist;
          }

          // -------------------------------------------------------------
          // compute the dependencies recursively, traversing the vertices
          // of the graph in non-increasing order of distance (reverse
          // ordering from the initial traversal)
          // -------------------------------------------------------------

          const graph_diameter = current_distance - 1;

          if here.id==0 {
            if DEBUG_KERNEL4 then
              writeln ( " graph diameter from starting node ", s,
                        "  is ", graph_diameter );
          }

          pragma "dont disable remote value forwarding"
            inline proc f4(BCaux, Between_Cent$, u) {
	    if useAtomicReal then
            BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
              ( BCaux[u].path_count$.read() /
                BCaux[v].path_count$.read() )      *
              ( 1.0 + BCaux[v].depend );
	    else
            BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
              ( BCaux[u].path_count$.readFF() /
                BCaux[v].path_count$.readFF() )      *
              ( 1.0 + BCaux[v].depend );
            Between_Cent$ (u) += BCaux[u].depend;
          }

          // back up to last level
          var curr_Level =  Active_Level[here.id].previous;

          for current_distance in 2 .. graph_diameter by -1 {
            curr_Level = curr_Level.previous;

            for u in curr_Level.Members do on vertex_domain.dist.idxToLocale(u) {
                f4(BCaux, Between_Cent$, u);
                /*
              BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
                ( BCaux[u].path_count$.readFF() /
                  BCaux[v].path_count$.readFF() )      *
                ( 1.0 + BCaux[v].depend );
              Between_Cent$ (u) += BCaux[u].depend;
                */
            }

            barrier.barrier();
          }
        }

        TPVM.releaseTPV(tid);

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	const n_edges          = G.num_edges;
	const N_VERTICES       = vertex_domain.numIndices;
	const N_START_VERTICES = if starting_vertices == G.vertices
                                 then N_VERTICES
                                      - + reduce [v in vertex_domain]
                                        (G.n_Neighbors (v) == 0)
                                 else starting_vertices.numIndices;
	const TEPS             = 7.0 * N_VERTICES * N_START_VERTICES / K4_time;
	const Adjusted_TEPS    = n_edges * N_START_VERTICES / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
	Sum_Min_Dist = Sum_Min_Dist$;

      Between_Cent = Between_Cent$;

      if DELETE_KERNEL4_DS {
        coforall t in TPVSpace do on t {
          var tpv = TPV[t];
          var al = tpv.Active_Level;
          coforall loc in Locales do on loc {
            var level = al[here.id];
            while level != nil {
                var l2 = level.next;
                delete level;
                level = l2;
            }
          }
          delete TPV[t];
        }
        delete TPVM;
      }

    } // end of Brandes' betweenness centrality calculation

  //
  // Addition support data structures for kernel 4
  //

  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
    var next : Level_Set (Sparse_Vertex_List);
  }

  //
  // Data structure to save Children lists between the forward
  //  and backwards passes
  //
  record child_struct {
    type vertex;
    var nd: domain(1);
    var Row_Children: [nd] vertex;
    var child_count: atomic int;

    // This function should only be called using unique vertices
    proc add_child ( new_child: vertex ) {
      var c = child_count.fetchAdd(1)+1;
      Row_Children[c] = new_child;
    }
  }

  //
  // Implementation of task-private variables for kernel 4
  //
  record taskPrivateArrayData {
    type vertex;
    var min_distance  : atomic int;
    var path_count$   : if useAtomicReal then atomic real else sync real;
    var depend        : real;
    var children_list : child_struct(vertex);
  }

  class taskPrivateData {
    type Sparse_Vertex_List;
    const vertex_domain;
    var used  : atomic bool;
    var BCaux : [vertex_domain] taskPrivateArrayData(index(vertex_domain));
    var Active_Level : [PrivateSpace] Level_Set (Sparse_Vertex_List);
  }

  // This is a simple class that hands out task private variables from the
  // array of task private variables, TPV.
  class TPVManager {
    const TPV;
    var currTPV: atomic int;
    proc gettid() {
      const tid = this.currTPV.fetchAdd(1)%numTPVs;
      on this.TPV[tid] do
        while this.TPV[tid].used.testAndSet() do chpl_task_yield();
      return tid;
    }
    proc getTPV(tid) {
      return this.TPV[tid];
    }
    proc releaseTPV(tid) {
      this.TPV[tid].used.clear();
    }
  }

  //
  // reusable barrier implementation
  //
  record Barrier {
    param reusable = true;
    var n: int;
    var count: atomic_int64;
    var done: atomicflag;

    proc Barrier(_n: int) {
      reset(_n);
    }

    inline proc reset(_n: int) {
      on this {
        n = _n;
        count.write(n);
        done.write(false);
      }
    }

    inline proc barrier() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to barrier()");
          if reusable {
            count.waitFor(n-1);
            count.add(1);
            done.clear();
          }
        } else {
          done.waitFor(true);
          if reusable {
            count.add(1);
            done.waitFor(false);
          }
        }
      }
    }

    inline proc notify() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to notify()");
        }
      }
    }

    inline proc wait() {
      on this {
        done.waitFor(true);
        if reusable {
          const myc = count.fetchAdd(1);
          if myc == n-1 then
            done.clear();
          done.waitFor(false);
        }
      }
    }

    inline proc try() {
      return done.read();
    }
  }

}


Add other env helpers to Exec.chpl.

Add other env helpers to Exec.chpl.

/* Execute commands... */

use "chpl_exec.h";
use "chpl_exec.c";

use Assert;

/* Run `cmd` and return exit code and output. */
proc run(cmd: string) {
  extern proc chpl_run(cmd: c_string, ref stdout: c_string_copy): int;

  var temp: c_string_copy,
    output: string;

  const env = getEnviron();
  for key in env.domain {
    setEnv(key, env[key]);
  }

  const result = chpl_run(cmd: c_string, temp);
  output = toString(temp);
  return output;
}

proc getEnviron() {
  extern proc chpl_get_environ_size(): int;
  extern proc chpl_get_environ(env: [] c_string_copy, envSize: int);

  const envSize = 2 * chpl_get_environ_size();
  var D = {1..envSize},
    tempEnv: [D] c_string_copy,
    envKeys: domain(string),
    env: [envKeys] string;

  chpl_get_environ(tempEnv, envSize);

  for i in 1..envSize by 2 {
    const key: string = tempEnv[i],
      value: string = tempEnv[i+1];
    env[key] = value;
  }

  return env;
}

/* Get the value of the envirionment variable. Returns empty string if variable
   is unset.

   :arg envVar: environment variable name
   :type envVar: string

   :returns: environment variable value
   :rtype: string
*/
proc getEnv(envVar: string): string {
  extern proc getenv(envKey: c_string): c_string;

  const value = getenv(envVar);
  return value: string;
}

/* Set the environment variable `envVar` to the given value.

   :arg envVar: environment variable name
   :type envVar: string

   :arg value: environment variable value
   :type value: string
*/
proc setEnv(envVar: string, value: string) {
  extern proc setenv(envKey: c_string, envValue: c_string, overwrite: bool): int;

  const result = setenv(envVar: c_string, value: c_string, true);
  assert(result == 0, "Failed to set env var " + envVar);
}

/* Delete variable `envVar` from environment.

   :arg envVar: environment variable name
   :type envVar: string
*/
proc unsetEnv(envVar: string) {
  extern proc unsetenv(envKey: c_string): int;

  const result = unsetenv(envVar: c_string);
  assert(result == 0, "Failed to unset env var " + envVar);
}

/* Execute commands... */

use "chpl_exec.h";
use "chpl_exec.c";

use Assert;

/* Run `cmd` and return exit code and output. */
proc run(cmd: string) {
  extern proc setenv(envKey: c_string, envValue: c_string, overwrite: bool): int;
  extern proc chpl_run(cmd: c_string, ref stdout: c_string_copy): int;

  var temp: c_string_copy,
    output: string;

  const env = getEnviron();

  for key in env.domain {
    const setResult = setenv(key: c_string, env[key]: c_string, true);
    assert(setResult == 0, "Failed to set env var " + key);
  }

  const result = chpl_run(cmd: c_string, temp);
  output = toString(temp);
  return output;
}

proc getEnviron() {
  extern proc chpl_get_environ_size(): int;
  extern proc chpl_get_environ(env: [] c_string_copy, envSize: int);

  const envSize = 2 * chpl_get_environ_size();
  var D = {1..envSize},
    tempEnv: [D] c_string_copy,
    envKeys: domain(string),
    env: [envKeys] string;

  chpl_get_environ(tempEnv, envSize);

  for i in 1..envSize by 2 {
    const key: string = tempEnv[i],
      value: string = tempEnv[i+1];
    env[key] = value;
  }

  return env;
}


Make _debugWrite and _debugWriteln take generic arguments.

Added a nested function in _debugWrite that checks if the type of its argument
is a simple type that can be cast to string. If yes, cast the argument to
string and return it, if no, issue a compiler error.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15505 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// class file
//
//  chapel-level implementations of read, write, writeln
//  chapel-level implementations of assert, halt

enum FileAccessMode { read, write };

//
// functions on _file primitive type, the C file pointer type
//
pragma "inline" def =(a: _file, b: _file) return b;
pragma "inline" def ==(a: _file, b: _file) return __primitive("==", a, b);
pragma "inline" def !=(a: _file, b: _file) return __primitive("!=", a, b);

pragma "inline" def _readLitChar(fp: _file, val: string, ignoreWhiteSpace: bool)
  return __primitive("_fscan_literal", fp, val, ignoreWhiteSpace);

const stdin  = new file("stdin", FileAccessMode.read, "/dev", __primitive("get_stdin"));
const stdout = new file("stdout", FileAccessMode.write, "/dev", __primitive("get_stdout"));
const stderr = new file("stderr", FileAccessMode.write, "/dev", __primitive("get_stderr"));

class file: Writer {
  var filename : string = "";
  var mode : FileAccessMode = FileAccessMode.read;
  var path : string = ".";
  var _fp : _file;
  var _lock : sync uint(64);    // for serializing output

  def open() {
    on this {
      if this == stdin || this == stdout || this == stderr then
        halt("***Error: It is not necessary to open \"", filename, "\"***");

      var fullFilename = path + "/" + filename;

      var modestring: string;
      select mode {
        when FileAccessMode.read  do modestring = "r";
        when FileAccessMode.write do modestring = "w";
      }
      _fp = __primitive("fopen", filename, modestring);

      if _fp == __primitive("get_nullfile") {
        const err = __primitive("get_errno");
        halt("***Error: Unable to open \"", fullFilename, "\": ", err, "***");
      }
    }
  }

  def _checkFileStateChangeLegality(state) {
    if (isOpen) {
      halt("Cannot change ", state, " of file ", path, "/", filename,
           " while it is open");
    }
  }

  def filename var : string {
    if setter then
      _checkFileStateChangeLegality("filename");
    return filename;
  }

  def path var : string {
    if setter then
      _checkFileStateChangeLegality("path");
    return path;
  }

  def mode var {
    if setter then
      _checkFileStateChangeLegality("mode");
    return mode;
  }

  def isOpen: bool {
    var openStatus: bool = false;
    if (_fp != __primitive("get_nullfile")) {
      openStatus = true;
    }
    return openStatus;
  }

  def close() {
    on this {
      if (this == stdin || this == stdout || this == stderr) {
        halt("***Error: You may not close \"", filename, "\"***");
      }
      if (_fp == __primitive("get_nullfile")) {
        var fullFilename = path + "/" + filename;
        halt("***Error: Trying to close \"", fullFilename,
             "\" which isn't open***");
      }
      var returnVal: int = __primitive("fclose", _fp);
      if (returnVal < 0) {
        var fullFilename = path + "/" + filename;
        const err = __primitive("get_errno");
        halt("***Error: The close of \"", fullFilename, "\" failed: ", err,
             "***");
      }
      _fp = __primitive("get_nullfile");
    }
  }
}

def file.writeThis(f: Writer) {
  f.write("(filename = ",this.filename);
  f.write(", path = ",this.path);
  f.write(", mode = ",this.mode);
  f.write(")");
}

def file.flush() {
  on this do __primitive("fflush", _fp);
}

def _checkOpen(f: file, isRead: bool) {
  if !f.isOpen {
    var fullFilename:string = f.path + "/" + f.filename;
    if isRead {
      halt("***Error: You must open \"", fullFilename,
           "\" before trying to read from it***");
    } else {
      halt("***Error: You must open \"", fullFilename,
           "\" before trying to write to it***");
    }
  }
}

def file.readln() {
  on this {
    if !isOpen then
      _checkOpen(this, isRead=true);
    __primitive("_readToEndOfLine",_fp);
  }
}

def file.readln(inout list ...?n) {
  read((...list));
  on this {
    __primitive("_readToEndOfLine",_fp);
  }
}

def file.readln(type t) {
  var val: t;
  this.readln(val);
  return val;
}

def file.read(inout first, inout rest ...?n) {
  read(first);
  for param i in 1..n do
    read(rest(i));
}

def file.read(inout val: int) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: int;
  on this {
    x = __primitive("_fscan_int32", _fp);
  }
  val = x;
}

def file.read(inout val: uint) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: uint;
  on this {
    x = __primitive("_fscan_uint32", _fp);
  }
  val = x;
}

def file.read(inout val: real) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: real;
  on this {
    x = __primitive("_fscan_real64", _fp);
  }
  val = x;
}

def file.read(inout val: complex) {
  var realPart: real;
  var imagPart: real;
  var imagI: string;
  var matchingCharWasRead: int;
  var isNeg: bool;

  read(realPart);
  on this {
    matchingCharWasRead = _readLitChar(_fp, "+", true);
  }
  if (matchingCharWasRead != 1) {
    on this {
      matchingCharWasRead = _readLitChar(_fp, "-", true);
    }
    if (matchingCharWasRead != 1) {
      halt("***Error: Incorrect format for complex numbers***");
    }
    isNeg = true;
  }

  read(imagPart);
  on this {
    matchingCharWasRead = _readLitChar(_fp, "i", false);
  }
  if (matchingCharWasRead != 1) {
    halt("***Error: Incorrect format for complex numbers***");
  }

  val.re = realPart;
  if (isNeg) {
    val.im = -imagPart;
  } else {
    val.im = imagPart;
  }
}

def file.read(inout val: string) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: string;
  on this {
    x = __primitive("_fscan_string", _fp);
  }
  val = x;
}

def file.read(inout val: bool) {
  var s: string;
  this.read(s);
  if (s == "true") {
    val = true;
  } else if (s == "false") {
    val = false;
  } else {
    halt("read of bool value that is neither true nor false");
  }
}

def file.read(type t) {
  var val: t;
  this.read(val);
  return val;
}

def string.writeThis(f: Writer) {
  f.writeIt(this);
}

def numeric.writeThis(f: Writer) {
  f.writeIt(this:string);
}

def enumerated.writeThis(f: Writer) {
  f.writeIt(this:string);
}

/*
def bool.writeThis(f: Writer) {
  f.writeIt(this:string);
}
*/

def file.writeIt(s: string) {
  if !isOpen then
    _checkOpen(this, isRead = false);
  if mode != FileAccessMode.write then
    halt("***Error: ", path, "/", filename, " not open for writing***");
  var status = __primitive("fprintf", _fp, "%s", s);
  if status < 0 {
    const err = __primitive("get_errno");
    halt("***Error: Write failed: ", err, "***");
  }
}

class StringClass: Writer {
  var s: string;
  def writeIt(s: string) { this.s += s; }
}

pragma "ref this" pragma "dont disable remote value forwarding"
def string.write(args ...?n) {
  var sc = new StringClass(this);
  sc.write((...args));
  this = sc.s;
  delete sc;
}

def file.lockWrite() {
  var me: uint(64) = __primitive("thread_id");
  if _lock.isFull then
    if _lock.readXX() == me then
      return false;
  _lock = me;
  return true;
}

def file.unlockWrite() {
  _lock.reset();
}

def isBoolean(b: bool) param return true;
def isBoolean(b) param return false;

class Writer {
  def writeIt(s: string) { }
  def lockWrite() return false;
  def unlockWrite() { }
  def write(args ...?n) {
    def isNilObject(val) {
      def helper(o: object) return o == nil;
      def helper(o)         return false;
      return helper(val);
    }

    on this {
      var need_release: bool;
      need_release = lockWrite();
      for param i in 1..n do
        if isNilObject(args(i)) then
          "nil".writeThis(this);
        else if isBoolean(args(i)) then
          // This isBoolean case is a workaround for a problem with
          // resolving a reference to a bool to one of the bool.writeThis
          // methods.
          if args(i) then
            "true".writeThis(this);
          else
            "false".writeThis(this);
        else
          args(i).writeThis(this);
      if need_release then
        unlockWrite();
    }
  }
  def writeln(args ...?n) {
    write((...args), "\n");
  }
  def writeln() {
    write("\n");
  }
}

def write(args ...?n) {
  stdout.write((...args));
  stdout.flush();
}

def writeln(args ...?n) {
  stdout.writeln((...args));
}

def writeln() {
  stdout.writeln();
}

def read(inout args ...?n) {
  stdin.read((...args));
}

def readln(inout args ...?n) {
  stdin.readln((...args));
}

def readln() {
  stdin.readln();
}

def readln(type t) {
  return stdin.readln(t);
}

def read(type t)
  return stdin.read(t);

def file.readln(type t ...?numTypes) where numTypes > 1 {
  var tupleVal: t;
  for param i in 1..numTypes-1 do
    tupleVal(i) = this.read(t(i));
  tupleVal(numTypes) = this.readln(t(numTypes));
  return tupleVal;
}

def file.read(type t ...?numTypes) where numTypes > 1 {
  var tupleVal: t;
  for param i in 1..numTypes do
    tupleVal(i) = this.read(t(i));
  return tupleVal;
}

def readln(type t ...?numTypes) where numTypes > 1
  return stdin.readln((...t));

def read(type t ...?numTypes) where numTypes > 1
  return stdin.read((...t));

def _tuple2string(t) {
  var s: string;
  for param i in 1..t.size do
    s.write(t(i));
  return s;
}

def assert(test: bool) {
  if !test then
    __primitive("chpl_error", "assert failed");
}

def assert(test: bool, args ...?numArgs) {
  if !test then
    __primitive("chpl_error", "assert failed - "+_tuple2string(args));
}

def halt() {
  __primitive("chpl_error", "halt reached");
}

def halt(args ...?numArgs) {
  __primitive("chpl_error", "halt reached - "+_tuple2string(args));
}

def _debugWrite(args...?n) {
  def getString(a: ?t) {
    if t == bool(8) || t == bool(16) || t == bool(32) || t == bool(64) ||
       t == int(8) || t == int(16) || t == int(32) || t == int(64) ||
       t == uint(8) || t == uint(16) || t == uint(32) || t == uint(64) ||
       t == real(32) || t == real(64) || t == imag(32) || t == imag(64) ||
       t == complex(64) || t == complex(128) ||
       t == bool || t == string || __primitive("isEnumType", t) then
      return a:string;
    else
      compilerError("Cannot call _debugWrite on value of type ",
                    typeToString(t));
  }
  for param i in 1..n {
    var status = __primitive("fprintf", __primitive("get_stdout"), "%s", getString(args(i)));
    if status < 0 {
      const err = __primitive("get_errno");
      halt("_debugWrite failed with status ", err);
    }
  }
  __primitive("fflush", __primitive("get_stdout"));
}

def _debugWriteln(args...?n) {
  _debugWrite((...args), "\n");
}

def _debugWriteln() {
  _debugWrite("\n");
}

def _ddata.writeThis(f: Writer) {
  halt("cannot write the _ddata class");
}

def format(fmt: string, x:?t) where _isIntegralType(t) || _isFloatType(t) {
  if fmt.substring(1) == "#" {
    var fmt2 = _getoutputformat(fmt);
    if _isImagType(t) then
      return (__primitive("_format", fmt2, _i2r(x))+"i");
    else
      return __primitive("_format", fmt2, x:real);
  } else
    return __primitive("_format", fmt, x);
}

def format(fmt: string, x:?t) where _isComplexType(t) {
  if fmt.substring(1) == "#" {
    var fmt2 = _getoutputformat(fmt);
    return (__primitive("_format", fmt2, x.re)+" + "+ __primitive("_format", fmt2, x.im)+"i");
  } else
    return __primitive("_format", fmt, x);
}

def format(fmt: string, x: ?t) where !(_isIntegralType(t) || _isFloatType(t) || _isComplexType(t)) {
  return __primitive("_format", fmt, x);
}

def _getoutputformat(s: string):string {
  var sn = length(s);
  var afterdot = false;
  var dplaces = 0;
  for i in 1..sn {
    if ((s.substring(i) == '#') & afterdot) then dplaces += 1;
    if (s.substring(i) == '.') then afterdot=true;
  }

  return("%" + sn + "." + dplaces + "f");
}

// class file
//
//  chapel-level implementations of read, write, writeln
//  chapel-level implementations of assert, halt

enum FileAccessMode { read, write };

//
// functions on _file primitive type, the C file pointer type
//
pragma "inline" def =(a: _file, b: _file) return b;
pragma "inline" def ==(a: _file, b: _file) return __primitive("==", a, b);
pragma "inline" def !=(a: _file, b: _file) return __primitive("!=", a, b);

pragma "inline" def _readLitChar(fp: _file, val: string, ignoreWhiteSpace: bool)
  return __primitive("_fscan_literal", fp, val, ignoreWhiteSpace);

const stdin  = new file("stdin", FileAccessMode.read, "/dev", __primitive("get_stdin"));
const stdout = new file("stdout", FileAccessMode.write, "/dev", __primitive("get_stdout"));
const stderr = new file("stderr", FileAccessMode.write, "/dev", __primitive("get_stderr"));

class file: Writer {
  var filename : string = "";
  var mode : FileAccessMode = FileAccessMode.read;
  var path : string = ".";
  var _fp : _file;
  var _lock : sync uint(64);    // for serializing output

  def open() {
    on this {
      if this == stdin || this == stdout || this == stderr then
        halt("***Error: It is not necessary to open \"", filename, "\"***");

      var fullFilename = path + "/" + filename;

      var modestring: string;
      select mode {
        when FileAccessMode.read  do modestring = "r";
        when FileAccessMode.write do modestring = "w";
      }
      _fp = __primitive("fopen", filename, modestring);

      if _fp == __primitive("get_nullfile") {
        const err = __primitive("get_errno");
        halt("***Error: Unable to open \"", fullFilename, "\": ", err, "***");
      }
    }
  }

  def _checkFileStateChangeLegality(state) {
    if (isOpen) {
      halt("Cannot change ", state, " of file ", path, "/", filename,
           " while it is open");
    }
  }

  def filename var : string {
    if setter then
      _checkFileStateChangeLegality("filename");
    return filename;
  }

  def path var : string {
    if setter then
      _checkFileStateChangeLegality("path");
    return path;
  }

  def mode var {
    if setter then
      _checkFileStateChangeLegality("mode");
    return mode;
  }

  def isOpen: bool {
    var openStatus: bool = false;
    if (_fp != __primitive("get_nullfile")) {
      openStatus = true;
    }
    return openStatus;
  }

  def close() {
    on this {
      if (this == stdin || this == stdout || this == stderr) {
        halt("***Error: You may not close \"", filename, "\"***");
      }
      if (_fp == __primitive("get_nullfile")) {
        var fullFilename = path + "/" + filename;
        halt("***Error: Trying to close \"", fullFilename,
             "\" which isn't open***");
      }
      var returnVal: int = __primitive("fclose", _fp);
      if (returnVal < 0) {
        var fullFilename = path + "/" + filename;
        const err = __primitive("get_errno");
        halt("***Error: The close of \"", fullFilename, "\" failed: ", err,
             "***");
      }
      _fp = __primitive("get_nullfile");
    }
  }
}

def file.writeThis(f: Writer) {
  f.write("(filename = ",this.filename);
  f.write(", path = ",this.path);
  f.write(", mode = ",this.mode);
  f.write(")");
}

def file.flush() {
  on this do __primitive("fflush", _fp);
}

def _checkOpen(f: file, isRead: bool) {
  if !f.isOpen {
    var fullFilename:string = f.path + "/" + f.filename;
    if isRead {
      halt("***Error: You must open \"", fullFilename,
           "\" before trying to read from it***");
    } else {
      halt("***Error: You must open \"", fullFilename,
           "\" before trying to write to it***");
    }
  }
}

def file.readln() {
  on this {
    if !isOpen then
      _checkOpen(this, isRead=true);
    __primitive("_readToEndOfLine",_fp);
  }
}

def file.readln(inout list ...?n) {
  read((...list));
  on this {
    __primitive("_readToEndOfLine",_fp);
  }
}

def file.readln(type t) {
  var val: t;
  this.readln(val);
  return val;
}

def file.read(inout first, inout rest ...?n) {
  read(first);
  for param i in 1..n do
    read(rest(i));
}

def file.read(inout val: int) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: int;
  on this {
    x = __primitive("_fscan_int32", _fp);
  }
  val = x;
}

def file.read(inout val: uint) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: uint;
  on this {
    x = __primitive("_fscan_uint32", _fp);
  }
  val = x;
}

def file.read(inout val: real) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: real;
  on this {
    x = __primitive("_fscan_real64", _fp);
  }
  val = x;
}

def file.read(inout val: complex) {
  var realPart: real;
  var imagPart: real;
  var imagI: string;
  var matchingCharWasRead: int;
  var isNeg: bool;

  read(realPart);
  on this {
    matchingCharWasRead = _readLitChar(_fp, "+", true);
  }
  if (matchingCharWasRead != 1) {
    on this {
      matchingCharWasRead = _readLitChar(_fp, "-", true);
    }
    if (matchingCharWasRead != 1) {
      halt("***Error: Incorrect format for complex numbers***");
    }
    isNeg = true;
  }

  read(imagPart);
  on this {
    matchingCharWasRead = _readLitChar(_fp, "i", false);
  }
  if (matchingCharWasRead != 1) {
    halt("***Error: Incorrect format for complex numbers***");
  }

  val.re = realPart;
  if (isNeg) {
    val.im = -imagPart;
  } else {
    val.im = imagPart;
  }
}

def file.read(inout val: string) {
  if !isOpen then
    _checkOpen(this, isRead=true);
  var x: string;
  on this {
    x = __primitive("_fscan_string", _fp);
  }
  val = x;
}

def file.read(inout val: bool) {
  var s: string;
  this.read(s);
  if (s == "true") {
    val = true;
  } else if (s == "false") {
    val = false;
  } else {
    halt("read of bool value that is neither true nor false");
  }
}

def file.read(type t) {
  var val: t;
  this.read(val);
  return val;
}

def string.writeThis(f: Writer) {
  f.writeIt(this);
}

def numeric.writeThis(f: Writer) {
  f.writeIt(this:string);
}

def enumerated.writeThis(f: Writer) {
  f.writeIt(this:string);
}

/*
def bool.writeThis(f: Writer) {
  f.writeIt(this:string);
}
*/

def file.writeIt(s: string) {
  if !isOpen then
    _checkOpen(this, isRead = false);
  if mode != FileAccessMode.write then
    halt("***Error: ", path, "/", filename, " not open for writing***");
  var status = __primitive("fprintf", _fp, "%s", s);
  if status < 0 {
    const err = __primitive("get_errno");
    halt("***Error: Write failed: ", err, "***");
  }
}

class StringClass: Writer {
  var s: string;
  def writeIt(s: string) { this.s += s; }
}

pragma "ref this" pragma "dont disable remote value forwarding"
def string.write(args ...?n) {
  var sc = new StringClass(this);
  sc.write((...args));
  this = sc.s;
  delete sc;
}

def file.lockWrite() {
  var me: uint(64) = __primitive("thread_id");
  if _lock.isFull then
    if _lock.readXX() == me then
      return false;
  _lock = me;
  return true;
}

def file.unlockWrite() {
  _lock.reset();
}

def isBoolean(b: bool) param return true;
def isBoolean(b) param return false;

class Writer {
  def writeIt(s: string) { }
  def lockWrite() return false;
  def unlockWrite() { }
  def write(args ...?n) {
    def isNilObject(val) {
      def helper(o: object) return o == nil;
      def helper(o)         return false;
      return helper(val);
    }

    on this {
      var need_release: bool;
      need_release = lockWrite();
      for param i in 1..n do
        if isNilObject(args(i)) then
          "nil".writeThis(this);
        else if isBoolean(args(i)) then
          // This isBoolean case is a workaround for a problem with
          // resolving a reference to a bool to one of the bool.writeThis
          // methods.
          if args(i) then
            "true".writeThis(this);
          else
            "false".writeThis(this);
        else
          args(i).writeThis(this);
      if need_release then
        unlockWrite();
    }
  }
  def writeln(args ...?n) {
    write((...args), "\n");
  }
  def writeln() {
    write("\n");
  }
}

def write(args ...?n) {
  stdout.write((...args));
  stdout.flush();
}

def writeln(args ...?n) {
  stdout.writeln((...args));
}

def writeln() {
  stdout.writeln();
}

def read(inout args ...?n) {
  stdin.read((...args));
}

def readln(inout args ...?n) {
  stdin.readln((...args));
}

def readln() {
  stdin.readln();
}

def readln(type t) {
  return stdin.readln(t);
}

def read(type t)
  return stdin.read(t);

def file.readln(type t ...?numTypes) where numTypes > 1 {
  var tupleVal: t;
  for param i in 1..numTypes-1 do
    tupleVal(i) = this.read(t(i));
  tupleVal(numTypes) = this.readln(t(numTypes));
  return tupleVal;
}

def file.read(type t ...?numTypes) where numTypes > 1 {
  var tupleVal: t;
  for param i in 1..numTypes do
    tupleVal(i) = this.read(t(i));
  return tupleVal;
}

def readln(type t ...?numTypes) where numTypes > 1
  return stdin.readln((...t));

def read(type t ...?numTypes) where numTypes > 1
  return stdin.read((...t));

def _tuple2string(t) {
  var s: string;
  for param i in 1..t.size do
    s.write(t(i));
  return s;
}

def assert(test: bool) {
  if !test then
    __primitive("chpl_error", "assert failed");
}

def assert(test: bool, args ...?numArgs) {
  if !test then
    __primitive("chpl_error", "assert failed - "+_tuple2string(args));
}

def halt() {
  __primitive("chpl_error", "halt reached");
}

def halt(args ...?numArgs) {
  __primitive("chpl_error", "halt reached - "+_tuple2string(args));
}

def _debugWrite(args: string ...?n) {
  for param i in 1..n {
    var status = __primitive("fprintf", __primitive("get_stdout"), "%s", args(i));
    if status < 0 {
      const err = __primitive("get_errno");
      halt("_debugWrite failed with status ", err);
    }
  }
  __primitive("fflush", __primitive("get_stdout"));
}

def _debugWriteln(args: string ...?n) {
  _debugWrite((...args), "\n");
}

def _debugWriteln() {
  _debugWrite("\n");
}

def _ddata.writeThis(f: Writer) {
  halt("cannot write the _ddata class");
}

def format(fmt: string, x:?t) where _isIntegralType(t) || _isFloatType(t) {
  if fmt.substring(1) == "#" {
    var fmt2 = _getoutputformat(fmt);
    if _isImagType(t) then
      return (__primitive("_format", fmt2, _i2r(x))+"i");
    else
      return __primitive("_format", fmt2, x:real);
  } else
    return __primitive("_format", fmt, x);
}

def format(fmt: string, x:?t) where _isComplexType(t) {
  if fmt.substring(1) == "#" {
    var fmt2 = _getoutputformat(fmt);
    return (__primitive("_format", fmt2, x.re)+" + "+ __primitive("_format", fmt2, x.im)+"i");
  } else
    return __primitive("_format", fmt, x);
}

def format(fmt: string, x: ?t) where !(_isIntegralType(t) || _isFloatType(t) || _isComplexType(t)) {
  return __primitive("_format", fmt, x);
}

def _getoutputformat(s: string):string {
  var sn = length(s);
  var afterdot = false;
  var dplaces = 0;
  for i in 1..sn {
    if ((s.substring(i) == '#') & afterdot) then dplaces += 1;
    if (s.substring(i) == '.') then afterdot=true;
  }

  return("%" + sn + "." + dplaces + "f");
}