Handle the case where the number of user specified task private variables is greater than the number of locales.

This is a convenient scenario for debugging.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20536 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  +==========================================================================+

{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      proc maxweight(s) {
	var mymax = 0;
	for w in G.edge_weight[s] do
	  if w > mymax then mymax = w;
	return mymax;
      }

      const heaviest_edge_weight = max reduce [s in G.vertices] maxweight(s);

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in G.NeighborPairs (s) do

	  if w == heaviest_edge_weight then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ", heaviest_edge_weight);
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in G.NeighborPairs (s) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
                                Heavy_Edge_List     : domain,
                                Heavy_Edge_Subgraph : [],
                                in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance             : [vertex_domain] atomic int;
        forall m in min_distance do m.write(-1);

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance ( y ).write(0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do {

              if min_distance(w).compareExchangeStrong(-1, path_length) then {
                Next_Level.add (w);
                Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
	      }

	      if min_distance(w).read() == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  use BlockDist;
  // For task-private temporary variables
  config const defaultNumTPVs = 16;
  config var numTPVs = min(defaultNumTPVs, numLocales);
  // Would be nice to use PriavteDist, but aliasing is not supported (yet)
  const PrivateSpace = {LocaleSpace} dmapped Block(boundingBox={LocaleSpace});

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
                                            Between_Cent : [] real,
                                            out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Considering using a dense 1-d array instead.  This would
      // complicate the Level_Set implementation a little, but would
      // probably be more efficient.
      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] atomic real;
      var Sum_Min_Dist$ : atomic real;

      //
      // Throughout kernel 4, we use distributed arrays that are
      // private to each iteration of the outer loop.  Because set-up
      // and clean-up of such variables is very expensive, it is
      // desirable to do it at most only once per task versus once
      // per iteration, as a task executes multiple iterations of a
      // forall loop.
      //
      // To avoid this per-iteration overhead, we have implemented
      // an optimization that make certain variables private to each
      // task.  These variables are reinitialized each iteration of
      // the loop, but not reallocated, reprivatized, etc.
      //
      // Note that the Chapel group and collaborators are currently in
      // the process of bringing the idea of task private variable to
      // the language.
      //

      numTPVs = min(numTPVs, starting_vertices.size);
      const TPVSpace = {0..#numTPVs};
      var TPVLocales: [TPVSpace] locale;

      if numTPVs <= numLocales {
        // Here we set up the locales array for our task private temporary
        // variables.  We use _computChunkStartEnd() to distribute the
        // variables approximately evenly across the 1d locales array.
        for t in TPVSpace {
          TPVLocales[t] = Locales[_computeChunkStartEnd(numLocales,
                                                        numTPVs, t+1)[1]-1];
        }
      } else {
        for t in TPVSpace {
          TPVLocales[t] = Locales[((t-1)/numTPVs)/numLocales];
        }
      }
      const TPVLocaleSpace = {TPVSpace} dmapped Block(boundingBox={TPVSpace},
                                                      targetLocales=TPVLocales);

      // There will be numTPVs copies of the temps, thus throttling the
      // number of starting vertices being considered simultaneously.
      var TPV: [TPVLocaleSpace] taskPrivateData(domain(index(vertex_domain)),
                                                vertex_domain.type);

      // Initialize
      coforall (t, i) in zip(TPVLocaleSpace, TPVSpace) do on TPVLocaleSpace.dist.idxToLocale(t) {
          TPV[i] = new taskPrivateData(domain(index(vertex_domain)), vertex_domain);
          var tpv = TPV[i];
          forall v in vertex_domain do
            tpv.BCaux[v].children_list.nd = {1..G.n_Neighbors[v]};
          for loc in Locales do on loc {
            tpv.Active_Level[here.id] = new Level_Set (Sparse_Vertex_List);
            tpv.Active_Level[here.id].previous = nil;
            tpv.Active_Level[here.id].next = new Level_Set (Sparse_Vertex_List);;
            tpv.Active_Level[here.id].next.previous = tpv.Active_Level[here.id];
            tpv.Active_Level[here.id].next.next = nil;
          }
      }
      var TPVM: TPVManager(TPV.type) = new TPVManager(TPV);

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      forall s in starting_vertices do on vertex_domain.dist.idxToLocale(s) {

        const shere = here.id;

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

        // Initialize task private variables
        const tid = TPVM.gettid();
        const tpv = TPVM.getTPV(tid);
        var BCaux => tpv.BCaux;
        pragma "dont disable remote value forwarding"
        inline proc f1(BCaux, v) {
          BCaux[v].path_count$.write(0.0);
        }
        forall v in vertex_domain do {
          BCaux[v].depend = 0.0;
          BCaux[v].min_distance.write(-1);
          f1(BCaux, v);
          BCaux[v].children_list.child_count.write(0);
        }
	var Lcl_Sum_Min_Dist: atomic real;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

        //
        // Used to check termination of the forward pass
        //
        // We could use a task-private reduction variable here.  This
        // is yet another concept that the Chapel group is planning to
        // implement.
        //

        var remaining = true;

        //
        // Each locale will have its own level sets.  A locale's level set
        // will only contain nodes that are physically allocated on that
        // particular locale.
        //
        var Active_Level => tpv.Active_Level;
        pragma "dont disable remote value forwarding"
        inline proc f2(BCaux, s) {
          BCaux[s].path_count$.write(1.0);
        }

        const barrier = tpv.barrier;

        coforall loc in Locales do on loc {
          Active_Level[here.id].Members.clear();
          Active_Level[here.id].next.Members.clear();
          if vertex_domain.dist.idxToLocale(s) == here {
            // Establish the initial level sets for the breadth-first
            // traversal from s
            Active_Level[here.id].Members.add(s);
            BCaux[s].min_distance.write(0);
            f2(BCaux, s);
          }
          barrier.barrier();

          var current_distance : int = 0;

          while remaining do {
	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;

            // The Chapel compiler is still a bit conservative when it
            // comes to forwarding read-only variables to remote
            // locales, so we make a const copy here to insure it is
            // forwarded to the remote locale in the following
            // coforall loop.
            const current_distance_c = current_distance;
            pragma "dont disable remote value forwarding"
            inline proc f3(BCaux, v, u, current_distance_c, Active_Level, Lcl_Sum_Min_Dist) {

                  // --------------------------------------------
                  // add any unmarked neighbors to the next level
                  // --------------------------------------------

                  if  BCaux[v].min_distance.compareExchangeStrong(-1, current_distance_c) {
                    Active_Level[here.id].next.Members.add (v);
                    if VALIDATE_BC then
                      Lcl_Sum_Min_Dist.add(current_distance_c);
                  }

                  // ------------------------------------------------
                  // only neighbors of  u  that are in the next level
                  // are on shortest paths from s through v.  Some
                  // task will have set  min_distance (v) by the
                  // time this code is reached, whether  v  lies in
                  // the previous, the current or the next level.
                  // ------------------------------------------------

                  if BCaux[v].min_distance.read() == current_distance_c {
                    BCaux[v].path_count$.add(BCaux[u].path_count$.read());
                    //f3(BCaux, v, u);
                    BCaux[u].children_list.add_child (v);
                  }

            }

            forall u in Active_Level[here.id].Members do {
              forall v in G.FilteredNeighbors(u) do on vertex_domain.dist.idxToLocale(v) {
                      f3(BCaux, v, u, current_distance_c, Active_Level, Lcl_Sum_Min_Dist);
                }
            };

            // This barrier is needed to insure all updates to the next
            // level are completed before updating to use the next level

            // do some work while we wait
            // barrier.notify(); // This is expensive without network atomics
            //  for now, just do a normal barrier

            if Active_Level[here.id].next.next == nil {
              Active_Level[here.id].next.next = new Level_Set (Sparse_Vertex_List);
              Active_Level[here.id].next.next.previous = Active_Level[here.id].next;
              Active_Level[here.id].next.next.next = nil;
            } else {
              Active_Level[here.id].next.next.Members.clear();
            }
            // barrier.wait(); // ditto
            barrier.barrier();
            if here.id==shere {
              remaining = false;
            }

            barrier.barrier();
            Active_Level[here.id] = Active_Level[here.id].next;
            if Active_Level[here.id].Members.numIndices:bool then
              remaining = true;

            barrier.barrier();

          };  // end forward pass

          if VALIDATE_BC then
            if here.id==0 then
              Sum_Min_Dist$.add(Lcl_Sum_Min_Dist);

          // -------------------------------------------------------------
          // compute the dependencies recursively, traversing the vertices
          // of the graph in non-increasing order of distance (reverse
          // ordering from the initial traversal)
          // -------------------------------------------------------------

          const graph_diameter = current_distance - 1;

          if DEBUG_KERNEL4 then
            if here.id==0 then
              writeln ( " graph diameter from starting node ", s,
                        "  is ", graph_diameter );

          pragma "dont disable remote value forwarding"
          inline proc f4(BCaux, Between_Cent$, u) {
            BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
              ( BCaux[u].path_count$.read() /
                BCaux[v].path_count$.read() )      *
              ( 1.0 + BCaux[v].depend );
            Between_Cent$(u).add(BCaux[u].depend);
          }

          // back up to last level
          var curr_Level =  Active_Level[here.id].previous;

          for current_distance in 2 .. graph_diameter by -1 {
            curr_Level = curr_Level.previous;

            for u in curr_Level.Members do on vertex_domain.dist.idxToLocale(u) {
                f4(BCaux, Between_Cent$, u);
            }

            barrier.barrier();
          }
        }

        TPVM.releaseTPV(tid);

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	const n_edges          = G.num_edges;
	const N_VERTICES       = vertex_domain.numIndices;
	const N_START_VERTICES = if starting_vertices == G.vertices
                                 then N_VERTICES
                                      - + reduce [v in vertex_domain]
                                        (G.n_Neighbors (v) == 0)
                                 else starting_vertices.numIndices;
	const TEPS             = 7.0 * N_VERTICES * N_START_VERTICES / K4_time;
	const Adjusted_TEPS    = n_edges * N_START_VERTICES / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
        Sum_Min_Dist = Sum_Min_Dist$.read();

      Between_Cent = Between_Cent$.read();

      if DELETE_KERNEL4_DS {
        coforall t in TPVSpace do on t {
          var tpv = TPV[t];
          delete tpv.barrier;
          var al = tpv.Active_Level;
          coforall loc in Locales do on loc {
            var level = al[here.id];
            while level != nil {
                var l2 = level.next;
                delete level;
                level = l2;
            }
          }
          delete TPV[t];
        }
        delete TPVM;
      }

    } // end of Brandes' betweenness centrality calculation

  //
  // Addition support data structures for kernel 4
  //

  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
    var next : Level_Set (Sparse_Vertex_List);
  }

  //
  // Data structure to save Children lists between the forward
  //  and backwards passes
  //
  record child_struct {
    type vertex;
    var nd: domain(1);
    var Row_Children: [nd] vertex;
    var child_count: atomic int;

    // This function should only be called using unique vertices
    proc add_child ( new_child: vertex ) {
      var c = child_count.fetchAdd(1)+1;
      Row_Children[c] = new_child;
    }
  }

  //
  // Implementation of task-private variables for kernel 4
  //
  record taskPrivateArrayData {
    type vertex;
    var min_distance  : atomic_int64; // used only on home locale
    var path_count$   : atomic real;
    var depend        : real;
    var children_list : child_struct(vertex);
  }

  class taskPrivateData {
    type Sparse_Vertex_List;
    const vertex_domain;
    var used  : atomic bool;
    var BCaux : [vertex_domain] taskPrivateArrayData(index(vertex_domain));
    var barrier = new Barrier(numLocales);
    var Active_Level : [PrivateSpace] Level_Set (Sparse_Vertex_List);
  }

  // This is a simple class that hands out task private variables from the
  // array of task private variables, TPV.
  class TPVManager {
    const TPV;
    var currTPV: atomic int;
    proc gettid() {
      const tid = this.currTPV.fetchAdd(1)%numTPVs;
      on this.TPV[tid] do
        while this.TPV[tid].used.testAndSet() do chpl_task_yield();
      return tid;
    }
    proc getTPV(tid) {
      return this.TPV[tid];
    }
    proc releaseTPV(tid) {
      this.TPV[tid].used.clear();
    }
  }

  //
  // reusable barrier implementation
  //
  class Barrier {
    param reusable = true;
    var n: int;
    var count: atomic int;
    var tasksFinished: [PrivateSpace] atomic bool;

    proc Barrier(_n: int) {
      reset(_n);
    }

    inline proc reset(_n: int) {
      on this {
        n = _n;
        count.write(n);
        for loc in tasksFinished {
          loc.write(false);
        }
      }
    }

    inline proc barrier() {
      const myc = count.fetchSub(1);
      if myc <= 1 {
        if tasksFinished[here.id].read() {
          halt("too many callers to barrier()");
        }
        for loc in tasksFinished {
          loc.write(true);
        }
        if reusable {
          count.waitFor(n-1);
          count.add(1);
          for loc in tasksFinished {
            loc.clear();
          }
        }
      } else {
        tasksFinished[here.id].waitFor(true);
        if reusable {
          count.add(1);
          tasksFinished[here.id].waitFor(false);
        }
      }
    }

    inline proc notify() {
      const myc = count.fetchSub(1);
      if myc <= 1 {
        if tasksFinished[here.id].read() {
          halt("too many callers to notify()");
        }
        for loc in tasksFinished {
          loc.write(true);
        }
      }
   }

    inline proc wait() {
      tasksFinished[here.id].waitFor(true);
      if reusable {
        var sum = count.fetchAdd(1);
        if sum == n-1 then tasksFinished.clear();
        else tasksFinished[here.id].waitFor(false);
      }
    }

    inline proc try() {
      return tasksFinished[here.id].read();
    }
  }
}

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  +==========================================================================+

{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      proc maxweight(s) {
	var mymax = 0;
	for w in G.edge_weight[s] do
	  if w > mymax then mymax = w;
	return mymax;
      }

      const heaviest_edge_weight = max reduce [s in G.vertices] maxweight(s);

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in G.NeighborPairs (s) do

	  if w == heaviest_edge_weight then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ", heaviest_edge_weight);
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in G.NeighborPairs (s) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
                                Heavy_Edge_List     : domain,
                                Heavy_Edge_Subgraph : [],
                                in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance             : [vertex_domain] atomic int;
        forall m in min_distance do m.write(-1);

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance ( y ).write(0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do {

              if min_distance(w).compareExchangeStrong(-1, path_length) then {
                Next_Level.add (w);
                Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
	      }

	      if min_distance(w).read() == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  use BlockDist;
  // For task-private temporary variables
  config const defaultNumTPVs = 16;
  config var numTPVs = min(defaultNumTPVs, numLocales);
  // Would be nice to use PriavteDist, but aliasing is not supported (yet)
  const PrivateSpace = {LocaleSpace} dmapped Block(boundingBox={LocaleSpace});

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
                                            Between_Cent : [] real,
                                            out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Considering using a dense 1-d array instead.  This would
      // complicate the Level_Set implementation a little, but would
      // probably be more efficient.
      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] atomic real;
      var Sum_Min_Dist$ : atomic real;

      //
      // Throughout kernel 4, we use distributed arrays that are
      // private to each iteration of the outer loop.  Because set-up
      // and clean-up of such variables is very expensive, it is
      // desirable to do it at most only once per task versus once
      // per iteration, as a task executes multiple iterations of a
      // forall loop.
      //
      // To avoid this per-iteration overhead, we have implemented
      // an optimization that make certain variables private to each
      // task.  These variables are reinitialized each iteration of
      // the loop, but not reallocated, reprivatized, etc.
      //
      // Note that the Chapel group and collaborators are currently in
      // the process of bringing the idea of task private variable to
      // the language.
      //

      numTPVs = min(numTPVs, starting_vertices.size);
      const TPVSpace = {0..#numTPVs};
      var TPVLocales: [TPVSpace] locale;

      // Here we set up the locales array for our task private temporary
      // variables.  We use _computChunkStartEnd() to distribute the
      // variables approximately evenly across the 1d locales array.
      for t in TPVSpace {
        TPVLocales[t] = Locales[_computeChunkStartEnd(numLocales,
                                                      numTPVs, t+1)[1]-1];
      }
      const TPVLocaleSpace = {TPVSpace} dmapped Block(boundingBox={TPVSpace},
                                                      targetLocales=TPVLocales);

      // There will be numTPVs copies of the temps, thus throttling the
      // number of starting vertices being considered simultaneously.
      var TPV: [TPVLocaleSpace] taskPrivateData(domain(index(vertex_domain)),
                                                vertex_domain.type);

      // Initialize
      coforall (t, i) in zip(TPVLocaleSpace, TPVSpace) do on TPVLocaleSpace.dist.idxToLocale(t) {
          TPV[i] = new taskPrivateData(domain(index(vertex_domain)), vertex_domain);
          var tpv = TPV[i];
          forall v in vertex_domain do
            tpv.BCaux[v].children_list.nd = {1..G.n_Neighbors[v]};
          for loc in Locales do on loc {
            tpv.Active_Level[here.id] = new Level_Set (Sparse_Vertex_List);
            tpv.Active_Level[here.id].previous = nil;
            tpv.Active_Level[here.id].next = new Level_Set (Sparse_Vertex_List);;
            tpv.Active_Level[here.id].next.previous = tpv.Active_Level[here.id];
            tpv.Active_Level[here.id].next.next = nil;
          }
      }
      var TPVM: TPVManager(TPV.type) = new TPVManager(TPV);

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      forall s in starting_vertices do on vertex_domain.dist.idxToLocale(s) {

        const shere = here.id;

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

        // Initialize task private variables
        const tid = TPVM.gettid();
        const tpv = TPVM.getTPV(tid);
        var BCaux => tpv.BCaux;
        pragma "dont disable remote value forwarding"
        inline proc f1(BCaux, v) {
          BCaux[v].path_count$.write(0.0);
        }
        forall v in vertex_domain do {
          BCaux[v].depend = 0.0;
          BCaux[v].min_distance.write(-1);
          f1(BCaux, v);
          BCaux[v].children_list.child_count.write(0);
        }
	var Lcl_Sum_Min_Dist: atomic real;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

        //
        // Used to check termination of the forward pass
        //
        // We could use a task-private reduction variable here.  This
        // is yet another concept that the Chapel group is planning to
        // implement.
        //

        var remaining = true;

        //
        // Each locale will have its own level sets.  A locale's level set
        // will only contain nodes that are physically allocated on that
        // particular locale.
        //
        var Active_Level => tpv.Active_Level;
        pragma "dont disable remote value forwarding"
        inline proc f2(BCaux, s) {
          BCaux[s].path_count$.write(1.0);
        }

        const barrier = tpv.barrier;

        coforall loc in Locales do on loc {
          Active_Level[here.id].Members.clear();
          Active_Level[here.id].next.Members.clear();
          if vertex_domain.dist.idxToLocale(s) == here {
            // Establish the initial level sets for the breadth-first
            // traversal from s
            Active_Level[here.id].Members.add(s);
            BCaux[s].min_distance.write(0);
            f2(BCaux, s);
          }
          barrier.barrier();

          var current_distance : int = 0;

          while remaining do {
	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;

            // The Chapel compiler is still a bit conservative when it
            // comes to forwarding read-only variables to remote
            // locales, so we make a const copy here to insure it is
            // forwarded to the remote locale in the following
            // coforall loop.
            const current_distance_c = current_distance;
            pragma "dont disable remote value forwarding"
            inline proc f3(BCaux, v, u, current_distance_c, Active_Level, Lcl_Sum_Min_Dist) {

                  // --------------------------------------------
                  // add any unmarked neighbors to the next level
                  // --------------------------------------------

                  if  BCaux[v].min_distance.compareExchangeStrong(-1, current_distance_c) {
                    Active_Level[here.id].next.Members.add (v);
                    if VALIDATE_BC then
                      Lcl_Sum_Min_Dist.add(current_distance_c);
                  }

                  // ------------------------------------------------
                  // only neighbors of  u  that are in the next level
                  // are on shortest paths from s through v.  Some
                  // task will have set  min_distance (v) by the
                  // time this code is reached, whether  v  lies in
                  // the previous, the current or the next level.
                  // ------------------------------------------------

                  if BCaux[v].min_distance.read() == current_distance_c {
                    BCaux[v].path_count$.add(BCaux[u].path_count$.read());
                    //f3(BCaux, v, u);
                    BCaux[u].children_list.add_child (v);
                  }

            }

            forall u in Active_Level[here.id].Members do {
              forall v in G.FilteredNeighbors(u) do on vertex_domain.dist.idxToLocale(v) {
                      f3(BCaux, v, u, current_distance_c, Active_Level, Lcl_Sum_Min_Dist);
                }
            };

            // This barrier is needed to insure all updates to the next
            // level are completed before updating to use the next level

            // do some work while we wait
            // barrier.notify(); // This is expensive without network atomics
            //  for now, just do a normal barrier

            if Active_Level[here.id].next.next == nil {
              Active_Level[here.id].next.next = new Level_Set (Sparse_Vertex_List);
              Active_Level[here.id].next.next.previous = Active_Level[here.id].next;
              Active_Level[here.id].next.next.next = nil;
            } else {
              Active_Level[here.id].next.next.Members.clear();
            }
            // barrier.wait(); // ditto
            barrier.barrier();
            if here.id==shere {
              remaining = false;
            }

            barrier.barrier();
            Active_Level[here.id] = Active_Level[here.id].next;
            if Active_Level[here.id].Members.numIndices:bool then
              remaining = true;

            barrier.barrier();

          };  // end forward pass

          if VALIDATE_BC then
            if here.id==0 then
              Sum_Min_Dist$.add(Lcl_Sum_Min_Dist);

          // -------------------------------------------------------------
          // compute the dependencies recursively, traversing the vertices
          // of the graph in non-increasing order of distance (reverse
          // ordering from the initial traversal)
          // -------------------------------------------------------------

          const graph_diameter = current_distance - 1;

          if DEBUG_KERNEL4 then
            if here.id==0 then
              writeln ( " graph diameter from starting node ", s,
                        "  is ", graph_diameter );

          pragma "dont disable remote value forwarding"
          inline proc f4(BCaux, Between_Cent$, u) {
            BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
              ( BCaux[u].path_count$.read() /
                BCaux[v].path_count$.read() )      *
              ( 1.0 + BCaux[v].depend );
            Between_Cent$(u).add(BCaux[u].depend);
          }

          // back up to last level
          var curr_Level =  Active_Level[here.id].previous;

          for current_distance in 2 .. graph_diameter by -1 {
            curr_Level = curr_Level.previous;

            for u in curr_Level.Members do on vertex_domain.dist.idxToLocale(u) {
                f4(BCaux, Between_Cent$, u);
            }

            barrier.barrier();
          }
        }

        TPVM.releaseTPV(tid);

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	const n_edges          = G.num_edges;
	const N_VERTICES       = vertex_domain.numIndices;
	const N_START_VERTICES = if starting_vertices == G.vertices
                                 then N_VERTICES
                                      - + reduce [v in vertex_domain]
                                        (G.n_Neighbors (v) == 0)
                                 else starting_vertices.numIndices;
	const TEPS             = 7.0 * N_VERTICES * N_START_VERTICES / K4_time;
	const Adjusted_TEPS    = n_edges * N_START_VERTICES / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
        Sum_Min_Dist = Sum_Min_Dist$.read();

      Between_Cent = Between_Cent$.read();

      if DELETE_KERNEL4_DS {
        coforall t in TPVSpace do on t {
          var tpv = TPV[t];
          delete tpv.barrier;
          var al = tpv.Active_Level;
          coforall loc in Locales do on loc {
            var level = al[here.id];
            while level != nil {
                var l2 = level.next;
                delete level;
                level = l2;
            }
          }
          delete TPV[t];
        }
        delete TPVM;
      }

    } // end of Brandes' betweenness centrality calculation

  //
  // Addition support data structures for kernel 4
  //

  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
    var next : Level_Set (Sparse_Vertex_List);
  }

  //
  // Data structure to save Children lists between the forward
  //  and backwards passes
  //
  record child_struct {
    type vertex;
    var nd: domain(1);
    var Row_Children: [nd] vertex;
    var child_count: atomic int;

    // This function should only be called using unique vertices
    proc add_child ( new_child: vertex ) {
      var c = child_count.fetchAdd(1)+1;
      Row_Children[c] = new_child;
    }
  }

  //
  // Implementation of task-private variables for kernel 4
  //
  record taskPrivateArrayData {
    type vertex;
    var min_distance  : atomic_int64; // used only on home locale
    var path_count$   : atomic real;
    var depend        : real;
    var children_list : child_struct(vertex);
  }

  class taskPrivateData {
    type Sparse_Vertex_List;
    const vertex_domain;
    var used  : atomic bool;
    var BCaux : [vertex_domain] taskPrivateArrayData(index(vertex_domain));
    var barrier = new Barrier(numLocales);
    var Active_Level : [PrivateSpace] Level_Set (Sparse_Vertex_List);
  }

  // This is a simple class that hands out task private variables from the
  // array of task private variables, TPV.
  class TPVManager {
    const TPV;
    var currTPV: atomic int;
    proc gettid() {
      const tid = this.currTPV.fetchAdd(1)%numTPVs;
      on this.TPV[tid] do
        while this.TPV[tid].used.testAndSet() do chpl_task_yield();
      return tid;
    }
    proc getTPV(tid) {
      return this.TPV[tid];
    }
    proc releaseTPV(tid) {
      this.TPV[tid].used.clear();
    }
  }

  //
  // reusable barrier implementation
  //
  class Barrier {
    param reusable = true;
    var n: int;
    var count: atomic int;
    var tasksFinished: [PrivateSpace] atomic bool;

    proc Barrier(_n: int) {
      reset(_n);
    }

    inline proc reset(_n: int) {
      on this {
        n = _n;
        count.write(n);
        for loc in tasksFinished {
          loc.write(false);
        }
      }
    }

    inline proc barrier() {
      const myc = count.fetchSub(1);
      if myc <= 1 {
        if tasksFinished[here.id].read() {
          halt("too many callers to barrier()");
        }
        for loc in tasksFinished {
          loc.write(true);
        }
        if reusable {
          count.waitFor(n-1);
          count.add(1);
          for loc in tasksFinished {
            loc.clear();
          }
        }
      } else {
        tasksFinished[here.id].waitFor(true);
        if reusable {
          count.add(1);
          tasksFinished[here.id].waitFor(false);
        }
      }
    }

    inline proc notify() {
      const myc = count.fetchSub(1);
      if myc <= 1 {
        if tasksFinished[here.id].read() {
          halt("too many callers to notify()");
        }
        for loc in tasksFinished {
          loc.write(true);
        }
      }
   }

    inline proc wait() {
      tasksFinished[here.id].waitFor(true);
      if reusable {
        var sum = count.fetchAdd(1);
        if sum == n-1 then tasksFinished.clear();
        else tasksFinished[here.id].waitFor(false);
      }
    }

    inline proc try() {
      return tasksFinished[here.id].read();
    }
  }
}


Improved synchonization for fragmented stream's cooperative printing

Inspired by David's example earlier today (and spurred on
by Steve's disgust that I was spin-waiting on a readXX()),
optimized the synchronization used to take turns printing
out an array by using an array of sync vars whose fullness/
emptiness indicates whose turn it is rather than a single
sync var whose value indicates whose turn it is.

For a 100 element problem size on 4 locales (on my 2 core
processor) with --printArrays on, this reduced the runtime
from 65 seconds to 1 second.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13555 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config const debugSync = false;

var myTurn$: [LocalesDomain] sync bool;
myTurn$(0) = true;

def waitForTurn() {
  if (debugSync) then _debugWriteln(localeID(), " waiting for turn");
  const tmp = myTurn$(localeID()); // BUG: tmp should not be required
  if (debugSync) then _debugWriteln(localeID(), " got for turn");
}

def passTurn() {
  if (debugSync) then _debugWriteln(localeID(), " passing turn");
  myTurn$((localeID() + 1) % numLocales) = true;
  if (debugSync) then _debugWriteln(localeID(), " passed turn");
  if (localeID() == 0) then {
    if (debugSync) then _debugWriteln(localeID(), " waiting for turn to come back around");
    const tmp = myTurn$(localeID()); // BUG: tmp should not be required
    if (debugSync) then _debugWriteln(localeID(), " resetting self");
    myTurn$(localeID()) = true;
    if (debugSync) then _debugWriteln(localeID(), " exiting passTurn");
  }
}

def writelnFragArray(startStr, X, endStr) {
  if (localeID() == 0) {
    write(startStr);
  }
  waitForTurn();
  if (localeID() != 0 && X.numElements != 0) {
    write(" ");
  }
  write(X);
  passTurn();
  if (localeID() == 0) {
    writeln(endStr);
  }
}

def BlockPartition(D, myBlock, numBlocks) {
  const lo = D.dim(1)._low;
  const hi = D.dim(1)._high;
  const numelems = hi - lo + 1;
  const nbAsIndexType = numBlocks:index(D);
  const myBlockAsIndexType:index(D) = myBlock;
  const blo = lo + (myBlockAsIndexType*numelems) / nbAsIndexType;
  const bhi = lo + (((myBlockAsIndexType+1)*numelems) / nbAsIndexType) - 1;
  return [blo..bhi];
}

config const debugSync = false;

var myTurn$: sync int = 0;
var barrierCount$: sync int = 0;
var barrierCount2$: sync int = 0;
var barrierDone$: sync bool = false;

def waitForTurn() {
  if (debugSync) then _debugWriteln(localeID(), " waiting for turn");
  while (myTurn$.readXX() != localeID()) { }
  if (debugSync) then _debugWriteln(localeID(), " got for turn");
  myTurn$;
}

def passTurn() {
  if (debugSync) then _debugWriteln(localeID(), " passing turn");
  myTurn$ = (localeID() + 1) % numLocales;
  if (debugSync) then _debugWriteln(localeID(), " passed turn");
  if (localeID() == 0) then while (myTurn$.readXX() != localeID()) { }
}

def barrier() {
  // BLC: I'm not confident this is correct
  if (debugSync) then _debugWriteln(localeID(), " entering barrier");
  barrierCount$ += 1;
  if (debugSync) then _debugWriteln(localeID(), " incremented barrier");
  if (localeID() == 0) {
    if (debugSync) then _debugWriteln(localeID(), " waiting on 0");
    while (barrierCount$.readXX() != numLocales) { }
    if (debugSync) then _debugWriteln(localeID(), " resetting vars");
    barrierCount2$.writeFF(0);
    barrierCount$.writeFF(0);
    barrierDone$.writeFF(true);
  }
  if (debugSync) then _debugWriteln(localeID(), " waiting for things to be done");
  while (barrierDone$.readXX() != true) { }
  barrierCount2$ -= 1;
  if (localeID() == 0) {
    while (barrierCount2$.readXX() != -numLocales) { }
    barrierCount2$.writeFF(0);
    barrierDone$.writeFF(false);
  }
}

def writelnFragArray(startStr, X, endStr) {
  if (localeID() == 0) {
    write(startStr);
  }
  waitForTurn();
  if (localeID() != 0 && X.numElements != 0) {
    write(" ");
  }
  write(X);
  passTurn();
  if (localeID() == 0) {
    writeln(endStr);
  }
}

def BlockPartition(D, myBlock, numBlocks) {
  const lo = D.dim(1)._low;
  const hi = D.dim(1)._high;
  const numelems = hi - lo + 1;
  const nbAsIndexType = numBlocks:index(D);
  const myBlockAsIndexType:index(D) = myBlock;
  const blo = lo + (myBlockAsIndexType*numelems) / nbAsIndexType;
  const bhi = lo + (((myBlockAsIndexType+1)*numelems) / nbAsIndexType) - 1;
  return [blo..bhi];
}


Oops, forgot copyright

Oops, forgot copyright

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

proc assert(test: bool) {
  if !test then
    __primitive("chpl_error", "assert failed");
}

proc assert(test: bool, args ...?numArgs) {
  if !test {
    var tmpstring: c_string;
    tmpstring.write((...args));
    __primitive("chpl_error", "assert failed - " + tmpstring);
  }
}


Better parallelization for regexdna

Better parallelization for regexdna

/* The Computer Language Benchmarks Game
 * http://benchmarksgame.alioth.debian.org/
 *
 * contributed by Ben Harshbarger
 * based on C++ RE2 implementation by Alexey Zolotov
 */

proc main() {
  var variants = [
    "agggtaaa|tttaccct",
    "[cgt]gggtaaa|tttaccc[acg]",
    "a[act]ggtaaa|tttacc[agt]t",
    "ag[act]gtaaa|tttac[agt]ct",
    "agg[act]taaa|ttta[agt]cct",
    "aggg[acg]aaa|ttt[cgt]ccct",
    "agggt[cgt]aa|tt[acg]accct",
    "agggta[cgt]a|t[acg]taccct",
    "agggtaa[cgt]|[acg]ttaccct"
  ];

  var subst = [
    ("B", "(c|g|t)"), ("D", "(a|g|t)"), ("H", "(a|c|t)"), ("K", "(g|t)"),
    ("M", "(a|c)"), ("N", "(a|c|g|t)"), ("R", "(a|g)"), ("S", "(c|g)"),
    ("V", "(a|c|g)"), ("W", "(a|t)"), ("Y", "(c|t)")
  ];

  var total, copy : string;
  stdin.readstring(total); // reads the entire file in
  const initLen = total.length;

  // remove newlines
  var noLines = compile(">.*\n|\n");
  total = noLines.sub("", total);
  const noLineLen = total.length;

  copy = total; // grab a copy so we can replace in parallel

  var results : [variants.domain] int;

  var isDone : sync bool;

  // fire off a thread to do replacing
  begin ref(copy) {
    for (f, r) in subst {
      const re = compile(f);
      copy = re.sub(r, copy);
    }
    isDone = true; // set the 'full' state
  }

  // count patterns
  forall (pattern, result) in zip(variants, results) {
    var re = compile(pattern);
    for m in re.matches(total) do result += 1;
  }

  isDone; // waits for the 'full' state

  // print results
  for (p,r) in zip(variants, results) do writeln(p, " ", r);

  writeln("\n", initLen);
  writeln(noLineLen);
  writeln(copy.length);
}

proc main() {
  var variants = [
    "agggtaaa|tttaccct",
    "[cgt]gggtaaa|tttaccc[acg]",
    "a[act]ggtaaa|tttacc[agt]t",
    "ag[act]gtaaa|tttac[agt]ct",
    "agg[act]taaa|ttta[agt]cct",
    "aggg[acg]aaa|ttt[cgt]ccct",
    "agggt[cgt]aa|tt[acg]accct",
    "agggta[cgt]a|t[acg]taccct",
    "agggtaa[cgt]|[acg]ttaccct"
  ];

  var subst = [
    ("B", "(c|g|t)"), ("D", "(a|g|t)"), ("H", "(a|c|t)"), ("K", "(g|t)"),
    ("M", "(a|c)"), ("N", "(a|c|g|t)"), ("R", "(a|g)"), ("S", "(c|g)"),
    ("V", "(a|c|g)"), ("W", "(a|t)"), ("Y", "(c|t)")
  ];

  var total : string;
  stdin.readstring(total);
  const initLen = total.length;

  var noLines = compile(">.*\n|\n");
  total = noLines.sub("", total);
  const noLineLen = total.length;

  var results : [variants.domain] int;

  forall (pattern, result) in zip(variants, results) {
    var re = compile(pattern);
    for match in re.matches(total) do result += 1;
  }

  for (p,r) in zip(variants, results) do writeln(p, " ", r);

  for (f, r) in subst {
    const re = compile(f);
    total = re.sub(r, total);
  }
  writeln("\n", initLen);
  writeln(noLineLen);
  writeln(total.length);
}


Created new interface

Created new interface

class Queue {
  type eltType;

  // Adds single element...
  proc enqueue(elt : eltType);

  // Adds multiple elements...
  proc enqueue(elt : eltType ... ?nElts);

  // Adds in another queue's elements to our own in FIFO order. This operation is *not*
  // lineraizable.
  proc enqueue(queue : Queue(eltType));

  // Add all elements yielded by some function. If a way to actually capture an iterator
  // exists, perfect! Otherwise, we'd need to improvise with some kind of lambda that
  // returns whatever is being yielded. (I hope this is possible. This would allow us to
  // add *any* element to our queue from another other container that contained an element,
  // *even* ones that were infinite streams... "Why would we find interest in infinite streams
  // for a queue that can run out of space?" - It won't run out for a bounded queue and is a cool
  // way to continuously add elements in an on-demand fashion. Perhaps if it is meant to be used
  // in this fashion it could be it's own function?) Example of a good use-case: "How does my application
  // scale when presented with near infinite (but bounded) amounts of data"? You can easily populate
  // the queue repeatedly with dummy data/work. Just an idea...
  proc enqueue(iterObj)

  // Normal dequeue
  proc dequeue() : (bool, eltType);

  // Dequeue *multiple* elements.
  proc dequeue(nElems) : (int, [] eltType);
}

class QueueFactory {
  /*
    (WIP - In Planning)

    Creates a bounded strict First-In-First-Out Queue. The queue utilizes a distributed
    array and provides wait-free dequeue operations. The queue is dervied from the
    'FFQ: A Fast Single-Producer/Multiple-Consumer Concurrent FIFO Queue' seen here:
    http://se.inf.tu-dresden.de/pubs/papers/ffq2017.pdf

    To allow safe concurrent enqueuers, an extension is required that combines the
    high performance of the CC-Synch algorithm to create a derived variant of H-Synch
    I call the 'FCHLock', or the 'Flat Combining Hierarichal Lock'. An extremely helpful
    slide can be seen here:
    https://opencourses.uoc.gr/courses/pluginfile.php/17173/mod_resource/content/0/HY586-Section3.pdf

    :type eltType: Element type

    :arg maxElems: Maximum number of elements in the queue; halts if value is 0.
    :type maxElems: uint

    :arg targetLocales: Locales to distribute across.
    :type targetLocales: [] locales

    :rtype: DistributedBoundedFIFO(eltType)
  */
  proc makeDistributedBoundedFIFO(
    type eltType,
    maxElems : uint = 0,
    targetLocales : [] locale = Locales
  ) : Queue(eltType);

  // Local variant
  proc makeBoundedFIFO(
    type eltType,
    maxElems : uint = 0
  ) : Queue(eltType);

  /*
    Creates an unbounded strict First-In-First-Out Queue. In this queue, each locale is
    given their own queue, which uses a wait-free round robin algorithm to fairly
    distribute computation, memory, and bandwidth and offers scalable performance.
    The queue also allows the user to use their own custom backbone queues but defaults
    to Michael Scott's two-locked synchronized queue, seen here:
    https://www.research.ibm.com/people/m/michael/podc-1996.pdf

    :type eltType: Element type

    :arg targetLocales: Locales to distribute across.
    :type targetLocales: [] locales

    :rtype: DistributedUnboundedFIFO(eltType)
  */
  proc makeDistributedUnboundedFIFO(
    type eltType,
    targetLocales : [] locale = Locales
  ) : Queue(eltType);

  // Local variant
  proc makeUnboundedFIFO(
    type eltType;
  ) : Queue(eltType);

  /*
    (WIP - Implementing)

  */
  proc makeDistributedWorkQueue(
    type eltType,
    targetLocales : [] locale = Locales
  ) : Queue(eltType);

  // Local variant
  proc makeWorkQueue(
    type eltType
  ) : Queue(eltType);
}

class Queue {
  type eltType;

  proc enqueue(elt : eltType) {
    halt("Enqueue Not Overloaded");
  }

  proc dequeue() : (bool, eltType) {
    halt ("Dequeue Not Overloaded");
  }
}


latest on the NN code, driving me nuts

latest on the NN code, driving me nuts

/*
 Another pass at the NN based on Alg. 6.3, 6.4 in Goodfellow, et. al. Chapter 6, approx p 204
 */

 module NN {
   use LinearAlgebra,
       Time,
       Norm,
       Random;

   class Sequential {
     var layerDom = {1..0},
         layers: [layerDom] Layer,
         batchSize: int,
         outDim: int,
         loss: Loss = new Loss(),
         momentum: real = 0.05,
         lr: real = 0.03,
         trained: bool = false,
         reportInterval: int = 100;

     proc init() { }

     proc add(d:Dense) {
       this.layers.push_back(new Layer(units=d.units, name="H" + (this.layerDom.high+1):string));
     }
     proc add(a: Activation) {
       ref currentLayer = this.layers[this.layerDom.last];
       currentLayer.activation = a;
     }

     proc compile(X:[], y:[]) {
       var inputDim = X.shape[1];
       if y.shape.size == 2 {
         this.outDim = y.shape[2];
       } else {
         this.outDim = 1;
       }
       this.batchSize = X.shape[1];
       var dataLayer = new Layer(units=X.shape[2], name="Data");
       dataLayer.wDom = {1..X.shape[1], 1..X.shape[2]};
       dataLayer.hDom = {1..X.shape[1], 1..X.shape[2]};
       dataLayer.h = X;
       this.layers.push_front(dataLayer);
       writeln(dataLayer);
       this.layers.push_back(new Layer(units=this.outDim, name="Yhat"));
       for l in this.layerDom.low+1..this.layerDom.high {
         ref lowerLayer = this.layers[l-1];
         ref currentLayer = this.layers[l];
         currentLayer.aDom = {1..this.batchSize, 1..currentLayer.units};
         currentLayer.hDom = {1..this.batchSize, 1..currentLayer.units};
         if !this.trained {
           currentLayer.wDom = {1..lowerLayer.units, 1..currentLayer.units};
           currentLayer.bDom = {1..currentLayer.units};  // will construct batchSize tall matrix later
           fillRandom(currentLayer.W);
           fillRandom(currentLayer.b);
           currentLayer.W = 0.25 * currentLayer.W;
           currentLayer.b = 0.25 * currentLayer.b;
         }
         writeln(currentLayer);
       }
     }
     proc fit(xTrain:[], yTrain:[], epochs: int, batchSize:int, lr: real=0.01) {
       this.lr = lr;
       this.compile(X=xTrain, y=yTrain);
       for i in 1..epochs {
         const o = this.feedForward(X=xTrain, y=yTrain);
         //writeln("at the top! Current output:\n ", o.h);
         this.layers[this.layerDom.high].error = this.loss.J(yHat=o.h, y=yTrain);
         if i % this.reportInterval == 0 {
           try! writeln("epoch: %5n norm error: %7.4dr".format(i, norm(this.layers[this.layerDom.high].error.T)));
         }
         this.backProp(X=xTrain, y=yTrain);
       }
       writeln("Done!  Current error:\n", this.layers[this.layerDom.high].h.T);
       this.trained = true;
     }
     proc feedForward(X:[], y:[]) {
       for l in this.layerDom.low+1..this.layerDom.high {
         ref lowerLayer = this.layers[l-1];
         ref currentLayer = this.layers[l];
         //try! this.printStep(upperLayer=currentLayer, lowerLayer=lowerLayer,direction="feedForward",step=l);
         var b:[1..this.batchSize, 1..currentLayer.units] real;
         for i in 1..this.batchSize {
           b[i,..] = currentLayer.b;
         }
         currentLayer.a = b.plus(lowerLayer.h.dot(currentLayer.W)); // Don't forget to add the bias
         currentLayer.h = currentLayer.activation.f(currentLayer.a);
       }
       return this.layers[this.layerDom.high];
     }
     /*
     Notice that under this schedule, the errors reach "up" and the gradients reach "down".
     The gradients depend on the errors.
      */
     proc backProp(X:[], y:[]) {
       var t:[1..this.batchSize, 1..this.outDim] real;
       for l in this.layerDom.low..this.layerDom.high-1 by -1 {
         ref upperLayer = this.layers[l+1];
         ref currentLayer = this.layers[l];
         //ref lowerLayer= this.layers[l-1];
         //try! this.printStep(upperLayer=currentLayer, lowerLayer=lowerLayer, direction="backProp",step=l);
         //try! this.printStep(upperLayer=upperLayer, lowerLayer=currentLayer, direction="backProp",step=l);
         currentLayer.error = currentLayer.h * (ones(currentLayer.h.domain) - currentLayer.h) * (upperLayer.error.dot(upperLayer.W.T));
         upperLayer.dW = currentLayer.h.T.dot(upperLayer.error);
         upperLayer.db = colSums(upperLayer.error);
         upperLayer.W_vel = this.momentum * upperLayer.W_vel - this.lr * upperLayer.dW;
         upperLayer.W = upperLayer.W.plus(upperLayer.W_vel);
         upperLayer.b_vel = this.momentum * upperLayer.b_vel - this.lr * upperLayer.db;
         upperLayer.b = upperLayer.b.plus(upperLayer.b_vel);

       }
     }

     proc printStep(upperLayer: Layer, lowerLayer: Layer, direction: string, step: int) throws {
       writeln(" * %s: %n".format(direction, step));
       writeln(upperLayer);
       writeln(lowerLayer);
     }

   }

   class Layer {
     var name: string,
         units: int,
         inputDim: int,
         activation: Activation,
         wDom: domain(2),
         bDom: domain(1),
         aDom: domain(2),
         hDom: domain(2),
         W:[wDom] real,
         dW:[wDom] real,
         W_vel:[wDom] real = 0.0,
         b:[bDom] real,
         db:[bDom] real,
         b_vel: [bDom] real = 0.0,
         a:[aDom] real,
         h:[hDom] real,
         g:[aDom] real,
         error: [hDom] real;

     proc init(name: string, units: int){
       this.name=name;
       this.units = units;
       this.activation = new Activation(name="DEFAULT");
     }

     proc readWriteThis(f) throws {
       f <~> "%6s".format(this.name)
         <~> " W:" <~> this.W.shape
         <~> " h:" <~> this.h.shape
         <~> " b:" <~> this.b.shape
         <~> " a:" <~> this.a.shape;

     }
   }

   class Activation {
     var name: string;
     proc init(name: string) {
       this.name=name;
     }

     proc f(x: real) {
       if name == "relu" {
         return sigmoid(x);
       } else if name == "logistic" {
         return exp(x);
       } else {
        return x;
       }
     }

     proc df(x:real) {
       if this.name == "relu" {
         return derivativesSigmoid(x);
       } else {
         return 1;
       }
     }

     proc sigmoid(x: real) {
       return (1/(1 + exp(-x)));
     }
     proc derivativesSigmoid(x) {
       return x * (1-x);
     }
  }

  class Dense {
    var units: int,
        inputDim: int;

    proc init(units:int, inputDim=0) {
      this.units=units;
      this.inputDim=inputDim;
    }
  }

  class Loss {
    var name: string;
    proc init(name: string="DEFAULT") {
      this.name = name;
    }
    proc J(yHat: [], y:[]) {
      var r: [yHat.domain] real;
      if this.name == "DEFAULT" {
        r = yHat - y;
      } else {
        r = yHat - y;
      }
      return r;
    }
  }

}

/*
 Another pass at the NN based on Alg. 6.3, 6.4 in Goodfellow, et. al. Chapter 6, approx p 204
 */

 module NN {
   use LinearAlgebra,
       Time,
       Norm,
       Random;

   class Sequential {
     var layerDom = {1..0},
         layers: [layerDom] Layer,
         batchSize: int,
         outDim: int,
         loss: Loss = new Loss(),
         momentum: real = 0.9,
         lr: real = 0.01,
         trained: bool = false,
         reportInterval: int = 100;

     proc init() { }

     proc add(d:Dense) {
       this.layers.push_back(new Layer(units=d.units, name="H" + (this.layerDom.high+1):string));
     }
     proc add(a: Activation) {
       ref currentLayer = this.layers[this.layerDom.last];
       currentLayer.activation = a;
     }

     proc compile(X:[], y:[]) {
       var inputDim = X.shape[1];
       if y.shape.size == 2 {
         this.outDim = y.shape[2];
       } else {
         this.outDim = 1;
       }
       this.batchSize = X.shape[1];
       var dataLayer = new Layer(units=X.shape[2], name="Data");
       dataLayer.wDom = {1..X.shape[1], 1..X.shape[2]};
       dataLayer.hDom = {1..X.shape[1], 1..X.shape[2]};
       dataLayer.h = X;
       this.layers.push_front(dataLayer);
       writeln(dataLayer);
       this.layers.push_back(new Layer(units=this.outDim, name="Yhat"));
       for l in this.layerDom.low+1..this.layerDom.high {
         ref lowerLayer = this.layers[l-1];
         ref currentLayer = this.layers[l];
         currentLayer.aDom = {1..this.batchSize, 1..currentLayer.units};
         currentLayer.hDom = {1..this.batchSize, 1..currentLayer.units};
         if !this.trained {
           currentLayer.wDom = {1..lowerLayer.units, 1..currentLayer.units};
           currentLayer.bDom = {1..currentLayer.units};  // will construct batchSize tall matrix later
           fillRandom(currentLayer.W);
           fillRandom(currentLayer.b);
           currentLayer.W = 0.25 * currentLayer.W;
           currentLayer.b = 0.25 * currentLayer.b;
         }
         writeln(currentLayer);
       }
     }
     proc fit(xTrain:[], yTrain:[], epochs: int, batchSize:int, lr: real=0.01) {
       this.lr = lr;
       this.compile(X=xTrain, y=yTrain);
       for i in 1..epochs {
         const o = this.feedForward(X=xTrain, y=yTrain);
         //writeln("at the top! Current output:\n ", o.h);
         this.layers[this.layerDom.high].error = this.loss.J(yHat=o.h, y=yTrain);
         if i % this.reportInterval == 0 {
           try! writeln("epoch: %5n norm error: %7.4dr".format(i, norm(this.layers[this.layerDom.high].error)));
         }
         this.backProp(X=xTrain, y=yTrain);
       }
       writeln("Done!  Current error:\n", this.layers[this.layerDom.high].h);
       this.trained = true;
     }
     proc feedForward(X:[], y:[]) {
       for l in this.layerDom.low+1..this.layerDom.high {
         ref lowerLayer = this.layers[l-1];
         ref currentLayer = this.layers[l];
         //try! this.printStep(upperLayer=currentLayer, lowerLayer=lowerLayer,direction="feedForward",step=l);
         var b:[1..this.batchSize, 1..currentLayer.units] real;
         for i in 1..this.batchSize {
           b[i,..] = currentLayer.b;
         }
         currentLayer.a = b.plus(lowerLayer.h.dot(currentLayer.W)); // Don't forget to add the bias
         currentLayer.h = currentLayer.activation.f(currentLayer.a);
       }
       return this.layers[this.layerDom.high];
     }
     proc backProp(X:[], y:[]) {
       var t:[1..this.batchSize, 1..this.outDim] real;
       for l in this.layerDom.low+1..this.layerDom.high by -1 {
         ref currentLayer = this.layers[l];
         ref lowerLayer= this.layers[l-1];
         //try! this.printStep(upperLayer=currentLayer, lowerLayer=lowerLayer, direction="backProp",step=l);
         currentLayer.dW = lowerLayer.h.T.dot(currentLayer.error);
         currentLayer.db = colSums(currentLayer.error);
         currentLayer.W_vel = this.momentum * currentLayer.W_vel - this.lr * currentLayer.dW;
         currentLayer.b_vel = this.momentum * currentLayer.b_vel - this.lr * currentLayer.db;
         // Need to add momentum before I update W, b
         currentLayer.W = currentLayer.W.plus(currentLayer.W_vel);
         currentLayer.b = currentLayer.b.plus(currentLayer.b_vel);
       }
     }

     proc printStep(upperLayer: Layer, lowerLayer: Layer, direction: string, step: int) throws {
       writeln(" * %s: %n".format(direction, step));
       writeln(upperLayer);
       writeln(lowerLayer);
     }

   }

   class Layer {
     var name: string,
         units: int,
         inputDim: int,
         activation: Activation,
         wDom: domain(2),
         bDom: domain(1),
         aDom: domain(2),
         hDom: domain(2),
         W:[wDom] real,
         dW:[wDom] real,
         W_vel:[wDom] real = 0.0,
         b:[bDom] real,
         db:[bDom] real,
         b_vel: [bDom] real = 0.0,
         a:[aDom] real,
         h:[hDom] real,
         g:[aDom] real,
         error: [hDom] real;

     proc init(name: string, units: int){
       this.name=name;
       this.units = units;
       this.activation = new Activation(name="DEFAULT");
     }

     proc readWriteThis(f) throws {
       f <~> "%6s".format(this.name)
         <~> " W:" <~> this.W.shape
         <~> " h:" <~> this.h.shape
         <~> " b:" <~> this.b.shape
         <~> " a:" <~> this.a.shape;

     }
   }

   class Activation {
     var name: string;
     proc init(name: string) {
       this.name=name;
     }

     proc f(x: real) {
       if name == "relu" {
         return sigmoid(x);
       } else if name == "logistic" {
         return exp(x);
       } else {
        return x;
       }
     }

     proc df(x:real) {
       if this.name == "relu" {
         return derivativesSigmoid(x);
       } else {
         return 1;
       }
     }

     proc sigmoid(x: real) {
       return (1/(1 + exp(-x)));
     }
     proc derivativesSigmoid(x) {
       return x * (1-x);
     }
  }

  class Dense {
    var units: int,
        inputDim: int;

    proc init(units:int, inputDim=0) {
      this.units=units;
      this.inputDim=inputDim;
    }
  }

  class Loss {
    var name: string;
    proc init(name: string="DEFAULT") {
      this.name = name;
    }
    proc J(yHat: [], y:[]) {
      var r: [yHat.domain] real;
      if this.name == "DEFAULT" {
        r = yHat - y;
      } else {
        r = yHat - y;
      }
      return r;
    }
  }

}


Fixed oops.

Fixed oops.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@5679 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

var i = 12;
var f = 12.0;

function mywrite(x) {
  function bar(tmp:integer) {
    writeln("mywrite integer for       ", x);
  }
  function bar(tmp:float) {
    writeln("myrite float for          ", x);
  }
  function bar(tmp) {
    writeln("mywrite not supported for ", x);
  }
  bar(x);
}

mywrite(i);
mywrite(f);
mywrite(true);
mywrite("hello world");

var i = 12;
var f = 12.0;

function mywrite(x) {
  function bar(tmp:integer) {
    writeln("mywrite integer for       ", x);
  }
  function bar(tmp:integer) {
    writeln("myrite float for          ", x);
  }
  function bar(tmp:integer) {
    writeln("mywrite not supported for ", x);
  }
  bar(x);
}

mywrite(i);
mywrite(f);
mywrite(true);
mywrite("hello world");


Fixed the 'local' failure in test/studies/ssca2/vass.

In this code:

      proc addEdgeOnVertex(uArg, vArg, wArg, firstAvail$: sync int) {
        on this do {
          const v = vArg, w = wArg;
          local {
            const edgePos = firstAvail$;
            ...

even when 'firstAvail$' is co-located with 'this', the above access
to it results in communication to the locale where addEdgeOnVertex()
was invoked.

The fix in this commit is a workaround until we fix the compiler
to do the right thing.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20213 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module analyze_RMAT_graph_associative_array {

  //
  // If graphInputFile != "", read in the graph from the files with
  // that prefix; do not run edge generation or Kernel 1.
  // Otherwise, if graphOutputFile != "" then write out the graph
  // generated by edge generation and Kernel 1 into those files.
  //
  // The *DStyle strings control the debugging printouts during I/O.
  // See interpretDStyle() for details.
  //
  // If graphVerifyFile != "" and the graph was read from graphInputFile,
  // write it out to graphVerifyFile. Could be used for correctness checking.
  //
  // The graph size in graphInputFile must match the SCALE config constant.
  //
  config const graphInputFile    = "";
  config const graphInputDStyle  = "-";
  config const graphOutputFile   = "";
  config const graphOutputDStyle = "-";
  config const graphVerifyFile   = "";
  config const graphVerifyDStyle = "-";

  // used during graph allocation
  config const initialRMATNeighborListLength =
    if graphInputFile == "" then 16 else 0;
  param initialFirstAvail = 1;

    //
    // We will represent the neighbor list as an array of nleType.
    // nle = Neighbor List Element.
    //
    type nleType = (int(64), int(64));
    // Extract each component from a neighbor.
    proc nleNID(nlElm: nleType)    return nlElm(1);
    proc nleWeight(nlElm: nleType) return nlElm(2);
    // Mark a use of a neighbor as a pair.
    proc nleAsPair(nlElm: nleType) return nlElm;
    // Produce a neighbor from components.
    proc nleMake(nID: int(64), wt: int(64)): nleType  return (nID, wt);

    //
    // VertexData: stores the neighbor list of a vertex.
    //
    record VertexData {
      var ndom = [initialFirstAvail..initialRMATNeighborListLength];
      var neighborList: [ndom] nleType;

      proc numNeighbors()  return ndom.numIndices;

      // firstAvail$ must be passed by reference
      proc addEdgeOnVertex(uArg, vArg, wArg, firstAvail$: sync int) {
        on this do {
          // todo: the compiler should make these values local automatically!
          const /*u = uArg,*/ v = vArg, w = wArg;
            // Lock and unlock should be within 'local', but currently
            // need to pull them out due to implementation.
            // lock the vertex
            const edgePos = firstAvail$;

          local {
            const prevNdomLen = ndom.high;
            if edgePos > prevNdomLen {
              // grow our arrays, by 2x
              // statistics: growCount += 1;
              ndom = [1..prevNdomLen * 2];
              // bounds checking below will ensure (edgePos <= ndom.high)
            }
            // store the edge
            neighborList[edgePos] = nleMake(v, w);
          }

            // release the lock
            firstAvail$ = edgePos + 1;
        } // on
      }

      // not parallel-safe
      proc tidyNeighbors(firstAvail$: sync int) {
        local {
          // no synchronization here
          var edgeCount = firstAvail$.readXX() - 1;
          RemoveDuplicates(1, edgeCount);
          // TODO: ideally if we don't save much memory, do not resize
          if edgeCount != ndom.numIndices {
            // statistics: shrinkCount += 1;
            ndom = 1..edgeCount;
          }
          // writeln("stats ", growCount, " ", shrinkCount, ".");
        }
      }

      //
      // Jargon: a "duplicate" is an edge v1->v2 for which
      // there is another edge v1->v2, possibly with a different weight.
      //
      proc RemoveDuplicates(lo, inout hi) {
        param showArrays = false;  // beware of 'local' in the caller
        const style = new iostyle(min_width = 3);
        if showArrays {
          writeln("starting ", lo, "..", hi);
          stdout.writeln(neighborList(lo..hi), style);
        }

        // TODO: remove the duplicates as we sort
        // InsertionSort, keep duplicates
        for i in lo+1..hi {
          const (ithNID, ithEDW) = nleAsPair(neighborList(i));
          var inserted = false;

          for j in lo..i-1 by -1 {
            if ithNID < nleNID(neighborList(j)) {
              neighborList(j+1) = neighborList(j);
            } else {
              neighborList(j+1) = nleMake(ithNID, ithEDW);
              inserted = true;
              break;
            }
          }

          if (!inserted) {
            neighborList(lo)= nleMake(ithNID, ithEDW);
          }
        }
        //writeln("sorted ", lo, "..", hi);

        // remove the duplicates
        var foundDup = false;
        var indexDup: int;
        var lastNID = nleNID(neighborList(lo));

        for i in lo+1..hi {
          const currNID = nleNID(neighborList(i));
          if lastNID == currNID {
            foundDup = true;
            indexDup = i;
            break;
          } else {
            lastNID = currNID;
          }
        }

        if foundDup {
          // indexDup points to a hole
          // the already-found dup is dropped before entering the loop
          for i in indexDup+1..hi {
            const currNID = nleNID(neighborList(i));
            if lastNID == currNID {
              // dropping this duplicate
            } else {
              // moving a non-duplicate value
              neighborList(indexDup) = nleMake(currNID,
                                               nleWeight(neighborList(i)));
              indexDup += 1;
              lastNID = currNID;
            }
          }
          hi = indexDup - 1;
        }
        //writeln("eliminated dups ", lo, "..", hi);

        // VerifySort
        if boundsChecking then
          for i in lo..hi-1 do
            if !( nleNID(neighborList(i)) < nleNID(neighborList(i+1)) ) then
              writeln("unsorted IDs for i = ", i, "   ",
                      neighborList(i), " !< ", neighborList(i+1));

        if showArrays {
          writeln("sorted ", lo, "..", hi);
          stdout.writeln(neighborList(lo..hi), style);
          writeln();
        }
      }  // RemoveDuplicates

    } // record VertexData

  // +========================================================================+
  // |  Define associative array-based representations for general sparse     |
  // |  graphs. Provide execution template to generate a random RMAT graph    |
  // |  of a specified size and execute and verify SSCA2 kernels 2 through 4. |
  // =========================================================================+

  proc  generate_and_analyze_associative_array_RMAT_graph_representation {

    // -----------------------------------------------------------------
    // compute a random power law graph with 2^SCALE vertices, using
    // the RMAT generator. Initially generate a list of triples.
    // Then convert it to a Chapel representation of a sparse graph,
    // timing this step (Kernel 1).  Finally, execute Kernels 2, 3 and 4
    // of SSCA #2, using identically the same code as in the various
    // torus cases.
    // -----------------------------------------------------------------

    use SSCA2_compilation_config_params, SSCA2_execution_config_consts;

    use SSCA2_driver, SSCA2_RMAT_graph_generator;

    use BlockDist;

    use io_RMAT_graph;

    var n_raw_edges = 8 * N_VERTICES;

    assert ( SCALE > 1, "SCALE must be greater than 1");

    select SCALE {
      when 2 do n_raw_edges = N_VERTICES / 2;
      when 3 do n_raw_edges = N_VERTICES;
      when 4 do n_raw_edges = 2 * N_VERTICES;
      when 5 do n_raw_edges = 4 * N_VERTICES;
      }

    writeln ('-------------------------------------');
    writeln ('Order of RMAT generated graph:', N_VERTICES);
    writeln ('          number of raw edges:', n_raw_edges);
    writeln ('-------------------------------------');
    writeln ();

    // ------------------------------------------------------------------------
    // The data structures below are chosen to implement an irregular (sparse)
    // graph using rectangular domains and arrays.
    // Each node in the graph has a list of neighbors and a corresponding list
    // of (integer) weights for the implicit edges.
    // ------------------------------------------------------------------------

    const vertex_domain =
      if DISTRIBUTION_TYPE == "BLOCK" then
        [1..N_VERTICES] dmapped Block ( [1..N_VERTICES] )
      else
	[1..N_VERTICES] ;

    class Associative_Graph {
      const vertices;
      var   Row      : [vertices] VertexData;

      // iterate over neighbor IDs, with filtering

      iter FilteredNeighbors( v : index (vertices) ) {
        for nlElm in Row(v).neighborList do
          if !FILTERING || nleWeight(nlElm)%8 != 0 then
            yield nleNID(nlElm);
      }

      // Stand-alone parallel iterator would be nice
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        pragma "no copy" pragma "no auto destroy"
        const myDom = Row(v).ndom._value; // Cache the value
        // 1-d, no stride assumed in the follower
        const wholeLow = myDom.ranges(1).low;
        for block in myDom.these(tag) do
          yield (block(1).low, block(1).high, wholeLow);
      }

      // WARNING: This can't be zippered with anything other than itself
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        pragma "no copy" pragma "no auto destroy"
        const neighbors => Row(v).neighborList;
        const (low, high, wholeLow) = followThis;
        // 1-d, no stride
        const myElems = (low..high) + wholeLow;
        for n in myElems {
          if !FILTERING || nleWeight(neighbors(n))%8 != 0 then
            yield nleNID(neighbors(n));
        }
      }

      // iterate over all neighbor (ID, weight) pairs

      proc NeighborPairs( v : index (vertices) ) {   // implies nleAsPair
        return Row (v).neighborList;
      }

      // iterate over all neighbor IDs

      iter Neighbors( v : index (vertices) ) {
        for nlElm in Row(v).neighborList do
          yield nleNID(nlElm);
      }

      iter Neighbors( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        for block in Row(v).neighborList._value.these(tag) do
          yield block;
      }

      iter Neighbors( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        for nlElm in Row(v).neighborList._value.these(tag, followThis) do
          yield nleNID(nlElm);
      }

      // iterate over all neighbor weights

      iter edge_weight( v : index (vertices) ) {
        for nlElm in Row(v).neighborList do
          yield nleWeight(nlElm);
      }

      iter edge_weight( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        for block in Row(v).neighborList._value.these(tag) do
          yield block;
      }

      iter edge_weight( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        for nlElm in Row(v).neighborList._value.these(tag, followThis) do
          yield nleWeight(nlElm);
      }

      // return the number of all neighbors

      proc   n_Neighbors (v : index (vertices) )
      {return Row (v).numNeighbors();}

    } // class Associative_Graph

    writeln("allocating Associative_Graph");
    var G = new Associative_Graph (vertex_domain);

    // ------------------------------------------------------------------
    // generate RMAT graph of the specified size, based on input config
    // values for quadrant assignment.
    // ------------------------------------------------------------------

   if graphInputFile == "" {
    Gen_RMAT_graph ( RMAT_a, RMAT_b, RMAT_c, RMAT_d, vertex_domain,
		     SCALE, N_VERTICES, n_raw_edges, MAX_EDGE_WEIGHT, G );

    if graphOutputFile != "" then
      Writeout_RMAT_graph ( G, graphOutputFile, graphOutputDStyle );

   } else {
    Readin_RMAT_graph ( G, graphInputFile, graphInputDStyle );

    if graphVerifyFile != "" then
      Writeout_RMAT_graph ( G, graphVerifyFile, graphVerifyDStyle );
   }

    execute_SSCA2 ( G );
    writeln (); writeln ();
    delete G;
  }

}

module analyze_RMAT_graph_associative_array {

  //
  // If graphInputFile != "", read in the graph from the files with
  // that prefix; do not run edge generation or Kernel 1.
  // Otherwise, if graphOutputFile != "" then write out the graph
  // generated by edge generation and Kernel 1 into those files.
  //
  // The *DStyle strings control the debugging printouts during I/O.
  // See interpretDStyle() for details.
  //
  // If graphVerifyFile != "" and the graph was read from graphInputFile,
  // write it out to graphVerifyFile. Could be used for correctness checking.
  //
  // The graph size in graphInputFile must match the SCALE config constant.
  //
  config const graphInputFile    = "";
  config const graphInputDStyle  = "-";
  config const graphOutputFile   = "";
  config const graphOutputDStyle = "-";
  config const graphVerifyFile   = "";
  config const graphVerifyDStyle = "-";

  // used during graph allocation
  config const initialRMATNeighborListLength =
    if graphInputFile == "" then 16 else 0;
  param initialFirstAvail = 1;

    //
    // We will represent the neighbor list as an array of nleType.
    // nle = Neighbor List Element.
    //
    type nleType = (int(64), int(64));
    // Extract each component from a neighbor.
    proc nleNID(nlElm: nleType)    return nlElm(1);
    proc nleWeight(nlElm: nleType) return nlElm(2);
    // Mark a use of a neighbor as a pair.
    proc nleAsPair(nlElm: nleType) return nlElm;
    // Produce a neighbor from components.
    proc nleMake(nID: int(64), wt: int(64)): nleType  return (nID, wt);

    //
    // VertexData: stores the neighbor list of a vertex.
    //
    record VertexData {
      var ndom = [initialFirstAvail..initialRMATNeighborListLength];
      var neighborList: [ndom] nleType;

      proc numNeighbors()  return ndom.numIndices;

      // firstAvail$ must be passed by reference
      proc addEdgeOnVertex(uArg, vArg, wArg, firstAvail$: sync int) {
        on this do {
          // todo: the compiler should make these values local automatically!
          const /*u = uArg,*/ v = vArg, w = wArg;
          local {
            // lock the vertex
            const edgePos = firstAvail$;

            const prevNdomLen = ndom.high;
            if edgePos > prevNdomLen {
              // grow our arrays, by 2x
              // statistics: growCount += 1;
              ndom = [1..prevNdomLen * 2];
              // bounds checking below will ensure (edgePos <= ndom.high)
            }
            // store the edge
            neighborList[edgePos] = nleMake(v, w);

            // release the lock
            firstAvail$ = edgePos + 1;
          }
        } // on
      }

      // not parallel-safe
      proc tidyNeighbors(firstAvail$: sync int) {
        local {
          // no synchronization here
          var edgeCount = firstAvail$.readXX() - 1;
          RemoveDuplicates(1, edgeCount);
          // TODO: ideally if we don't save much memory, do not resize
          if edgeCount != ndom.numIndices {
            // statistics: shrinkCount += 1;
            ndom = 1..edgeCount;
          }
          // writeln("stats ", growCount, " ", shrinkCount, ".");
        }
      }

      //
      // Jargon: a "duplicate" is an edge v1->v2 for which
      // there is another edge v1->v2, possibly with a different weight.
      //
      proc RemoveDuplicates(lo, inout hi) {
        param showArrays = false;  // beware of 'local' in the caller
        const style = new iostyle(min_width = 3);
        if showArrays {
          writeln("starting ", lo, "..", hi);
          stdout.writeln(neighborList(lo..hi), style);
        }

        // TODO: remove the duplicates as we sort
        // InsertionSort, keep duplicates
        for i in lo+1..hi {
          const (ithNID, ithEDW) = nleAsPair(neighborList(i));
          var inserted = false;

          for j in lo..i-1 by -1 {
            if ithNID < nleNID(neighborList(j)) {
              neighborList(j+1) = neighborList(j);
            } else {
              neighborList(j+1) = nleMake(ithNID, ithEDW);
              inserted = true;
              break;
            }
          }

          if (!inserted) {
            neighborList(lo)= nleMake(ithNID, ithEDW);
          }
        }
        //writeln("sorted ", lo, "..", hi);

        // remove the duplicates
        var foundDup = false;
        var indexDup: int;
        var lastNID = nleNID(neighborList(lo));

        for i in lo+1..hi {
          const currNID = nleNID(neighborList(i));
          if lastNID == currNID {
            foundDup = true;
            indexDup = i;
            break;
          } else {
            lastNID = currNID;
          }
        }

        if foundDup {
          // indexDup points to a hole
          // the already-found dup is dropped before entering the loop
          for i in indexDup+1..hi {
            const currNID = nleNID(neighborList(i));
            if lastNID == currNID {
              // dropping this duplicate
            } else {
              // moving a non-duplicate value
              neighborList(indexDup) = nleMake(currNID,
                                               nleWeight(neighborList(i)));
              indexDup += 1;
              lastNID = currNID;
            }
          }
          hi = indexDup - 1;
        }
        //writeln("eliminated dups ", lo, "..", hi);

        // VerifySort
        if boundsChecking then
          for i in lo..hi-1 do
            if !( nleNID(neighborList(i)) < nleNID(neighborList(i+1)) ) then
              writeln("unsorted IDs for i = ", i, "   ",
                      neighborList(i), " !< ", neighborList(i+1));

        if showArrays {
          writeln("sorted ", lo, "..", hi);
          stdout.writeln(neighborList(lo..hi), style);
          writeln();
        }
      }  // RemoveDuplicates

    } // record VertexData

  // +========================================================================+
  // |  Define associative array-based representations for general sparse     |
  // |  graphs. Provide execution template to generate a random RMAT graph    |
  // |  of a specified size and execute and verify SSCA2 kernels 2 through 4. |
  // =========================================================================+

  proc  generate_and_analyze_associative_array_RMAT_graph_representation {

    // -----------------------------------------------------------------
    // compute a random power law graph with 2^SCALE vertices, using
    // the RMAT generator. Initially generate a list of triples.
    // Then convert it to a Chapel representation of a sparse graph,
    // timing this step (Kernel 1).  Finally, execute Kernels 2, 3 and 4
    // of SSCA #2, using identically the same code as in the various
    // torus cases.
    // -----------------------------------------------------------------

    use SSCA2_compilation_config_params, SSCA2_execution_config_consts;

    use SSCA2_driver, SSCA2_RMAT_graph_generator;

    use BlockDist;

    use io_RMAT_graph;

    var n_raw_edges = 8 * N_VERTICES;

    assert ( SCALE > 1, "SCALE must be greater than 1");

    select SCALE {
      when 2 do n_raw_edges = N_VERTICES / 2;
      when 3 do n_raw_edges = N_VERTICES;
      when 4 do n_raw_edges = 2 * N_VERTICES;
      when 5 do n_raw_edges = 4 * N_VERTICES;
      }

    writeln ('-------------------------------------');
    writeln ('Order of RMAT generated graph:', N_VERTICES);
    writeln ('          number of raw edges:', n_raw_edges);
    writeln ('-------------------------------------');
    writeln ();

    // ------------------------------------------------------------------------
    // The data structures below are chosen to implement an irregular (sparse)
    // graph using rectangular domains and arrays.
    // Each node in the graph has a list of neighbors and a corresponding list
    // of (integer) weights for the implicit edges.
    // ------------------------------------------------------------------------

    const vertex_domain =
      if DISTRIBUTION_TYPE == "BLOCK" then
        [1..N_VERTICES] dmapped Block ( [1..N_VERTICES] )
      else
	[1..N_VERTICES] ;

    class Associative_Graph {
      const vertices;
      var   Row      : [vertices] VertexData;

      // iterate over neighbor IDs, with filtering

      iter FilteredNeighbors( v : index (vertices) ) {
        for nlElm in Row(v).neighborList do
          if !FILTERING || nleWeight(nlElm)%8 != 0 then
            yield nleNID(nlElm);
      }

      // Stand-alone parallel iterator would be nice
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        pragma "no copy" pragma "no auto destroy"
        const myDom = Row(v).ndom._value; // Cache the value
        // 1-d, no stride assumed in the follower
        const wholeLow = myDom.ranges(1).low;
        for block in myDom.these(tag) do
          yield (block(1).low, block(1).high, wholeLow);
      }

      // WARNING: This can't be zippered with anything other than itself
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        pragma "no copy" pragma "no auto destroy"
        const neighbors => Row(v).neighborList;
        const (low, high, wholeLow) = followThis;
        // 1-d, no stride
        const myElems = (low..high) + wholeLow;
        for n in myElems {
          if !FILTERING || nleWeight(neighbors(n))%8 != 0 then
            yield nleNID(neighbors(n));
        }
      }

      // iterate over all neighbor (ID, weight) pairs

      proc NeighborPairs( v : index (vertices) ) {   // implies nleAsPair
        return Row (v).neighborList;
      }

      // iterate over all neighbor IDs

      iter Neighbors( v : index (vertices) ) {
        for nlElm in Row(v).neighborList do
          yield nleNID(nlElm);
      }

      iter Neighbors( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        for block in Row(v).neighborList._value.these(tag) do
          yield block;
      }

      iter Neighbors( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        for nlElm in Row(v).neighborList._value.these(tag, followThis) do
          yield nleNID(nlElm);
      }

      // iterate over all neighbor weights

      iter edge_weight( v : index (vertices) ) {
        for nlElm in Row(v).neighborList do
          yield nleWeight(nlElm);
      }

      iter edge_weight( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        for block in Row(v).neighborList._value.these(tag) do
          yield block;
      }

      iter edge_weight( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        for nlElm in Row(v).neighborList._value.these(tag, followThis) do
          yield nleWeight(nlElm);
      }

      // return the number of all neighbors

      proc   n_Neighbors (v : index (vertices) )
      {return Row (v).numNeighbors();}

    } // class Associative_Graph

    writeln("allocating Associative_Graph");
    var G = new Associative_Graph (vertex_domain);

    // ------------------------------------------------------------------
    // generate RMAT graph of the specified size, based on input config
    // values for quadrant assignment.
    // ------------------------------------------------------------------

   if graphInputFile == "" {
    Gen_RMAT_graph ( RMAT_a, RMAT_b, RMAT_c, RMAT_d, vertex_domain,
		     SCALE, N_VERTICES, n_raw_edges, MAX_EDGE_WEIGHT, G );

    if graphOutputFile != "" then
      Writeout_RMAT_graph ( G, graphOutputFile, graphOutputDStyle );

   } else {
    Readin_RMAT_graph ( G, graphInputFile, graphInputDStyle );

    if graphVerifyFile != "" then
      Writeout_RMAT_graph ( G, graphVerifyFile, graphVerifyDStyle );
   }

    execute_SSCA2 ( G );
    writeln (); writeln ();
    delete G;
  }

}