Fixing bug in SSCA2 torus representation

In the update to use the new domain map syntax, a -1 was inadvertantly
converted to 1 which made a sparse subdomain's index set fall outside
of its dense parent domain.  We never caught this because we compiled
with --fast (no bounds checking).

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20805 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module analyze_torus_graphs {

  // +========================================================================+
  // |  Define sparse subdomain stencil representations for                   |
  // |  k-dimensional torus graphs. For each k, provide execution template    |
  // |  for executing and verifying SSCA2 kernels 2 through 4.                |
  // +========================================================================+
  // |                          VERSION 1                                     |
  // |  Explicit neighbor lists as sparse arithmetic arrays                   |
  // |  defined over the torus stencil as a sparse domain.  Storage overhead  |
  // |  is as many tuples as nonzeros.                                        |
  // +========================================================================+

  use SSCA2_compilation_config_params,
      SSCA2_execution_config_consts,
      SSCA2_driver,
      torus_graph_generator_utilities,
      BlockDist;

  // =====================================================================
  // generic class for stencil-based graph for torus of any dimensionality
  // =====================================================================

  class torus_explicit_stencil_graph {

    const vertex_domain;
    const torus_stencil;
    param dimensions = vertex_domain.rank;

    const vertices = vertex_domain;
    type  vertex = index (vertex_domain);

    var Neighbors   : [vertex_domain] [torus_stencil] vertex;
    var edge_weight : [vertex_domain] [torus_stencil] int;

    proc num_edges {
      return + reduce [v in vertex_domain] n_Neighbors (v);
    }

    iter FilteredNeighbors( v : index (vertices) ) {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil do
        if (FILTERING && weights(n)%8 != 0) || !FILTERING then
          yield neighbors(n);
    }

    // Simply forward the domain's parallel iterator
    iter FilteredNeighbors( v : index (vertices), param tag: iterKind)
    where tag == iterKind.leader {
      for block in torus_stencil._value.these(tag) do
        yield block;
    }

    iter FilteredNeighbors( v : index (vertices), param tag: iterKind, followThis)
    where tag == iterKind.follower {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil._value.these(tag, followThis) do
        if (FILTERING && weights(n)%8 != 0) || !FILTERING then
          yield neighbors(n);
    }

    // iterate over all neighbor (ID, weight) pairs

    iter NeighborPairs( v : index (vertices) ) {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil do
        yield (neighbors(n), weights(n));
    }

    // Simply forward the domain's parallel iterator
    iter NeighborPairs( v : index (vertices), param tag: iterKind)
    where tag == iterKind.leader {
      for block in torus_stencil._value.these(tag) do
        yield block;
    }

    iter NeighborPairs( v : index (vertices), param tag: iterKind, followThis)
    where tag == iterKind.follower {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil._value.these(tag, followThis) do
        yield (neighbors(n), weights(n));
    }

    proc n_Neighbors (v : index (vertices) ) return 2*dimensions;
  }

  // ====================================================
  // compute graph of one dimensional torus with  2^SCALE
  // vertices, then execute Kernels 2, 3 and 4 of SSCA #2
  // ====================================================

  proc generate_and_analyze_1D_torus {

    const d = 2**SCALE;

    const vertex_domain =
      if DISTRIBUTION_TYPE == "BLOCK" then
        { Torus_Base_Index..#d } dmapped Block ( {Torus_Base_Index..#d} )
      else
        { Torus_Base_Index..#d };

    type  vertex = index (vertex_domain);

    const dense_stencil = {-1..1};
    const torus_stencil : sparse subdomain (dense_stencil) = ( -1, 1 );

    var G = new torus_explicit_stencil_graph ( vertex_domain,
					       torus_stencil );

    const exact_between_centrality = 0.25 * (d - 2) * d  -  d/2 + 1;

    writeln ();
    writeln ("One D Torus: all nodes should have",
	     " exact betweenness centrality: ", exact_between_centrality );
    writeln ("             (upper bound on approximate or filtered",
                         " betweenness centrality)");
    writeln ();

    writeln ( "--------------------------------" );
    writeln ( "One D Torus with ", d, " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h = G.vertices.high, l = G.vertices.low;
    var   torus_vertex : [extended_domain] vertex = extended_domain;

    if DEBUG_GRAPH_GENERATOR then {
      writeln ();
      writeln ( "One Dimension Torus");
      writeln ( "   index lower bound: ", l );
      writeln ( "   index upper bound: ", h );
      writeln ();
      writeln ("vertex   Neighbors");}

    // torus_vertex wraps the torus with an additional plane in each
    //  dimension.  fill in the wrap-around vertices in the ghost cells

    torus_vertex (l-1) = torus_vertex (h);
    torus_vertex (h+1) = torus_vertex (l);

    // =======================================================
    // generate graph corresponding to a one dimensional torus
    // =======================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights (MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

  // ==============================================
  // compute graph of an approximately square two
  // dimensional torus with  2^SCALE vertices, then
  // execute Kernels 2, 3 and 4 of SSCA #2
  // ==============================================

  proc generate_and_analyze_2D_torus {

    const lg2d1 : int = SCALE / 2,
          lg2d2 = SCALE - lg2d1,
          d1    = 2**lg2d1,
          d2    = 2**lg2d2;

    const vertex_domain =
     if DISTRIBUTION_TYPE == "BLOCK" then
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2 }
	 dmapped Block ( { Torus_Base_Index..#d1,
			   Torus_Base_Index..#d2 } )
     else
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2 };

    type  vertex = index (vertex_domain);

    const dense_stencil = {-1..1,-1..1};
    const torus_stencil : sparse subdomain (dense_stencil) =
                                        ( (-1,0), (1,0), (0,-1), (0,1) );

    var G = new torus_explicit_stencil_graph ( vertex_domain, torus_stencil );

    var exact_between_centrality : real;

    if ( SCALE % 2 == 1 ) then
      exact_between_centrality = 0.75 * 2 ** ( (3*SCALE-1) / 2 )
                                 - 2**SCALE + 1;
    else
      exact_between_centrality = 2 ** ( (3*SCALE/2) - 1)
	                         - 2**SCALE + 1;

    writeln ();
    writeln ("Two D Torus: all nodes should have",
	     " exact betweenness centrality: ", exact_between_centrality );
    writeln ("             (upper bound on approximate or filtered",
                         " betweenness centrality)");
    writeln ();

    writeln ( "--------------------------------" );
    writeln ( "Two D Torus with ", d1, " x ", d2, " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h1 = G.vertices.high (1), l1 = G.vertices.low (1);
    const h2 = G.vertices.high (2), l2 = G.vertices.low (2);
    var   torus_vertex : [extended_domain] vertex = extended_domain;

    if DEBUG_GRAPH_GENERATOR then {
      writeln ();
      writeln ( "Two Dimension Torus");
      writeln ( "   index lower bounds: ", l1, ", ", l2 );
      writeln ( "   index upper bounds: ", h1, ", ", h2 );
      writeln ();
      writeln ("vertex    Neighbors");
    }

    // torus_vertex wraps the torus with an additional plane in each dimension
    // fill in the wrap-around vertices in the ghost cells

    torus_vertex (l1-1, ..) = torus_vertex (h1 ,..);
    torus_vertex (h1+1, ..) = torus_vertex (l1,..);
    torus_vertex (.., l2-1) = torus_vertex (.., h2);
    torus_vertex (.., h2+1) = torus_vertex (.., l2);

  // =======================================================
  // generate graph corresponding to a two dimensional torus
  // =======================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights ( MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

  // ===============================================
  // compute graph of an approximately cubical three
  // dimensional torus with  2^SCALE vertices, then
  // execute Kernels 2, 3 and 4 of SSCA #2
  // ===============================================

  proc generate_and_analyze_3D_torus {

    const lg2d1 : int = SCALE / 3,
          lg2d2 : int = lg2d1,
          lg2d3 = SCALE - ( lg2d1 + lg2d2);

    const d1 = 2**lg2d1,
          d2 = 2**lg2d2,
          d3 = 2**lg2d3;

    const vertex_domain =
     if DISTRIBUTION_TYPE == "BLOCK" then
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3 }
	 dmapped Block ( { Torus_Base_Index..#d1,
			   Torus_Base_Index..#d2,
			   Torus_Base_Index..#d3 } )
     else
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3 };

    type  vertex = index (vertex_domain);
    const dense_stencil = {-1..1, -1..1, -1..1};
    const torus_stencil : sparse subdomain (dense_stencil) =
      ( (-1, 0, 0 ), (1, 0, 0), (0, -1, 0), (0, 1, 0), (0, 0, -1), (0, 0, 1) );

    var G = new torus_explicit_stencil_graph ( vertex_domain, torus_stencil );

    writeln ( "--------------------------------" );
    writeln ( "Three D Torus with ", d1, " x ", d2, " x ", d3,
	      " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h1 = G.vertices.high (1), l1 = G.vertices.low (1);
    const h2 = G.vertices.high (2), l2 = G.vertices.low (2);
    const h3 = G.vertices.high (3), l3 = G.vertices.low (3);
    var   torus_vertex : [extended_domain] vertex = extended_domain;

    if DEBUG_GRAPH_GENERATOR then {
      writeln ();
      writeln ( "Three Dimension Torus");
      writeln ( "   index lower bounds: ", l1, ", ", l2, ", ", l3 );
      writeln ( "   index upper bounds: ", h1, ", ", h2, ", ", h3);
      writeln ();
      writeln ("vertex       Neighbors");
    }

    // torus_vertex wraps the torus with an additional plane in each dimension
    // fill in the wrap-around vertices in the ghost cells

    torus_vertex (l1-1, .., ..) = torus_vertex (h1, .., ..);
    torus_vertex (h1+1, .., ..) = torus_vertex (l1, .., ..);
    torus_vertex (.., l2-1, ..) = torus_vertex (.., h2, ..);
    torus_vertex (.., h2+1, ..) = torus_vertex (.., l2, ..);
    torus_vertex (.., .., l3-1) = torus_vertex (.., .., h3);
    torus_vertex (.., .., h3+1) = torus_vertex (.., .., l3);

  // =========================================================
  // generate graph corresponding to a three dimensional torus
  // =========================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights ( MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

  // ============================================================
  // compute graph of an approximately four-cubical torus with
  // 2^SCALE vertices, then execute Kernels 2, 3 and 4 of SSCA #2
  // ============================================================

  proc generate_and_analyze_4D_torus {

    const lg2d1 : int = SCALE / 4,
          lg2d2 : int = lg2d1,
          lg2d3 : int = lg2d1,
          lg2d4 = SCALE - ( lg2d1 + lg2d2 + lg2d3);

    const d1 = 2**lg2d1,
          d2 = 2**lg2d2,
          d3 = 2**lg2d3,
          d4 = 2**lg2d4;

    const vertex_domain =
     if DISTRIBUTION_TYPE == "BLOCK" then
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3,
	 Torus_Base_Index..#d4 }
	 dmapped Block ( { Torus_Base_Index..#d1,
			   Torus_Base_Index..#d2,
			   Torus_Base_Index..#d3,
			   Torus_Base_Index..#d4 })
     else
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3,
	 Torus_Base_Index..#d4 };

    type  vertex = index (vertex_domain);
    const dense_stencil = {-1..1, -1..1, -1..1, -1..1};
    const torus_stencil : sparse subdomain (dense_stencil) =
      ( (-1, 0, 0, 0), (1, 0, 0, 0), (0, -1, 0, 0), (0, 1, 0, 0),
	(0, 0, -1, 0), (0, 0, 1, 0), (0, 0, 0, -1), (0, 0, 0, 1) );

    var G = new torus_explicit_stencil_graph ( vertex_domain, torus_stencil );

    writeln ( "--------------------------------" );
    writeln ( "Four D Torus with ", d1, " x ", d2,
	      " x ", d3, " x ", d4, " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h1 = G.vertices.high (1), l1 = G.vertices.low (1);
    const h2 = G.vertices.high (2), l2 = G.vertices.low (2);
    const h3 = G.vertices.high (3), l3 = G.vertices.low (3);
    const h4 = G.vertices.high (4), l4 = G.vertices.low (4);
    var   torus_vertex : [extended_domain] vertex = extended_domain;

      if DEBUG_GRAPH_GENERATOR then {
	writeln ();
	writeln ( "Four Dimension Torus");
	writeln ( "   index lower bounds: ", l1, ", ", l2, ", ",
		  l3, ", ", l4 );
	writeln ( "   index upper bounds: ", h1, ", ", h2, ", ",
		  h3, ", ", h4);
	writeln ();
	writeln ("vertex          Neighbors");}

    // torus_vertex wraps the torus with an additional plane in each dimension
    // fill in the wrap-around vertices in the ghost cells

    torus_vertex (l1-1, .., .., ..) = torus_vertex (h1, .., .., ..);
    torus_vertex (h1+1, .., .., ..) = torus_vertex (l1, .., .., ..);
    torus_vertex (.., l2-1, .., ..) = torus_vertex (.., h2, .., ..);
    torus_vertex (.., h2+1, .., ..) = torus_vertex (.., l2, .., ..);
    torus_vertex (.., .., l3-1, ..) = torus_vertex (.., .., h3, ..);
    torus_vertex (.., .., h3+1, ..) = torus_vertex (.., .., l3, ..);
    torus_vertex (.., .., .., l4-1) = torus_vertex (.., .., .., h4);
    torus_vertex (.., .., .., h4+1) = torus_vertex (.., .., .., l4);

  // ========================================================
  // generate graph corresponding to a four dimensional torus
  // ========================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights ( MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

}

module analyze_torus_graphs {

  // +========================================================================+
  // |  Define sparse subdomain stencil representations for                   |
  // |  k-dimensional torus graphs. For each k, provide execution template    |
  // |  for executing and verifying SSCA2 kernels 2 through 4.                |
  // +========================================================================+
  // |                          VERSION 1                                     |
  // |  Explicit neighbor lists as sparse arithmetic arrays                   |
  // |  defined over the torus stencil as a sparse domain.  Storage overhead  |
  // |  is as many tuples as nonzeros.                                        |
  // +========================================================================+

  use SSCA2_compilation_config_params,
      SSCA2_execution_config_consts,
      SSCA2_driver,
      torus_graph_generator_utilities,
      BlockDist;

  // =====================================================================
  // generic class for stencil-based graph for torus of any dimensionality
  // =====================================================================

  class torus_explicit_stencil_graph {

    const vertex_domain;
    const torus_stencil;
    param dimensions = vertex_domain.rank;

    const vertices = vertex_domain;
    type  vertex = index (vertex_domain);

    var Neighbors   : [vertex_domain] [torus_stencil] vertex;
    var edge_weight : [vertex_domain] [torus_stencil] int;

    proc num_edges {
      return + reduce [v in vertex_domain] n_Neighbors (v);
    }

    iter FilteredNeighbors( v : index (vertices) ) {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil do
        if (FILTERING && weights(n)%8 != 0) || !FILTERING then
          yield neighbors(n);
    }

    // Simply forward the domain's parallel iterator
    iter FilteredNeighbors( v : index (vertices), param tag: iterKind)
    where tag == iterKind.leader {
      for block in torus_stencil._value.these(tag) do
        yield block;
    }

    iter FilteredNeighbors( v : index (vertices), param tag: iterKind, followThis)
    where tag == iterKind.follower {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil._value.these(tag, followThis) do
        if (FILTERING && weights(n)%8 != 0) || !FILTERING then
          yield neighbors(n);
    }

    // iterate over all neighbor (ID, weight) pairs

    iter NeighborPairs( v : index (vertices) ) {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil do
        yield (neighbors(n), weights(n));
    }

    // Simply forward the domain's parallel iterator
    iter NeighborPairs( v : index (vertices), param tag: iterKind)
    where tag == iterKind.leader {
      for block in torus_stencil._value.these(tag) do
        yield block;
    }

    iter NeighborPairs( v : index (vertices), param tag: iterKind, followThis)
    where tag == iterKind.follower {
      const neighbors => Neighbors[v];
      const weights => edge_weight[v];
      for n in torus_stencil._value.these(tag, followThis) do
        yield (neighbors(n), weights(n));
    }

    proc n_Neighbors (v : index (vertices) ) return 2*dimensions;
  }

  // ====================================================
  // compute graph of one dimensional torus with  2^SCALE
  // vertices, then execute Kernels 2, 3 and 4 of SSCA #2
  // ====================================================

  proc generate_and_analyze_1D_torus {

    const d = 2**SCALE;

    const vertex_domain =
      if DISTRIBUTION_TYPE == "BLOCK" then
        { Torus_Base_Index..#d } dmapped Block ( {Torus_Base_Index..#d} )
      else
        { Torus_Base_Index..#d };

    type  vertex = index (vertex_domain);

    const dense_stencil = {-1..1};
    const torus_stencil : sparse subdomain (dense_stencil) = ( -1, 1 );

    var G = new torus_explicit_stencil_graph ( vertex_domain,
					       torus_stencil );

    const exact_between_centrality = 0.25 * (d - 2) * d  -  d/2 + 1;

    writeln ();
    writeln ("One D Torus: all nodes should have",
	     " exact betweenness centrality: ", exact_between_centrality );
    writeln ("             (upper bound on approximate or filtered",
                         " betweenness centrality)");
    writeln ();

    writeln ( "--------------------------------" );
    writeln ( "One D Torus with ", d, " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h = G.vertices.high, l = G.vertices.low;
    var   torus_vertex : [extended_domain] vertex = extended_domain;

    if DEBUG_GRAPH_GENERATOR then {
      writeln ();
      writeln ( "One Dimension Torus");
      writeln ( "   index lower bound: ", l );
      writeln ( "   index upper bound: ", h );
      writeln ();
      writeln ("vertex   Neighbors");}

    // torus_vertex wraps the torus with an additional plane in each
    //  dimension.  fill in the wrap-around vertices in the ghost cells

    torus_vertex (l-1) = torus_vertex (h);
    torus_vertex (h+1) = torus_vertex (l);

    // =======================================================
    // generate graph corresponding to a one dimensional torus
    // =======================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights (MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

  // ==============================================
  // compute graph of an approximately square two
  // dimensional torus with  2^SCALE vertices, then
  // execute Kernels 2, 3 and 4 of SSCA #2
  // ==============================================

  proc generate_and_analyze_2D_torus {

    const lg2d1 : int = SCALE / 2,
          lg2d2 = SCALE - lg2d1,
          d1    = 2**lg2d1,
          d2    = 2**lg2d2;

    const vertex_domain =
     if DISTRIBUTION_TYPE == "BLOCK" then
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2 }
	 dmapped Block ( { Torus_Base_Index..#d1,
			   Torus_Base_Index..#d2 } )
     else
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2 };

    type  vertex = index (vertex_domain);

    const dense_stencil = {1..1,-1..1};
    const torus_stencil : sparse subdomain (dense_stencil) =
                                        ( (-1,0), (1,0), (0,-1), (0,1) );

    var G = new torus_explicit_stencil_graph ( vertex_domain, torus_stencil );

    var exact_between_centrality : real;

    if ( SCALE % 2 == 1 ) then
      exact_between_centrality = 0.75 * 2 ** ( (3*SCALE-1) / 2 )
                                 - 2**SCALE + 1;
    else
      exact_between_centrality = 2 ** ( (3*SCALE/2) - 1)
	                         - 2**SCALE + 1;

    writeln ();
    writeln ("Two D Torus: all nodes should have",
	     " exact betweenness centrality: ", exact_between_centrality );
    writeln ("             (upper bound on approximate or filtered",
                         " betweenness centrality)");
    writeln ();

    writeln ( "--------------------------------" );
    writeln ( "Two D Torus with ", d1, " x ", d2, " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h1 = G.vertices.high (1), l1 = G.vertices.low (1);
    const h2 = G.vertices.high (2), l2 = G.vertices.low (2);
    var   torus_vertex : [extended_domain] vertex = extended_domain;

    if DEBUG_GRAPH_GENERATOR then {
      writeln ();
      writeln ( "Two Dimension Torus");
      writeln ( "   index lower bounds: ", l1, ", ", l2 );
      writeln ( "   index upper bounds: ", h1, ", ", h2 );
      writeln ();
      writeln ("vertex    Neighbors");
    }

    // torus_vertex wraps the torus with an additional plane in each dimension
    // fill in the wrap-around vertices in the ghost cells

    torus_vertex (l1-1, ..) = torus_vertex (h1 ,..);
    torus_vertex (h1+1, ..) = torus_vertex (l1,..);
    torus_vertex (.., l2-1) = torus_vertex (.., h2);
    torus_vertex (.., h2+1) = torus_vertex (.., l2);

  // =======================================================
  // generate graph corresponding to a two dimensional torus
  // =======================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights ( MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

  // ===============================================
  // compute graph of an approximately cubical three
  // dimensional torus with  2^SCALE vertices, then
  // execute Kernels 2, 3 and 4 of SSCA #2
  // ===============================================

  proc generate_and_analyze_3D_torus {

    const lg2d1 : int = SCALE / 3,
          lg2d2 : int = lg2d1,
          lg2d3 = SCALE - ( lg2d1 + lg2d2);

    const d1 = 2**lg2d1,
          d2 = 2**lg2d2,
          d3 = 2**lg2d3;

    const vertex_domain =
     if DISTRIBUTION_TYPE == "BLOCK" then
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3 }
	 dmapped Block ( { Torus_Base_Index..#d1,
			   Torus_Base_Index..#d2,
			   Torus_Base_Index..#d3 } )
     else
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3 };

    type  vertex = index (vertex_domain);
    const dense_stencil = {-1..1, -1..1, -1..1};
    const torus_stencil : sparse subdomain (dense_stencil) =
      ( (-1, 0, 0 ), (1, 0, 0), (0, -1, 0), (0, 1, 0), (0, 0, -1), (0, 0, 1) );

    var G = new torus_explicit_stencil_graph ( vertex_domain, torus_stencil );

    writeln ( "--------------------------------" );
    writeln ( "Three D Torus with ", d1, " x ", d2, " x ", d3,
	      " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h1 = G.vertices.high (1), l1 = G.vertices.low (1);
    const h2 = G.vertices.high (2), l2 = G.vertices.low (2);
    const h3 = G.vertices.high (3), l3 = G.vertices.low (3);
    var   torus_vertex : [extended_domain] vertex = extended_domain;

    if DEBUG_GRAPH_GENERATOR then {
      writeln ();
      writeln ( "Three Dimension Torus");
      writeln ( "   index lower bounds: ", l1, ", ", l2, ", ", l3 );
      writeln ( "   index upper bounds: ", h1, ", ", h2, ", ", h3);
      writeln ();
      writeln ("vertex       Neighbors");
    }

    // torus_vertex wraps the torus with an additional plane in each dimension
    // fill in the wrap-around vertices in the ghost cells

    torus_vertex (l1-1, .., ..) = torus_vertex (h1, .., ..);
    torus_vertex (h1+1, .., ..) = torus_vertex (l1, .., ..);
    torus_vertex (.., l2-1, ..) = torus_vertex (.., h2, ..);
    torus_vertex (.., h2+1, ..) = torus_vertex (.., l2, ..);
    torus_vertex (.., .., l3-1) = torus_vertex (.., .., h3);
    torus_vertex (.., .., h3+1) = torus_vertex (.., .., l3);

  // =========================================================
  // generate graph corresponding to a three dimensional torus
  // =========================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights ( MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

  // ============================================================
  // compute graph of an approximately four-cubical torus with
  // 2^SCALE vertices, then execute Kernels 2, 3 and 4 of SSCA #2
  // ============================================================

  proc generate_and_analyze_4D_torus {

    const lg2d1 : int = SCALE / 4,
          lg2d2 : int = lg2d1,
          lg2d3 : int = lg2d1,
          lg2d4 = SCALE - ( lg2d1 + lg2d2 + lg2d3);

    const d1 = 2**lg2d1,
          d2 = 2**lg2d2,
          d3 = 2**lg2d3,
          d4 = 2**lg2d4;

    const vertex_domain =
     if DISTRIBUTION_TYPE == "BLOCK" then
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3,
	 Torus_Base_Index..#d4 }
	 dmapped Block ( { Torus_Base_Index..#d1,
			   Torus_Base_Index..#d2,
			   Torus_Base_Index..#d3,
			   Torus_Base_Index..#d4 })
     else
       { Torus_Base_Index..#d1,
	 Torus_Base_Index..#d2,
	 Torus_Base_Index..#d3,
	 Torus_Base_Index..#d4 };

    type  vertex = index (vertex_domain);
    const dense_stencil = {-1..1, -1..1, -1..1, -1..1};
    const torus_stencil : sparse subdomain (dense_stencil) =
      ( (-1, 0, 0, 0), (1, 0, 0, 0), (0, -1, 0, 0), (0, 1, 0, 0),
	(0, 0, -1, 0), (0, 0, 1, 0), (0, 0, 0, -1), (0, 0, 0, 1) );

    var G = new torus_explicit_stencil_graph ( vertex_domain, torus_stencil );

    writeln ( "--------------------------------" );
    writeln ( "Four D Torus with ", d1, " x ", d2,
	      " x ", d3, " x ", d4, " nodes" );
    writeln ( "explicit stencil representation");
    writeln ( "--------------------------------" );
    writeln ();

    const extended_domain = vertex_domain.expand(1);
    const h1 = G.vertices.high (1), l1 = G.vertices.low (1);
    const h2 = G.vertices.high (2), l2 = G.vertices.low (2);
    const h3 = G.vertices.high (3), l3 = G.vertices.low (3);
    const h4 = G.vertices.high (4), l4 = G.vertices.low (4);
    var   torus_vertex : [extended_domain] vertex = extended_domain;

      if DEBUG_GRAPH_GENERATOR then {
	writeln ();
	writeln ( "Four Dimension Torus");
	writeln ( "   index lower bounds: ", l1, ", ", l2, ", ",
		  l3, ", ", l4 );
	writeln ( "   index upper bounds: ", h1, ", ", h2, ", ",
		  h3, ", ", h4);
	writeln ();
	writeln ("vertex          Neighbors");}

    // torus_vertex wraps the torus with an additional plane in each dimension
    // fill in the wrap-around vertices in the ghost cells

    torus_vertex (l1-1, .., .., ..) = torus_vertex (h1, .., .., ..);
    torus_vertex (h1+1, .., .., ..) = torus_vertex (l1, .., .., ..);
    torus_vertex (.., l2-1, .., ..) = torus_vertex (.., h2, .., ..);
    torus_vertex (.., h2+1, .., ..) = torus_vertex (.., l2, .., ..);
    torus_vertex (.., .., l3-1, ..) = torus_vertex (.., .., h3, ..);
    torus_vertex (.., .., h3+1, ..) = torus_vertex (.., .., l3, ..);
    torus_vertex (.., .., .., l4-1) = torus_vertex (.., .., .., h4);
    torus_vertex (.., .., .., h4+1) = torus_vertex (.., .., .., l4);

  // ========================================================
  // generate graph corresponding to a four dimensional torus
  // ========================================================

    forall i in G.vertices do {
      forall nbr in torus_stencil do
	G.Neighbors (i) (nbr) = torus_vertex ( i + nbr );

      if DEBUG_GRAPH_GENERATOR then
	writeln ( i, ":    ", G.Neighbors (i) );
    }

    torus_random_edge_weights ( MAX_EDGE_WEIGHT, G );

    execute_SSCA2 ( G );
    delete G;

    writeln (); writeln ();
  }

}


Added another idea for how to implement async buffers (Like Engin mentioned)...

use CyclicDist;
use local_lock_free_queue;

/*
  A distributed FIFO queue.

  The queue makes use of a counter to cycle and to properly load-balance across
  all registered locales, where each locale has a per-locale queue. The per-locale
  queue further ensures a FIFO ordering, ensuring that any extra items enqueued
  after cycling through can be dequeued in the same order. This is because a FIFO
  of FIFOs is still a FIFO. For a useful diagram...

  Imagine there are N locales, where we denote a head at position Y as H_Y and a
  tail at position Z as T_Z.

  The Queue is empty:
    H_0, T_0 -- Note that the position just needs to be the same...

    In this case, if the head and tail are equal, then a dequeue will fail if it was read
    as such at the time, but an enqueue would be able to advance the tail.

  Enqueue:
    H_0, T_1

    In this case, the locale at position 0 mod N would end up with an enqueue.

  Dequeue:

    H_1, T_1

    There are two things that can happen: T_0 has not yet finished being enqueued,
    upon which it will spin until it has finished (I.E: by repeatedly attempting to dequeue
    if it is non-blocking), or it has finished, upon which will do a simple dequeue.
    Notice as well that while both Enqueue and Dequeue can be in progress, other tasks
    can advance the head and tail to obtain their next index. Overlapping operations are
    discussed below.

  The position can be obtained and updated in a non-blocking fashion, using the
  available atomic primitives such as Compare-And-Swap, of which is the only global
  synchronization actually needed. Since the only global synchronization is non-blocking,
  lock-free, and wait-free, this ensures scalability. To reduce communication as well,
  we perform atomically updating and obtaining the index on the locale owning the queue.
  The head and tail of the global counter denotes a sort of 'promise' that a task will,
  on enqueue add a new item, and on dequeue remove an existing item. Given that the global counter
  is linearizable, we know that the index obtained from the current counter is correct at the time
  it was retrieved, and enforcing that all tasks uphold such a promise will ensure it always remains
  correct.

  As each per-locale queue is updated on the locale it belongs to, meaning majority
  of the computation can be performed remotely, it ensures that the workload is very
  well balanced. In the cases of overlapping operations on a local queue, a non-blocking
  queue is the most optimal, as this would allow enqueues to be fully non-blocking and
  lock-free, as the counter is proven non-blocking, lock-free, and wait-free.
  While enqueues can be non-blocking (depending on queue implementation), a dequeues
  cannot be as it is possible for an enqueue to 'promise' but not finish before another
  dequeue on that node occurs. Due to this, a dequeue would need to spin as the index
  assigned to it is correct and that *one* of the enqueuers for that index has not completed
  but will eventually. In this regard, dequeue is *mostly* non-blocking.

  In terms of correctness for the per-locale queues, we know that we have a 'promise'
  to perform some task, but beyond that it is not guaranteed that the order that
  tasks that received their indices will perform them in the same order. That is
  if a task t1 obtained its indice before t2, and both t1 and t2 hash to the same index (read: locale),
  it is possible that t2 can complete its operation before t1, therefore it is non-deterministic
  in terms of order of completion. However, because overlapping concurrent operations
  are non-deterministic in nature, and that the FIFO ordering is in fact preserved on some
  level (on the per-locale queue), it is still a valid operation and transparent from the outside.
  Considering that PGAS ensures that multi-locale operations finish before continuing, we can
  verify that indeed an operation will finish in-order on a sequential task. The only way for
  operations to overlap is through overlapping tasks. Compare to a normal synchronized queue,
  and even if it is possible that t1 began its operation before t2, it is still possible for t2
  to obtain the lock before t1 in cases of unexpected delays. Therefore, the non-deterministic
  nature of the per-locale queues are nothing special, and if anything increase overall concurrency.

  Per-locale buffering, which allows the user to maintain a collection of items to be added later,
  can significantly reduce the overall number of communication calls. For example, given N locales,
  and M elements to be buffered, normally we would have O(M) calls, as each item would need to be
  individually added to each of the per-locale queues, while buffering allows for only O(N) calls
  given that we add at most M/N elements for each locale. As well, buffering and adding in bulk
  bypasses the need for contesting for the counter O(M) times to just O(1) as we make a 'promise'
  to add 'M' elements. Furthermore, we can make quite a few optimizations, such as the per-locale queues
  using bulk operations.

  Per-locale buffering would require a second set of per-locale queues, of which can be enqueued to
  maintain the same FIFO ordering, and with a non-blocking queue it becomes non-blocking and scalable
  among multiple tasks. To empty the buffer, we can repeatedly dequeue until we run out, assigning
  each value to some container, maintaining its insertion order; this ensures the FIFO ordering remains
  and that it allows other concurrent last-minute enqueuers to submit their own without needing to block.
  When it is time to drain the buffer (concurrently) and add to each queue, we make a promise based on the
  number of elements removed, and the elements are added in FIFO order. We also utilize a local 'flush lock'
  to ensure the buffer is only flushed by one local writer, as otherwise it can violate the overall FIFO ordering.
  When elements are added in parallel, it ensures that all PGAS communications begin and become deferred to the other
  locales that own them.

  Per-locale buffering does not come without potential issues. In the case where we 'promise' more than one element,
  it is possible for a task to receive an item later in the queue than it should. For example, if we have
  8 locales, 32 elements promised, we would have 4 elements being added in bulk each. A task that is waiting
  on the the 32nd element can return earlier than a task waiting for the 1st element based on which operation finishes
  first. This non-deterministic behavior may seem alarming at first, but it should be noted that this can only occur
  on overlapping operations. The only way to obtain the 32nd item is to have 32 consumers concurrently, for example.
  This non-deterministic behavior is inherent in any queue that releases their lock before returning
  an item, as it is possible, again, for all other threads to be preempted before 32nd. Again, this non-deterministic
  behavior is inherent in all concurrent data structures, and is used to promote further scalability by allowing
  concurrent operations even for bulk insertion.

*/
class Distributed_FIFO {
  type eltType;

  // Our position counter: Upper 32 bits = Head, Lower 32 bits = Tail
  var position : atomic uint(64);

  // Our domain: We use a cyclic distribution
  var domainMapping = {1 .. numLocales};
  var cyclicDomain = domainMapping dmapped Cyclic(startIdx=1);
  var localQueues : [cyclicDomain] queue(eltType);

  proc Distributed_FIFO(type eltType) {
    coforall i in 1..numLocales do {
      var q = new queue(eltType);
      q.initialize();
      localQueues[i] = q;
    }
  }

  proc enqueue(elem : eltType) {
    // The index we are going to be working on...
    var idx : uint(64);

    // Position is queried on the locale that owns the queue
    on position do {
      var pos : uint(64);
      var head, tail : uint(64);

      // Obtain our desired position
      while (true) {
        pos = position.read();
        head = pos & 0xFFFFFFFF00000000;
        tail = pos & 0x00000000FFFFFFFF;

        // Some platforms yield higher performance due to reduced memory contention
        // under concurrent writes to the same cache line. We update the position
        // atomically.
        if (position.compareExchangeWeak(pos, head | (tail + 1))) {
          break;
        }
      }

      // Translates to a single network call to write to the orginal locale's index
      var nLocales : uint(64) = numLocales : uint(64);
      idx = max(1, tail % nLocales);

      /*writeln("Head: ", head, "; Tail: ", tail);*/
    }

    /*writeln(here, ": Assigned index: ", idx);*/

    // Now we add our item
    ref queue = localQueues[idx : int(64)];
    on queue do queue.enqueue(elem);
  }

  proc dequeue() : (bool, eltType) {
    // The index we are going to be working on...
    var idx : uint(64);
    var hasElem : bool = true;

    // Position is queried on the locale that owns the queue
    on position do {
      var pos : uint(64);
      var head, tail : uint(64);
      var localHasElem : bool = true;

      // Obtain our desired position
      while (true) {
        pos = position.read();
        head = pos & 0xFFFFFFFF00000000;
        tail = pos & 0x00000000FFFFFFFF;

        if ((head >> 32) == tail) {
          /*writeln("Queue is empty... Pos: ", pos);*/
          localHasElem = false;
          break;
        }

        // Some platforms yield higher performance due to reduced memory contention
        // under concurrent writes to the same cache line. We update the position
        // atomically.
        if (position.compareExchangeWeak(pos, (head + (1 << 32)) | (tail))) {
          break;
        }
      }

      if localHasElem {
        var nLocales : uint(64) = numLocales : uint(64);
        // Translates to a single network call to write to the orginal locale's index
        idx = max(1, head % nLocales);

        /*writeln("Head: ", head >> 32, "; Tail: ", tail);*/
      }

      hasElem = localHasElem;
    }

    if !hasElem {
      var dummy : eltType;
      return (false, dummy);
    }

    /*writeln(here, ": Assigned index: ", idx);*/

    // Now we get our item from the queue
    // Note that at the index given, its possible that an enqueueing task has not
    // finished yet, but we know there *should* be at least something for us, so we can
    // spin until it has what we want.
    ref queue = localQueues[idx : int(64)];
    var elem : eltType;
    on queue do {
      var retval : (bool, eltType);
      while !retval[1] {
        retval = queue.dequeue();

        if (!retval[1]) {
          writeln(here, ": Spinning... HasElem: ", hasElem, ";");
        }
      }

      // We have our value... this also translates into a network call (PUT)
      elem = retval[2];
    }

    return (true, elem);
  }
}

use CyclicDist;
use local_lock_free_queue;

/*
  A distributed FIFO queue.

  The queue makes use of a counter to cycle and to properly load-balance across
  all registered locales, where each locale has a per-locale queue. The per-locale
  queue further ensures a FIFO ordering, ensuring that any extra items enqueued
  after cycling through can be dequeued in the same order. This is because a FIFO
  of FIFOs is still a FIFO. For a useful diagram...

  Imagine there are N locales, where we denote a head at position Y as H_Y and a
  tail at position Z as T_Z.

  The Queue is empty:
    H_0, T_0 -- Note that the position just needs to be the same...

    In this case, if the head and tail are equal, then a dequeue will fail if it was read
    as such at the time, but an enqueue would be able to advance the tail.

  Enqueue:
    H_0, T_1

    In this case, the locale at position 0 mod N would end up with an enqueue.

  Dequeue:

    H_1, T_1

    There are two things that can happen: T_0 has not yet finished being enqueued,
    upon which it will spin until it has finished (I.E: by repeatedly attempting to dequeue
    if it is non-blocking), or it has finished, upon which will do a simple dequeue.
    Notice as well that while both Enqueue and Dequeue can be in progress, other tasks
    can advance the head and tail to obtain their next index. Overlapping operations are
    discussed below.

  The position can be obtained and updated in a non-blocking fashion, using the
  available atomic primitives such as Compare-And-Swap, of which is the only global
  synchronization actually needed. Since the only global synchronization is non-blocking,
  lock-free, and wait-free, this ensures scalability. To reduce communication as well,
  we perform atomically updating and obtaining the index on the locale owning the queue.
  The head and tail of the global counter denotes a sort of 'promise' that a task will,
  on enqueue add a new item, and on dequeue remove an existing item. Given that the global counter
  is linearizable, we know that the index obtained from the current counter is correct at the time
  it was retrieved, and enforcing that all tasks uphold such a promise will ensure it always remains
  correct.

  As each per-locale queue is updated on the locale it belongs to, meaning majority
  of the computation can be performed remotely, it ensures that the workload is very
  well balanced. In the cases of overlapping operations on a local queue, a non-blocking
  queue is the most optimal, as this would allow enqueues to be fully non-blocking and
  lock-free, as the counter is proven non-blocking, lock-free, and wait-free.
  While enqueues can be non-blocking (depending on queue implementation), a dequeues
  cannot be as it is possible for an enqueue to 'promise' but not finish before another
  dequeue on that node occurs. Due to this, a dequeue would need to spin as the index
  assigned to it is correct and that *one* of the enqueuers for that index has not completed
  but will eventually. In this regard, dequeue is *mostly* non-blocking.

  In terms of correctness for the per-locale queues, we know that we have a 'promise'
  to perform some task, but beyond that it is not guaranteed that the order that
  tasks that received their indices will perform them in the same order. That is
  if a task t1 obtained its indice before t2, and both t1 and t2 hash to the same index (read: locale),
  it is possible that t2 can complete its operation before t1, therefore it is non-deterministic
  in terms of order of completion. However, because overlapping concurrent operations
  are non-deterministic in nature, and that the FIFO ordering is in fact preserved on some
  level (on the per-locale queue), it is still a valid operation and transparent from the outside.
  Considering that PGAS ensures that multi-locale operations finish before continuing, we can
  verify that indeed an operation will finish in-order on a sequential task. The only way for
  operations to overlap is through overlapping tasks. Compare to a normal synchronized queue,
  and even if it is possible that t1 began its operation before t2, it is still possible for t2
  to obtain the lock before t1 in cases of unexpected delays. Therefore, the non-deterministic
  nature of the per-locale queues are nothing special, and if anything increase overall concurrency.

  Non-Blocking Per-Locale Queue:
    + Allows enqueues to be entirely lock-free
      - Dequeues may need to spin if an enqueue has not completed
    + Possibly the 'optimal' solution
    - Difficult to implement
      - Hazard Pointers for memory reclamation
      - (Very Rare) ABA problem for memory allocation
        - Could require MCAS algorithm to correctly implement
  Two-Locked Per-Locale Queue:
    + Allows for a single enqueue and dequeue concurrent operation
    + Practical for majority of use-cases
    + Very straight-forward to implement
      + Memory reclamation is simple, correctness easy to prove
*/
class Distributed_FIFO {
  type eltType;

  // Our position counter: Upper 32 bits = Head, Lower 32 bits = Tail
  var position : atomic uint(64);

  // Our domain: We use a cyclic distribution
  var domainMapping = {1 .. numLocales};
  var cyclicDomain = domainMapping dmapped Cyclic(startIdx=1);
  var localQueues : [cyclicDomain] queue(eltType);

  proc Distributed_FIFO(type eltType) {
    coforall i in 1..numLocales do {
      var q = new queue(eltType);
      q.initialize();
      localQueues[i] = q;
    }
  }

  proc enqueue(elem : eltType) {
    // The index we are going to be working on...
    var idx : uint(64);

    // Position is queried on the locale that owns the queue
    on position do {
      var pos : uint(64);
      var head, tail : uint(64);

      // Obtain our desired position
      while (true) {
        pos = position.read();
        head = pos & 0xFFFFFFFF00000000;
        tail = pos & 0x00000000FFFFFFFF;

        // Some platforms yield higher performance due to reduced memory contention
        // under concurrent writes to the same cache line. We update the position
        // atomically.
        if (position.compareExchangeWeak(pos, head | (tail + 1))) {
          break;
        }
      }

      // Translates to a single network call to write to the orginal locale's index
      var nLocales : uint(64) = numLocales : uint(64);
      idx = max(1, tail % nLocales);

      /*writeln("Head: ", head, "; Tail: ", tail);*/
    }

    /*writeln(here, ": Assigned index: ", idx);*/

    // Now we add our item
    ref queue = localQueues[idx : int(64)];
    on queue do queue.enqueue(elem);
  }

  proc dequeue() : (bool, eltType) {
    // The index we are going to be working on...
    var idx : uint(64);
    var hasElem : bool = true;

    // Position is queried on the locale that owns the queue
    on position do {
      var pos : uint(64);
      var head, tail : uint(64);
      var localHasElem : bool = true;

      // Obtain our desired position
      while (true) {
        pos = position.read();
        head = pos & 0xFFFFFFFF00000000;
        tail = pos & 0x00000000FFFFFFFF;

        if ((head >> 32) == tail) {
          /*writeln("Queue is empty... Pos: ", pos);*/
          localHasElem = false;
          break;
        }

        // Some platforms yield higher performance due to reduced memory contention
        // under concurrent writes to the same cache line. We update the position
        // atomically.
        if (position.compareExchangeWeak(pos, (head + (1 << 32)) | (tail))) {
          break;
        }
      }

      if localHasElem {
        var nLocales : uint(64) = numLocales : uint(64);
        // Translates to a single network call to write to the orginal locale's index
        idx = max(1, head % nLocales);

        /*writeln("Head: ", head >> 32, "; Tail: ", tail);*/
      }

      hasElem = localHasElem;
    }

    if !hasElem {
      var dummy : eltType;
      return (false, dummy);
    }

    /*writeln(here, ": Assigned index: ", idx);*/

    // Now we get our item from the queue
    // Note that at the index given, its possible that an enqueueing task has not
    // finished yet, but we know there *should* be at least something for us, so we can
    // spin until it has what we want.
    ref queue = localQueues[idx : int(64)];
    var elem : eltType;
    on queue do {
      var retval : (bool, eltType);
      while !retval[1] {
        retval = queue.dequeue();

        if (!retval[1]) {
          writeln(here, ": Spinning... HasElem: ", hasElem, ";");
        }
      }

      // We have our value... this also translates into a network call (PUT)
      elem = retval[2];
    }

    return (true, elem);
  }
}


I missed SSCA2 when I originally checked in the zippered iterator syntax change?

This should correct the failures seen in nightly testing from 9/15 to 9/17.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20459 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  +==========================================================================+

{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      proc maxweight(s) {
	var mymax = 0;
	for w in G.edge_weight[s] do
	  if w > mymax then mymax = w;
	return mymax;
      }

      const heaviest_edge_weight = max reduce [s in G.vertices] maxweight(s);

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in G.NeighborPairs (s) do

	  if w == heaviest_edge_weight then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ", heaviest_edge_weight);
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in G.NeighborPairs (s) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
                                Heavy_Edge_List     : domain,
                                Heavy_Edge_Subgraph : [],
                                in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance             : [vertex_domain] atomic int;
        forall m in min_distance do m.write(-1);

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance ( y ).write(0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do {

              if min_distance(w).compareExchangeStrong(-1, path_length) then {
                Next_Level.add (w);
                Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
	      }

	      if min_distance(w).read() == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  use BlockDist;
  config param useOnClause = CHPL_NETWORK_ATOMICS!="ugni";
  // For task-private temporary variables
  config const defaultNumTPVs = 16;
  config var numTPVs = min(defaultNumTPVs, numLocales);
  // Would be nice to use PriavteDist, but aliasing is not supported (yet)
  const PrivateSpace = {LocaleSpace} dmapped Block(boundingBox={LocaleSpace});

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
                                            Between_Cent : [] real,
                                            out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Considering using a dense 1-d array instead.  This would
      // complicate the Level_Set implementation a little, but would
      // probably be more efficient.
      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] atomic real;
      var Sum_Min_Dist$ : atomic real;

      //
      // Throughout kernel 4, we use distributed arrays that are
      // private to each iteration of the outer loop.  Because set-up
      // and clean-up of such variables is very expensive, it is
      // desirable to do it at most only once per task versus once
      // per iteration, as a task executes multiple iterations of a
      // forall loop.
      //
      // To avoid this per-iteration overhead, we have implemented
      // an optimization that make certain variables private to each
      // task.  These variables are reinitialized each iteration of
      // the loop, but not reallocated, reprivatized, etc.
      //
      // Note that the Chapel group and collaborators are currently in
      // the process of bringing the idea of task private variable to
      // the language.
      //

      numTPVs = min(numTPVs, starting_vertices.size);
      const TPVSpace = {0..#numTPVs};
      var TPVLocales: [TPVSpace] locale;

      // Here we set up the locales array for our task private temporary
      // variables.  We use _computChunkStartEnd() to distribute the
      // variables approximately evenly across the 1d locales array.
      for t in TPVSpace {
        TPVLocales[t] = Locales[_computeChunkStartEnd(numLocales,
                                                      numTPVs, t+1)[1]-1];
      }
      const TPVLocaleSpace = {TPVSpace} dmapped Block(boundingBox={TPVSpace},
                                                      targetLocales=TPVLocales);

      // There will be numTPVs copies of the temps, thus throttling the
      // number of starting vertices being considered simultaneously.
      var TPV: [TPVLocaleSpace] taskPrivateData(domain(index(vertex_domain)),
                                                vertex_domain.type);

      // Initialize
      coforall (t, i) in zip(TPVLocaleSpace, TPVSpace) do on TPVLocaleSpace.dist.idxToLocale(t) {
          TPV[i] = new taskPrivateData(domain(index(vertex_domain)), vertex_domain);
          var tpv = TPV[i];
          forall v in vertex_domain do
            tpv.BCaux[v].children_list.nd = {1..G.n_Neighbors[v]};
          for loc in Locales do on loc {
            tpv.Active_Level[here.id] = new Level_Set (Sparse_Vertex_List);
            tpv.Active_Level[here.id].previous = nil;
            tpv.Active_Level[here.id].next = new Level_Set (Sparse_Vertex_List);;
            tpv.Active_Level[here.id].next.previous = tpv.Active_Level[here.id];
            tpv.Active_Level[here.id].next.next = nil;
          }
      }
      var TPVM: TPVManager(TPV.type) = new TPVManager(TPV);

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      forall s in starting_vertices do on vertex_domain.dist.idxToLocale(s) {

        const shere = here.id;

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

        // Initialize task private variables
        const tid = TPVM.gettid();
        const tpv = TPVM.getTPV(tid);
        var BCaux => tpv.BCaux;
        pragma "dont disable remote value forwarding"
        inline proc f1(BCaux, v) {
          BCaux[v].path_count$.write(0.0);
        }
        forall v in vertex_domain do {
          BCaux[v].depend = 0.0;
          BCaux[v].min_distance.write(-1);
          f1(BCaux, v);
          BCaux[v].children_list.child_count.write(0);
        }
	var Lcl_Sum_Min_Dist: atomic real;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

        //
        // Used to check termination of the forward pass
        //
        // We could use a task-private reduction variable here.  This
        // is yet another concept that the Chapel group is planning to
        // implement.
        //

        var remaining = true;

        //
        // Each locale will have its own level sets.  A locale's level set
        // will only contain nodes that are physically allocated on that
        // particular locale.
        //
        var Active_Level => tpv.Active_Level;
        pragma "dont disable remote value forwarding"
        inline proc f2(BCaux, s) {
          BCaux[s].path_count$.write(1.0);
        }

        var barrier = new Barrier(numLocales);

        coforall loc in Locales do on loc {
          Active_Level[here.id].Members.clear();
          Active_Level[here.id].next.Members.clear();
          if vertex_domain.dist.idxToLocale(s) == here {
            // Establish the initial level sets for the breadth-first
            // traversal from s
            Active_Level[here.id].Members.add(s);
            BCaux[s].min_distance.write(0);
            f2(BCaux, s);
          }
          barrier.barrier();

          var current_distance : int = 0;

          while remaining do {
	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;

            // The Chapel compiler is still a bit conservative when it
            // comes to forwarding read-only variables to remote
            // locales, so we make a const copy here to insure it is
            // forwarded to the remote locale in the following
            // coforall loop.
            const current_distance_c = current_distance;
            pragma "dont disable remote value forwarding"
            inline proc f3(BCaux, v, u) {
              BCaux[v].path_count$.add(BCaux[u].path_count$.read());
            }

            forall u in Active_Level[here.id].Members do {
              forall v in G.FilteredNeighbors(u) do
                on if useOnClause then vertex_domain.dist.idxToLocale(v) else here {
                  // --------------------------------------------
                  // add any unmarked neighbors to the next level
                  // --------------------------------------------

                  if  BCaux[v].min_distance.compareExchangeStrong(-1, current_distance_c) {
                    var aloc = if useOnClause then here.id else vertex_domain.dist.idxToLocale(v).id;
                    Active_Level[aloc].next.Members.add (v);
                    if VALIDATE_BC then
                      Lcl_Sum_Min_Dist.add(current_distance_c);
                  }

                  // ------------------------------------------------
                  // only neighbors of  u  that are in the next level
                  // are on shortest paths from s through v.  Some
                  // task will have set  min_distance (v) by the
                  // time this code is reached, whether  v  lies in
                  // the previous, the current or the next level.
                  // ------------------------------------------------

                  if BCaux[v].min_distance.read() == current_distance_c {
                    f3(BCaux, v, u);
                    BCaux[u].children_list.add_child (v);
                  }

                }
            };

            // This barrier is needed to insure all updates to the next
            // level are completed before updating to use the next level

            // do some work while we wait
            // barrier.notify(); // This is expensive without network atomics
            //  for now, just do a normal barrier

            if Active_Level[here.id].next.next == nil {
              Active_Level[here.id].next.next = new Level_Set (Sparse_Vertex_List);
              Active_Level[here.id].next.next.previous = Active_Level[here.id].next;
              Active_Level[here.id].next.next.next = nil;
            } else {
              Active_Level[here.id].next.next.Members.clear();
            }
            // barrier.wait(); // ditto
            barrier.barrier();
            if here.id==shere {
              remaining = false;
            }

            barrier.barrier();
            Active_Level[here.id] = Active_Level[here.id].next;
            if Active_Level[here.id].Members.numIndices:bool then
              remaining = true;

            barrier.barrier();

          };  // end forward pass

          if VALIDATE_BC then
            if here.id==0 then
              Sum_Min_Dist$.add(Lcl_Sum_Min_Dist);

          // -------------------------------------------------------------
          // compute the dependencies recursively, traversing the vertices
          // of the graph in non-increasing order of distance (reverse
          // ordering from the initial traversal)
          // -------------------------------------------------------------

          const graph_diameter = current_distance - 1;

          if DEBUG_KERNEL4 then
            if here.id==0 then
              writeln ( " graph diameter from starting node ", s,
                        "  is ", graph_diameter );

          pragma "dont disable remote value forwarding"
          inline proc f4(BCaux, Between_Cent$, u) {
            BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
              ( BCaux[u].path_count$.read() /
                BCaux[v].path_count$.read() )      *
              ( 1.0 + BCaux[v].depend );
            Between_Cent$(u).add(BCaux[u].depend);
          }

          // back up to last level
          var curr_Level =  Active_Level[here.id].previous;

          for current_distance in 2 .. graph_diameter by -1 {
            curr_Level = curr_Level.previous;

            for u in curr_Level.Members do on vertex_domain.dist.idxToLocale(u) {
                f4(BCaux, Between_Cent$, u);
            }

            barrier.barrier();
          }
        }

        TPVM.releaseTPV(tid);

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	const n_edges          = G.num_edges;
	const N_VERTICES       = vertex_domain.numIndices;
	const N_START_VERTICES = if starting_vertices == G.vertices
                                 then N_VERTICES
                                      - + reduce [v in vertex_domain]
                                        (G.n_Neighbors (v) == 0)
                                 else starting_vertices.numIndices;
	const TEPS             = 7.0 * N_VERTICES * N_START_VERTICES / K4_time;
	const Adjusted_TEPS    = n_edges * N_START_VERTICES / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
        Sum_Min_Dist = Sum_Min_Dist$.read();

      Between_Cent = Between_Cent$.read();

      if DELETE_KERNEL4_DS {
        coforall t in TPVSpace do on t {
          var tpv = TPV[t];
          var al = tpv.Active_Level;
          coforall loc in Locales do on loc {
            var level = al[here.id];
            while level != nil {
                var l2 = level.next;
                delete level;
                level = l2;
            }
          }
          delete TPV[t];
        }
        delete TPVM;
      }

    } // end of Brandes' betweenness centrality calculation

  //
  // Addition support data structures for kernel 4
  //

  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
    var next : Level_Set (Sparse_Vertex_List);
  }

  //
  // Data structure to save Children lists between the forward
  //  and backwards passes
  //
  record child_struct {
    type vertex;
    var nd: domain(1);
    var Row_Children: [nd] vertex;
    var child_count: atomic int;

    // This function should only be called using unique vertices
    proc add_child ( new_child: vertex ) {
      var c = child_count.fetchAdd(1)+1;
      Row_Children[c] = new_child;
    }
  }

  //
  // Implementation of task-private variables for kernel 4
  //
  record taskPrivateArrayData {
    type vertex;
    var min_distance  : atomic int;
    var path_count$   : atomic real;
    var depend        : real;
    var children_list : child_struct(vertex);
  }

  class taskPrivateData {
    type Sparse_Vertex_List;
    const vertex_domain;
    var used  : atomic bool;
    var BCaux : [vertex_domain] taskPrivateArrayData(index(vertex_domain));
    var Active_Level : [PrivateSpace] Level_Set (Sparse_Vertex_List);
  }

  // This is a simple class that hands out task private variables from the
  // array of task private variables, TPV.
  class TPVManager {
    const TPV;
    var currTPV: atomic int;
    proc gettid() {
      const tid = this.currTPV.fetchAdd(1)%numTPVs;
      on this.TPV[tid] do
        while this.TPV[tid].used.testAndSet() do chpl_task_yield();
      return tid;
    }
    proc getTPV(tid) {
      return this.TPV[tid];
    }
    proc releaseTPV(tid) {
      this.TPV[tid].used.clear();
    }
  }

  //
  // reusable barrier implementation
  //
  record Barrier {
    param reusable = true;
    var n: int;
    var count: atomic_int64;
    var done: atomicflag;

    proc Barrier(_n: int) {
      reset(_n);
    }

    inline proc reset(_n: int) {
      on this {
        n = _n;
        count.write(n);
        done.write(false);
      }
    }

    inline proc barrier() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to barrier()");
          if reusable {
            count.waitFor(n-1);
            count.add(1);
            done.clear();
          }
        } else {
          done.waitFor(true);
          if reusable {
            count.add(1);
            done.waitFor(false);
          }
        }
      }
    }

    inline proc notify() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to notify()");
        }
      }
    }

    inline proc wait() {
      on this {
        done.waitFor(true);
        if reusable {
          const myc = count.fetchAdd(1);
          if myc == n-1 then
            done.clear();
          done.waitFor(false);
        }
      }
    }

    inline proc try() {
      return done.read();
    }
  }

}

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  +==========================================================================+

{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      proc maxweight(s) {
	var mymax = 0;
	for w in G.edge_weight[s] do
	  if w > mymax then mymax = w;
	return mymax;
      }

      const heaviest_edge_weight = max reduce [s in G.vertices] maxweight(s);

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in G.NeighborPairs (s) do

	  if w == heaviest_edge_weight then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ", heaviest_edge_weight);
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in G.NeighborPairs (s) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
                                Heavy_Edge_List     : domain,
                                Heavy_Edge_Subgraph : [],
                                in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance             : [vertex_domain] atomic int;
        forall m in min_distance do m.write(-1);

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance ( y ).write(0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do {

              if min_distance(w).compareExchangeStrong(-1, path_length) then {
                Next_Level.add (w);
                Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
	      }

	      if min_distance(w).read() == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  use BlockDist;
  config param useOnClause = CHPL_NETWORK_ATOMICS!="ugni";
  // For task-private temporary variables
  config const defaultNumTPVs = 16;
  config var numTPVs = min(defaultNumTPVs, numLocales);
  // Would be nice to use PriavteDist, but aliasing is not supported (yet)
  const PrivateSpace = {LocaleSpace} dmapped Block(boundingBox={LocaleSpace});

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
                                            Between_Cent : [] real,
                                            out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Considering using a dense 1-d array instead.  This would
      // complicate the Level_Set implementation a little, but would
      // probably be more efficient.
      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] atomic real;
      var Sum_Min_Dist$ : atomic real;

      //
      // Throughout kernel 4, we use distributed arrays that are
      // private to each iteration of the outer loop.  Because set-up
      // and clean-up of such variables is very expensive, it is
      // desirable to do it at most only once per task versus once
      // per iteration, as a task executes multiple iterations of a
      // forall loop.
      //
      // To avoid this per-iteration overhead, we have implemented
      // an optimization that make certain variables private to each
      // task.  These variables are reinitialized each iteration of
      // the loop, but not reallocated, reprivatized, etc.
      //
      // Note that the Chapel group and collaborators are currently in
      // the process of bringing the idea of task private variable to
      // the language.
      //

      numTPVs = min(numTPVs, starting_vertices.size);
      const TPVSpace = {0..#numTPVs};
      var TPVLocales: [TPVSpace] locale;

      // Here we set up the locales array for our task private temporary
      // variables.  We use _computChunkStartEnd() to distribute the
      // variables approximately evenly across the 1d locales array.
      for t in TPVSpace {
        TPVLocales[t] = Locales[_computeChunkStartEnd(numLocales,
                                                      numTPVs, t+1)[1]-1];
      }
      const TPVLocaleSpace = {TPVSpace} dmapped Block(boundingBox={TPVSpace},
                                                      targetLocales=TPVLocales);

      // There will be numTPVs copies of the temps, thus throttling the
      // number of starting vertices being considered simultaneously.
      var TPV: [TPVLocaleSpace] taskPrivateData(domain(index(vertex_domain)),
                                                vertex_domain.type);

      // Initialize
      coforall (t, i) in (TPVLocaleSpace, TPVSpace) do on TPVLocaleSpace.dist.idxToLocale(t) {
          TPV[i] = new taskPrivateData(domain(index(vertex_domain)), vertex_domain);
          var tpv = TPV[i];
          forall v in vertex_domain do
            tpv.BCaux[v].children_list.nd = {1..G.n_Neighbors[v]};
          for loc in Locales do on loc {
            tpv.Active_Level[here.id] = new Level_Set (Sparse_Vertex_List);
            tpv.Active_Level[here.id].previous = nil;
            tpv.Active_Level[here.id].next = new Level_Set (Sparse_Vertex_List);;
            tpv.Active_Level[here.id].next.previous = tpv.Active_Level[here.id];
            tpv.Active_Level[here.id].next.next = nil;
          }
      }
      var TPVM: TPVManager(TPV.type) = new TPVManager(TPV);

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      forall s in starting_vertices do on vertex_domain.dist.idxToLocale(s) {

        const shere = here.id;

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

        // Initialize task private variables
        const tid = TPVM.gettid();
        const tpv = TPVM.getTPV(tid);
        var BCaux => tpv.BCaux;
        pragma "dont disable remote value forwarding"
        inline proc f1(BCaux, v) {
          BCaux[v].path_count$.write(0.0);
        }
        forall v in vertex_domain do {
          BCaux[v].depend = 0.0;
          BCaux[v].min_distance.write(-1);
          f1(BCaux, v);
          BCaux[v].children_list.child_count.write(0);
        }
	var Lcl_Sum_Min_Dist: atomic real;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

        //
        // Used to check termination of the forward pass
        //
        // We could use a task-private reduction variable here.  This
        // is yet another concept that the Chapel group is planning to
        // implement.
        //

        var remaining = true;

        //
        // Each locale will have its own level sets.  A locale's level set
        // will only contain nodes that are physically allocated on that
        // particular locale.
        //
        var Active_Level => tpv.Active_Level;
        pragma "dont disable remote value forwarding"
        inline proc f2(BCaux, s) {
          BCaux[s].path_count$.write(1.0);
        }

        var barrier = new Barrier(numLocales);

        coforall loc in Locales do on loc {
          Active_Level[here.id].Members.clear();
          Active_Level[here.id].next.Members.clear();
          if vertex_domain.dist.idxToLocale(s) == here {
            // Establish the initial level sets for the breadth-first
            // traversal from s
            Active_Level[here.id].Members.add(s);
            BCaux[s].min_distance.write(0);
            f2(BCaux, s);
          }
          barrier.barrier();

          var current_distance : int = 0;

          while remaining do {
	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;

            // The Chapel compiler is still a bit conservative when it
            // comes to forwarding read-only variables to remote
            // locales, so we make a const copy here to insure it is
            // forwarded to the remote locale in the following
            // coforall loop.
            const current_distance_c = current_distance;
            pragma "dont disable remote value forwarding"
            inline proc f3(BCaux, v, u) {
              BCaux[v].path_count$.add(BCaux[u].path_count$.read());
            }

            forall u in Active_Level[here.id].Members do {
              forall v in G.FilteredNeighbors(u) do
                on if useOnClause then vertex_domain.dist.idxToLocale(v) else here {
                  // --------------------------------------------
                  // add any unmarked neighbors to the next level
                  // --------------------------------------------

                  if  BCaux[v].min_distance.compareExchangeStrong(-1, current_distance_c) {
                    var aloc = if useOnClause then here.id else vertex_domain.dist.idxToLocale(v).id;
                    Active_Level[aloc].next.Members.add (v);
                    if VALIDATE_BC then
                      Lcl_Sum_Min_Dist.add(current_distance_c);
                  }

                  // ------------------------------------------------
                  // only neighbors of  u  that are in the next level
                  // are on shortest paths from s through v.  Some
                  // task will have set  min_distance (v) by the
                  // time this code is reached, whether  v  lies in
                  // the previous, the current or the next level.
                  // ------------------------------------------------

                  if BCaux[v].min_distance.read() == current_distance_c {
                    f3(BCaux, v, u);
                    BCaux[u].children_list.add_child (v);
                  }

                }
            };

            // This barrier is needed to insure all updates to the next
            // level are completed before updating to use the next level

            // do some work while we wait
            // barrier.notify(); // This is expensive without network atomics
            //  for now, just do a normal barrier

            if Active_Level[here.id].next.next == nil {
              Active_Level[here.id].next.next = new Level_Set (Sparse_Vertex_List);
              Active_Level[here.id].next.next.previous = Active_Level[here.id].next;
              Active_Level[here.id].next.next.next = nil;
            } else {
              Active_Level[here.id].next.next.Members.clear();
            }
            // barrier.wait(); // ditto
            barrier.barrier();
            if here.id==shere {
              remaining = false;
            }

            barrier.barrier();
            Active_Level[here.id] = Active_Level[here.id].next;
            if Active_Level[here.id].Members.numIndices:bool then
              remaining = true;

            barrier.barrier();

          };  // end forward pass

          if VALIDATE_BC then
            if here.id==0 then
              Sum_Min_Dist$.add(Lcl_Sum_Min_Dist);

          // -------------------------------------------------------------
          // compute the dependencies recursively, traversing the vertices
          // of the graph in non-increasing order of distance (reverse
          // ordering from the initial traversal)
          // -------------------------------------------------------------

          const graph_diameter = current_distance - 1;

          if DEBUG_KERNEL4 then
            if here.id==0 then
              writeln ( " graph diameter from starting node ", s,
                        "  is ", graph_diameter );

          pragma "dont disable remote value forwarding"
          inline proc f4(BCaux, Between_Cent$, u) {
            BCaux[u].depend = + reduce [v in BCaux[u].children_list.Row_Children[1..BCaux[u].children_list.child_count.read()]]
              ( BCaux[u].path_count$.read() /
                BCaux[v].path_count$.read() )      *
              ( 1.0 + BCaux[v].depend );
            Between_Cent$(u).add(BCaux[u].depend);
          }

          // back up to last level
          var curr_Level =  Active_Level[here.id].previous;

          for current_distance in 2 .. graph_diameter by -1 {
            curr_Level = curr_Level.previous;

            for u in curr_Level.Members do on vertex_domain.dist.idxToLocale(u) {
                f4(BCaux, Between_Cent$, u);
            }

            barrier.barrier();
          }
        }

        TPVM.releaseTPV(tid);

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	const n_edges          = G.num_edges;
	const N_VERTICES       = vertex_domain.numIndices;
	const N_START_VERTICES = if starting_vertices == G.vertices
                                 then N_VERTICES
                                      - + reduce [v in vertex_domain]
                                        (G.n_Neighbors (v) == 0)
                                 else starting_vertices.numIndices;
	const TEPS             = 7.0 * N_VERTICES * N_START_VERTICES / K4_time;
	const Adjusted_TEPS    = n_edges * N_START_VERTICES / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
        Sum_Min_Dist = Sum_Min_Dist$.read();

      Between_Cent = Between_Cent$.read();

      if DELETE_KERNEL4_DS {
        coforall t in TPVSpace do on t {
          var tpv = TPV[t];
          var al = tpv.Active_Level;
          coforall loc in Locales do on loc {
            var level = al[here.id];
            while level != nil {
                var l2 = level.next;
                delete level;
                level = l2;
            }
          }
          delete TPV[t];
        }
        delete TPVM;
      }

    } // end of Brandes' betweenness centrality calculation

  //
  // Addition support data structures for kernel 4
  //

  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
    var next : Level_Set (Sparse_Vertex_List);
  }

  //
  // Data structure to save Children lists between the forward
  //  and backwards passes
  //
  record child_struct {
    type vertex;
    var nd: domain(1);
    var Row_Children: [nd] vertex;
    var child_count: atomic int;

    // This function should only be called using unique vertices
    proc add_child ( new_child: vertex ) {
      var c = child_count.fetchAdd(1)+1;
      Row_Children[c] = new_child;
    }
  }

  //
  // Implementation of task-private variables for kernel 4
  //
  record taskPrivateArrayData {
    type vertex;
    var min_distance  : atomic int;
    var path_count$   : atomic real;
    var depend        : real;
    var children_list : child_struct(vertex);
  }

  class taskPrivateData {
    type Sparse_Vertex_List;
    const vertex_domain;
    var used  : atomic bool;
    var BCaux : [vertex_domain] taskPrivateArrayData(index(vertex_domain));
    var Active_Level : [PrivateSpace] Level_Set (Sparse_Vertex_List);
  }

  // This is a simple class that hands out task private variables from the
  // array of task private variables, TPV.
  class TPVManager {
    const TPV;
    var currTPV: atomic int;
    proc gettid() {
      const tid = this.currTPV.fetchAdd(1)%numTPVs;
      on this.TPV[tid] do
        while this.TPV[tid].used.testAndSet() do chpl_task_yield();
      return tid;
    }
    proc getTPV(tid) {
      return this.TPV[tid];
    }
    proc releaseTPV(tid) {
      this.TPV[tid].used.clear();
    }
  }

  //
  // reusable barrier implementation
  //
  record Barrier {
    param reusable = true;
    var n: int;
    var count: atomic_int64;
    var done: atomicflag;

    proc Barrier(_n: int) {
      reset(_n);
    }

    inline proc reset(_n: int) {
      on this {
        n = _n;
        count.write(n);
        done.write(false);
      }
    }

    inline proc barrier() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to barrier()");
          if reusable {
            count.waitFor(n-1);
            count.add(1);
            done.clear();
          }
        } else {
          done.waitFor(true);
          if reusable {
            count.add(1);
            done.waitFor(false);
          }
        }
      }
    }

    inline proc notify() {
      on this {
        const myc = count.fetchSub(1);
        if myc<=1 {
          if done.testAndSet() then
            halt("Too many callers to notify()");
        }
      }
    }

    inline proc wait() {
      on this {
        done.waitFor(true);
        if reusable {
          const myc = count.fetchAdd(1);
          if myc == n-1 then
            done.clear();
          done.waitFor(false);
        }
      }
    }

    inline proc try() {
      return done.read();
    }
  }

}