comms optimization

comms optimization

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@21831 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use Time;
use thermo;
use MDForce;
use BlockDist;

// used in testing system
config param printOriginal = false;
config param printPerf = true;
config param printCorrect = false;

// Stores user-configurable variables
// See printHelp for more information
config var input_file = "in.lj.miniMD";
config var numSteps = 100;
config var size = 32;
config var neigh_bins = -1;
config var units = "lj";
config var force = "lj";
config var data_file = "";
config var yaml_output = 1;
config var yaml_screen = false;

// user/dev utilities
config const debug = false;
config const pHelp = false;

// 3D vector type alias
type v3 = 3*real;
type v3int = 3*int;

// system variables from input files
var problemSize : v3 = (size:real,size:real,size:real);
var initialTemp = 1.44;
var density = .8442;
var dt = .005;
var dtforce = .0025;
var force_cut = 2.5;
var cutneigh = 2.8;
var neigh_every = 20;
var thermoEvery = 100;

// Used in deterministic 'random' number generator
// Based on Park-Miller
const IA = 16807;
const IM = 2147483647;
const AM = 1.0/IM;
const IQ = 127773;
const IR = 2836;
const MASK = 123459876;

// Dot product between two vectors
inline proc dot(a,b : v3) {
  var c = a * b;
  return c(1) + c(2) + c(3);
}

inline proc dot(a,b : v3int) {
  var c = a*b;
  return c(1) + c(2) + c(3);
}

record atom {
  // velocity, force
  var v, f : v3;

  // define storage for neighbor list, which stores the bin and index
  // of a neighboring atom
  var nspace : domain(1) = {1..100};
  var neighs : [nspace] (v3int,int);
  var ncount : int = 0;
}

// dimensions of simulation space
var box : v3;
var volume : real;
var boxlo, boxhi : v3;

// atom storage
var numAtoms : int;

// default mass
var mass : real = 1.0;

// number of bins that define our piece of the continguous space
var numBins : v3int;

// constants from c++ version, used in neighboring
const small = 1.0e-6;
const factor = .999;

// time for binAtoms and buildNeighbors
var buildTime : real;

// time for force computation (inside integration)
var forceTime : real;

// Time for communication
//
// Since chapel abstracts much of this away, sometimes we do communication
// without knowing (or caring). This value may not be completely accurate,
// but should reflect the bulk of communication that occurs.
var commTime : real;

// will halt if true
if pHelp then printHelp();

// entry point - read configuration file if one exists
inputFile();
dtforce = .5 * dt;

// generate our own atoms, or read from a data file?
const generating = data_file == "" || data_file == "none";

const cutneighsq = cutneigh * cutneigh;

// io for reading the data file
var dataFile : file;
var dataReader : channel(false, iokind.dynamic,false);

// no data file, use input file to generate uniform lattice
if generating {
  const lattice : real = (4.0 / density) ** (1.0 / 3.0);
  box = problemSize * (lattice,lattice,lattice);
  volume = box(1) * box(2) * box(3);

  boxhi = box;
  numAtoms = (4 * problemSize(1) * problemSize(2) * problemSize(3)) : int;

  for i in 1..3 do
    numBins(i) = (5.0/6.0 * problemSize(i)) : int;
} else {
  dataFile = open(data_file, iomode.r);
  dataReader = dataFile.reader();

  dataReader.readln(); // skip first line

  // we can only have 1 atom type
  var types = dataReader.readln(int);
  assert(types == 1, "You can only have one type of atom.");

  // sim dimensions
  for i in 1..3 do
    (boxlo(i), boxhi(i)) = dataReader.readln(real,real);
  box = boxhi - boxlo;
  volume = box(1) * box(2) * box(3);

  const masses = dataReader.readln(string);
  assert(masses == "Masses");
  var (mass_type, mass) = dataReader.readln(int,real);

  // density overriden if data file provided
  density = numAtoms / (volume);
  const nbs : real = (density * 16) ** (1.0/3.0 : real);
  for i in 1..3 do
    numBins(i) = (box(i) / nbs) : int;
}

initThermo();

// user-defined
if neigh_bins > 0 {
  numBins = (neigh_bins,neigh_bins,neigh_bins);
}

// physical size of a bin
const binsize = box / numBins;

// 1/binsize
const bininv = (1.0,1.0,1.0) / binsize;

// number of bins we need for neighboring
var numNeed : v3int;
for i in 1..3 {
  numNeed(i) = (cutneigh * bininv(i)) : int;
  if numNeed(i) * binsize(i) < factor * cutneigh then numNeed(i) += 1;
}

// stencil that defines our bin neighbor offsets
// for example, (0,1,0) will represent the bin above us
const stencil = {-numNeed(1) .. numNeed(1),
                  -numNeed(2) .. numNeed(2),
                  -numNeed(3) .. numNeed(3)};

// Defines the problem space
const binSpace = {1..numBins(1), 1..numBins(2), 1..numBins(3)};

// The 'special sauce'. Will distribute stuff across locales
const DistSpace = binSpace dmapped Block(binSpace);

// bin storage. we can likely assume that each bin will store
// about the same number of atoms
//
// TODO: can we deal with this on a per-locale basis? How do we deal with
// the case of not enough space when copying inside communication?
var perBinSpace : domain(1) = {1..8};

// locale's chunk of 'bins' (consists of whole bins)
class Chunk {
  var Fluff : domain(3);                          // real and ghost bins
  var Real : domain(3);                           // real bins
  var NeighDom : domain(3);                       // nearest neighbors
  var Bins : [Real] [perBinSpace] atom;           // bin and atom storage
  var Pos : [Fluff] [perBinSpace] v3;
  var Count : [Fluff] int(32);                    // # of actual atoms in a bin
  var DestSlice, SrcSlice : [NeighDom] domain(3); // local/remote ghost slices
  var Neighs : [NeighDom] (int,int,int);          // array of locale indices
  var PosOffset : [NeighDom] v3;
}

// This breaks the abstraction layer a bit. hopefully we can push this into a
// Chapel distribution at some point.
//
// LocaleGridDom is a set of indices that map into LocaleGrid, which
// contains 'locale' objects with which we can use the 'on' statement
// for actual distribution. These values are automatically calculated
// by the Block distribution, and it was simplest to re-use them in our
// code.
//
// TODO: Discuss implementing an interface for grabbing such useful values.
const LocaleGridDom = DistSpace._value.dist.targetLocDom;
const LocaleGrid = DistSpace._value.dist.targetLocales;

// storage across all locales
var Grid : [LocaleGridDom] Chunk;

// map of bin (3*int) to LocaleGridDom index (3*int)
var BinToLocale : [DistSpace] (int,int,int);

setupComms();

var fobj : Force;

if force == "lj" {
  fobj = new ForceLJ(force_cut);
} else {
  fobj = new ForceEAM(force_cut);
  mass = fobj.mass;
}

if printOriginal then writeln("# Create System:");

// if there's no data file to read from, generate a lattice of atoms
if generating {
  create_atoms();
  create_velocity();
} else {
  dataReader.readln(); // skip 'Atoms' line

  // read in positions
  var tempPos : [1..numAtoms] v3;
  for x in tempPos {
    dataReader.read(int,int);
    dataReader.readln(x(1),x(2),x(3));
  }

  // read velocities and add
  for x in tempPos {
    var v : v3;
    dataReader.read(int,int);
    dataReader.readln(v(1),v(2),v(3));
    addatom(new atom(v), x, coord2bin(x));
  }

  // cleanup
  dataReader.close();
  dataFile.close();
}

// setup/store slices and neighbors so we don't have to recompute them every
// time. Our neighbor bins won't change, and we'll always need the same slices.
proc setupComms() {
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      // grab our chunk of the real domain
      const MyLocDom = DistSpace._value.locDoms[ijk].myBlock;
      if debug then writeln(here, " owns ", MyLocDom);

      // expand the maximum distance we might need
      const WithFluff = MyLocDom.expand(numNeed);
      if debug then writeln(here, "'s fluff: ", WithFluff);

      // directions of our nearest neighboring locales
      const NeighDom = {-1..1, -1..1, -1..1};

      var MyChunk = new Chunk(Fluff=WithFluff, Real=MyLocDom, NeighDom=NeighDom);
      Grid[ijk] = MyChunk;

      // alias to save typing
      var DestSlice => MyChunk.DestSlice;
      var SrcSlice => MyChunk.SrcSlice;
      var Neighs => MyChunk.Neighs;

      // Create 3D slices representing an overlap
      forall (Src, Dest, neigh) in zip(SrcSlice, DestSlice, NeighDom) {
        // the number of bins (numNeed) we need in a direction (neigh)
        Dest = MyLocDom.exterior(neigh * numNeed);

        // in the case of many locales, most of the time source and destination
        // will share the same indices. The wrapping case is handled soon.
        Src = Dest;
      }

      // use so we can move atoms around more easily
      BinToLocale[MyLocDom] = ijk;

      // adjust our remote-translated slices and neighbor pointer
      // store for future use
      forall (Src, Dest, PosOffset, neigh) in
        zip(SrcSlice, DestSlice, MyChunk.PosOffset, NeighDom) {
        // no comms with ourself, can't 'continue' inside a forall
        if neigh != (0,0,0) {
          // Get the index of the neighboring Locale
          var neighbor = ijk + neigh;

          // some of our slices will overlap with 'real' cells, but slices
          // on the exterior will need their cells to be offset so we can index
          // into the neighboring locale's bins.
          var SrcOffset = (0,0,0);

          // Adjust the neighbor index so we're actually referencing an
          // existing locale. Also compute the offset for source slices,
          // if necessary.
          for i in 1..3 {
            if neighbor(i) < 0 {
              neighbor(i) = LocaleGridDom.high(i);
              SrcOffset(i) = binSpace.high(i) - Dest.high(i);
            }
            else if neighbor(i) > LocaleGridDom.high(i) {
              neighbor(i) = 0;
              SrcOffset(i) = -binSpace.high(i) + binSpace.low(i) - 1;
            }
          }
          Src = Src.translate(SrcOffset);
          Neighs[neigh] = neighbor;

          var mask = (1.0,1.0,1.0);
          for i in 1..3 do
            if Dest.low(i) == Src.low(i) then mask(i) = 0.0;

          // ghost's position offset
          PosOffset = (neigh) * box * mask;

          if debug
            then writeln(Src, " is the remote neighbor of ", Dest);
          }
      }
    } // end of on statement
  } // end of coforall
}

// Reads an input file
proc inputFile() {
  var err : syserr;
  var fchan = open(err, input_file, iomode.r);

  if err then {
    input_file = "none";
    return;
  }

  var r = fchan.reader();

  // skip first line
  r.readln();

  // get unit type ('lj' vs 'metal')
  units = r.readln(string);

  // use data file?
  data_file = r.readln(string);

  // get force type : 'lj' vs 'eam'
  force = r.readln(string);

  // 'size of problem'
  problemSize = r.readln(int,int,int);

  // # iterations
  numSteps = r.readln(int);

  // simulated time step size
  dt = r.readln(real);

  // starting temperature
  initialTemp = r.readln(real);

  // start density (overriden if data file), used in EAM
  density = r.readln(real);

  // recompute neighbors every N iterations
  neigh_every = r.readln(int);

  // force cutoff: used to determine significance of force computation
  // neighbor cutoff: used to determine if a nearby atom may be significant
  (force_cut,cutneigh) = r.readln(real, real);

  // do thermo compute every N iterations
  thermoEvery = r.readln(int);

  // cutneigh = skin + force_cutoff, used in neighboring algorithm
  cutneigh += force_cut;

  // cleanup
  r.close();
  fchan.close();
}

proc printHelp() {
  writeln("Commandline Options:");
  writeln("\n  Chapel help:");
  writeln("\t--chapel:                Show Chapel's command line options");
  writeln("\n  Execution configuration:");
  writeln("\n  Simulation setup:");
  writeln("\t--input_file <string>:   set input file to be used (default: in.lj.miniMD)");
  writeln("\t--numSteps <int>:        set number of timesteps for simulation");
  writeln("\t--size <int>:            set linear dimension of systembox");
  writeln("\t--neigh_bins <int>:      set linear dimension of neighbor bin grid");
  writeln("\t--units <string>:        set units (lj or metal), see LAMMPS documentation");
  writeln("\t--force <string>:        set interaction model (lj or eam)");
  writeln("\t--data_file <string>:    read configuration from LAMMPS data file");
  writeln("\n  Miscelaneous:");
  writeln("\t                                within rcut_neighbor (outer force cutoff)");
  writeln("\t--yaml_output <int>:     level of yaml output (default 1)");
  writeln("\t--yaml_screen:                write yaml output also to screen");
  writeln("\t--pHelp:                  display this help message");
  writeln("\t--------------------------------------------------");
  exit(0);
}

// Print simulation settings
proc printSim() {
  if !printOriginal then return;
  writeln("# Done .... ");
  writeln("# miniMD-Reference 0.1 (Chapel) output ...");
  writeln("# Systemparameters: ");
  writeln("\t# input_file: ", input_file);
  if data_file != "" then
    writeln("\t# datafile: ", data_file);
  else
    writeln("\t# datafile: none");
  writeln("\t# force: ", force);
  write("\t# units: ");
  if units == "metal" then writeln("METAL");
  else writeln("LJ");
  writeln("\t# atoms: ", numAtoms);
  write("\t# System size: ");
  writef("%.2dr %.2dr %.2dr (unit cells: %i %i %i)\n",
      box(1), box(2), box(3), problemSize(1), problemSize(2), problemSize(3));
  writef("\t%# density: %.6dr\n", density);
  writef("\t%# Force cutoff: %.6dr\n", force_cut);
  writef("\t%# neigh cutoff: %.6dr\n", cutneigh);
  writeln("\t# Neighbor bins: ", numBins(1), " ", numBins(2), " ", numBins(3));
  writeln("\t# neighbor frequency ", neigh_every);
  writef("\t%# timestep size: %.6dr\n", dt );
  writeln("\t# thermo frequency: ", thermoEvery);
}

proc create_atoms() {
  const lattice : real = (4.0 / density) ** (1.0 / 3.0);

  var lo, hi : v3int;

  for i in 1..3 {
    lo(i) = (boxlo(i) / (.5 * lattice)) : int;
    hi(i) = (boxhi(i) / (.5 * lattice)) : int;
  }

  for i in 1..3 {
    lo(i) = max(lo(i), 0);
    hi(i) = min(hi(i), 2 * problemSize(i) - 1) : int;
  }

  var temp, v : v3;
  var s, o : v3int;
  var m, n : int;
  var curCoord : v3int;
  var subboxdim = 8;
  var flag = false;

  while o(3) * subboxdim <= hi(3) {
    curCoord = (o * (subboxdim,subboxdim,subboxdim) + s);

    if flag then continue;

    var withinBounds = true;
    for i in 1..3 {
      withinBounds = withinBounds && curCoord(i) >= lo(i) && curCoord(i) <= hi(i);
    }

    if ((curCoord(1) + curCoord(2) + curCoord(3)) % 2 == 0) && withinBounds {
      for i in 1..3 do
        temp(i) = .5 * lattice * curCoord(i);

      withinBounds = true;
      for i in 1..3 {
        withinBounds = withinBounds && temp(i) >= boxlo(i) && temp(i) < boxhi(i);
      }

      if withinBounds {
        n = (curCoord(3) * (2 * problemSize(2)) * (2*problemSize(1)) + curCoord(2) * (2 * problemSize(1)) + curCoord(1) + 1) : int;
        for i in 1..3 {
          for m in 1..5 { pmrand(n); }
          v(i) = pmrand(n);
        }
        addatom(new atom(v), temp, coord2bin(temp));
      }
    }

    s(1) += 1;

    if s(1) == subboxdim {
      s(1) = 0;
      s(2) += 1;
    }

    if s(2) == subboxdim {
      s(2) = 0;
      s(3) += 1;
    }

    if s(3) == subboxdim {
      s(3) = 0;
      o(1) += 1;
    }

    if o(1) * subboxdim > hi(1) {
      o(1) = 0;
      o(2) += 1;
    }

    if o(2) * subboxdim > hi(2) {
      o(2) = 0;
      o(3) += 1;
    }
  }
}

proc create_velocity() {
  var vtot : v3;

  // find the total velocity
  var vtotDist : [LocaleGridDom] v3;
  coforall (ijk) in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      vtotDist[ijk] = + reduce forall bin in Me.Real do
      + reduce forall ind in 1..Me.Count[bin] do Me.Bins[bin][ind].v;
    }
  }
  vtot = + reduce vtotDist;

  // get the average
  vtot /= (numAtoms,numAtoms,numAtoms);

  // adjust using average
  coforall (ijk) in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      forall (bin,c) in zip(Me.Bins[Me.Real], Me.Count[Me.Real]) {
        for a in bin[1..c] {
          a.v -= vtot;
        }
      }
    }
  }

  const factor = sqrt(initialTemp / temperature());

  coforall (ijk) in LocaleGridDom {
    on LocaleGrid[ijk] {
      const fac = (factor,factor,factor);
      const Me = Grid[ijk];
      forall (bin,c) in zip(Me.Bins[Me.Real], Me.Count[Me.Real]) {
        for a in bin[1..c] {
          a.v *= fac;
        }
      }
    }
  }
}

// Park/Miller RNG w/out MASKING
proc pmrand(ref n : int) : real {
  var k : int;
  var ans : real;

  k = n / IQ;
  n = IA * (n - k * IQ) - IR * k;
  if n < 0 then n += IM;
  ans = AM * n;
  return ans;
}

proc updateFluff() {
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      forall (Src, Dest, Neighbor, Offset, neigh) in
        zip(Me.SrcSlice, Me.DestSlice, Grid[Me.Neighs], Me.PosOffset, Me.NeighDom) {
        // don't look at ourself
        if neigh != (0,0,0) {
          // copy over the bins' counts
          Me.Count[Dest] = Neighbor.Count[Src];

          // copy over fluff atoms' positions
          Me.Pos[Dest] = Neighbor.Pos[Src];

          // offset atoms if necessary
          if Offset != (0.0,0.0,0.0) {
            forall (p,c) in zip(Me.Pos[Dest], Me.Count[Dest]) do p[1..c] += Offset;
          }
        }
      }
    }
  }
}

// if atoms moved outside the box, wrap them around
proc pbc() {
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      const b = box;
      forall (p,c) in zip(Me.Pos[Me.Real],Me.Count[Me.Real]) {
        for x in p[1..c] {
          for i in 1..3 {
            if x(i) < 0 then x(i) += b(i);
            else if x(i) >= b(i) then x(i) -= b(i);
          }
        }
      }
    }
  }
}

proc buildNeighbors() {
  if debug then writeln("starting to build...");

  var neighTimer : Timer;
  neighTimer.start();

  // enforce boundaries
  pbc();

  commTime += neighTimer.elapsed();
  neighTimer.stop();
  neighTimer.clear();
  neighTimer.start();

  // if any atoms moved outside a bin, relocate
  binAtoms();

  buildTime += neighTimer.elapsed();
  neighTimer.stop();
  neighTimer.clear();
  neighTimer.start();

  if debug then writeln("starting comms...");

  // grab ghost/overlapping atoms
  updateFluff();

  if debug then writeln("comms done...");

  commTime += neighTimer.elapsed();
  neighTimer.stop();
  neighTimer.clear();
  neighTimer.start();

  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      forall (bin,posBin,r,c) in zip(Me.Bins, Me.Pos[Me.Real],Me.Real, Me.Count[Me.Real]) {
        const existing = 1..c;
        for (a,pos,i) in zip(bin[existing],posBin[existing],existing) {
          // reset neighbor list
          a.ncount = 0;

          // for each surrounding bin
          for s in stencil {
            const o = r + s;
            const existing = 1..Me.Count[o];
            for (npos,x) in zip(Me.Pos[o][existing],existing) {
              // if we're looking at ourself
              if r == o && x == i then continue;

              // are we within range?
              const del = pos - npos;
              const rsq = dot(del,del);
              if rsq <= cutneighsq {
                a.ncount += 1;

                // resize neighbor list if necessary
                var h = a.nspace.high;
                while a.ncount > h {
                  h = ((a.nspace.high*1.2) : int);
                  a.nspace = {1..h};
                }

                // store atom's bin and index
                a.neighs[a.ncount] = (o,x);
              }
            }
          }
        }
      }
    }
  }

  buildTime += neighTimer.elapsed();

  if debug then writeln("building done...");
} // end of buildNeighbors

// add an atom 'a' to bin 'b'
// resize if necessary
inline proc addatom(a : atom, x : v3, b : v3int) {
  const Target = Grid[BinToLocale[b]];

  // increment bin's # of atoms
  Target.Count[b] += 1;
  const end = Target.Count[b];

  // resize bin storage if needed
  if end >= perBinSpace.high {
    var h = (perBinSpace.high * 1.5) : int;
    perBinSpace = {1..h};
  }

  // add to the end of the bin
  Target.Bins[b][end] = a;
  Target.Pos[b][end] = x;
}

// helper class for moving atoms around
class Mov {
  var Space : domain(1) = {1..50};
  var List : [Space] (atom, v3, v3int);
  var Count : int;
}

// TODO: reduce communication here
proc binAtoms() {
  var Moved : [LocaleGridDom] Mov;
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      var ToMove = new Mov();
      Moved[ijk] = ToMove;
      const Me = Grid[ijk];
      for (bin, pos, r, c) in zip(Me.Bins, Me.Pos[Me.Real] ,Me.Real ,Me.Count[Me.Real]) {
        var cur = 1;

        // for each atom, check if moved
        // because we move from the end, this setup allows us to examine
        // the atom pulled from the end
        while(cur <= c) {
          const destBin = coord2bin(pos[cur]);

          // atom moved
          if destBin != r {
            ToMove.Count += 1;

            // resize storage as needed
            if ToMove.Count >= ToMove.Space.high then
              ToMove.Space = {1..ToMove.Space.high * 2};

            ToMove.List[ToMove.Count] = (bin[cur], pos[cur], destBin);

            // replace with atom at end of list, if one exists
            if cur < c {
              bin[cur] = bin[c];
              pos[cur] = pos[c];
            }

            // correct bin count
            c -= 1;
          } else cur += 1;
        }
      }
    }
  }

  // actually move the atoms
  for ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      for (a, x, b) in Moved[ijk].List[1..Moved[ijk].Count] {
        addatom(a,x,b);
      }
    }
  }
}

// compute atom's correct bin based on its physical position
proc coord2bin(x : v3){
  var cur : v3int;

  // create a simple mask on a per-dimension basis
  var mask : v3int;
  for i in 1..3 do
    mask(i) = if x(i) >= box(i) then 1 else 0;

  // if the position has drifted outside the physical space,
  // adjust. Divide position by the size of a bin, and add
  // (1,1,1) so we're starting at the lowest possible bin
  const temp = (x - box*mask) * bininv + (1,1,1);

  // can't cast from 3*real to 3*int (yet?)
  for i in 1..3 do
    cur(i) = temp(i) : int;

  return cur;
}

use Time;
use thermo;
use MDForce;
use BlockDist;

// used in testing system
config param printOriginal = false;
config param printPerf = true;
config param printCorrect = false;

// Stores user-configurable variables
// See printHelp for more information
config var input_file = "in.lj.miniMD";
config var numSteps = 100;
config var size = 32;
config var neigh_bins = -1;
config var units = "lj";
config var force = "lj";
config var data_file = "";
config var yaml_output = 1;
config var yaml_screen = false;

// user/dev utilities
config const debug = false;
config const pHelp = false;

// 3D vector type alias
type v3 = 3*real;
type v3int = 3*int;

// system variables from input files
var problemSize : v3 = (size:real,size:real,size:real);
var initialTemp = 1.44;
var density = .8442;
var dt = .005;
var dtforce = .0025;
var force_cut = 2.5;
var cutneigh = 2.8;
var neigh_every = 20;
var thermoEvery = 100;

// Used in deterministic 'random' number generator
// Based on Park-Miller
const IA = 16807;
const IM = 2147483647;
const AM = 1.0/IM;
const IQ = 127773;
const IR = 2836;
const MASK = 123459876;

// Dot product between two vectors
inline proc dot(a,b : v3) {
  var c = a * b;
  return c(1) + c(2) + c(3);
}

inline proc dot(a,b : v3int) {
  var c = a*b;
  return c(1) + c(2) + c(3);
}

record atom {
  // velocity, force
  var v, f : v3;

  // define storage for neighbor list, which stores the bin and index
  // of a neighboring atom
  var nspace : domain(1) = {1..100};
  var neighs : [nspace] (v3int,int);
  var ncount : int = 0;
}

// dimensions of simulation space
var box : v3;
var volume : real;
var boxlo, boxhi : v3;

// atom storage
var numAtoms : int;

// default mass
var mass : real = 1.0;

// number of bins that define our piece of the continguous space
var numBins : v3int;

// constants from c++ version, used in neighboring
const small = 1.0e-6;
const factor = .999;

// time for binAtoms and buildNeighbors
var buildTime : real;

// time for force computation (inside integration)
var forceTime : real;

// Time for communication
//
// Since chapel abstracts much of this away, sometimes we do communication
// without knowing (or caring). This value may not be completely accurate,
// but should reflect the bulk of communication that occurs.
var commTime : real;

// will halt if true
if pHelp then printHelp();

// entry point - read configuration file if one exists
inputFile();
dtforce = .5 * dt;

// generate our own atoms, or read from a data file?
const generating = data_file == "" || data_file == "none";

const cutneighsq = cutneigh * cutneigh;

// io for reading the data file
var dataFile : file;
var dataReader : channel(false, iokind.dynamic,false);

// no data file, use input file to generate uniform lattice
if generating {
  const lattice : real = (4.0 / density) ** (1.0 / 3.0);
  box = problemSize * (lattice,lattice,lattice);
  volume = box(1) * box(2) * box(3);

  boxhi = box;
  numAtoms = (4 * problemSize(1) * problemSize(2) * problemSize(3)) : int;

  for i in 1..3 do
    numBins(i) = (5.0/6.0 * problemSize(i)) : int;
} else {
  dataFile = open(data_file, iomode.r);
  dataReader = dataFile.reader();

  dataReader.readln(); // skip first line

  // we can only have 1 atom type
  var types = dataReader.readln(int);
  assert(types == 1, "You can only have one type of atom.");

  // sim dimensions
  for i in 1..3 do
    (boxlo(i), boxhi(i)) = dataReader.readln(real,real);
  box = boxhi - boxlo;
  volume = box(1) * box(2) * box(3);

  const masses = dataReader.readln(string);
  assert(masses == "Masses");
  var (mass_type, mass) = dataReader.readln(int,real);

  // density overriden if data file provided
  density = numAtoms / (volume);
  const nbs : real = (density * 16) ** (1.0/3.0 : real);
  for i in 1..3 do
    numBins(i) = (box(i) / nbs) : int;
}

initThermo();

// user-defined
if neigh_bins > 0 {
  numBins = (neigh_bins,neigh_bins,neigh_bins);
}

// physical size of a bin
const binsize = box / numBins;

// 1/binsize
const bininv = (1.0,1.0,1.0) / binsize;

// number of bins we need for neighboring
var numNeed : v3int;
for i in 1..3 {
  numNeed(i) = (cutneigh * bininv(i)) : int;
  if numNeed(i) * binsize(i) < factor * cutneigh then numNeed(i) += 1;
}

// stencil that defines our bin neighbor offsets
// for example, (0,1,0) will represent the bin above us
const stencil = {-numNeed(1) .. numNeed(1),
                  -numNeed(2) .. numNeed(2),
                  -numNeed(3) .. numNeed(3)};

// Defines the problem space
const binSpace = {1..numBins(1), 1..numBins(2), 1..numBins(3)};

// The 'special sauce'. Will distribute stuff across locales
const DistSpace = binSpace dmapped Block(binSpace);

// bin storage. we can likely assume that each bin will store
// about the same number of atoms
//
// TODO: can we deal with this on a per-locale basis? How do we deal with
// the case of not enough space when copying inside communication?
var perBinSpace : domain(1) = {1..8};

// locale's chunk of 'bins' (consists of whole bins)
class Chunk {
  var Fluff : domain(3);                          // real and ghost bins
  var Real : domain(3);                           // real bins
  var NeighDom : domain(3);                       // nearest neighbors
  var Bins : [Real] [perBinSpace] atom;           // bin and atom storage
  var Pos : [Fluff] [perBinSpace] v3;
  var Count : [Fluff] int(32);                    // # of actual atoms in a bin
  var DestSlice, SrcSlice : [NeighDom] domain(3); // local/remote ghost slices
  var Neighs : [NeighDom] (int,int,int);          // array of locale indices
  var PosOffset : [NeighDom] v3;
}

// This breaks the abstraction layer a bit. hopefully we can push this into a
// Chapel distribution at some point.
//
// LocaleGridDom is a set of indices that map into LocaleGrid, which
// contains 'locale' objects with which we can use the 'on' statement
// for actual distribution. These values are automatically calculated
// by the Block distribution, and it was simplest to re-use them in our
// code.
//
// TODO: Discuss implementing an interface for grabbing such useful values.
const LocaleGridDom = DistSpace._value.dist.targetLocDom;
const LocaleGrid = DistSpace._value.dist.targetLocales;

// storage across all locales
var Grid : [LocaleGridDom] Chunk;

// map of bin (3*int) to LocaleGridDom index (3*int)
var BinToLocale : [DistSpace] (int,int,int);

setupComms();

var fobj : Force;

if force == "lj" {
  fobj = new ForceLJ(force_cut);
} else {
  fobj = new ForceEAM(force_cut);
  mass = fobj.mass;
}

if printOriginal then writeln("# Create System:");

// if there's no data file to read from, generate a lattice of atoms
if generating {
  create_atoms();
  create_velocity();
} else {
  dataReader.readln(); // skip 'Atoms' line

  // read in positions
  var tempPos : [1..numAtoms] v3;
  for x in tempPos {
    dataReader.read(int,int);
    dataReader.readln(x(1),x(2),x(3));
  }

  // read velocities and add
  for x in tempPos {
    var v : v3;
    dataReader.read(int,int);
    dataReader.readln(v(1),v(2),v(3));
    addatom(new atom(v), x, coord2bin(x));
  }

  // cleanup
  dataReader.close();
  dataFile.close();
}

// setup/store slices and neighbors so we don't have to recompute them every
// time. Our neighbor bins won't change, and we'll always need the same slices.
proc setupComms() {
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      // grab our chunk of the real domain
      const MyLocDom = DistSpace._value.locDoms[ijk].myBlock;
      if debug then writeln(here, " owns ", MyLocDom);

      // expand the maximum distance we might need
      const WithFluff = MyLocDom.expand(numNeed);
      if debug then writeln(here, "'s fluff: ", WithFluff);

      // directions of our nearest neighboring locales
      const NeighDom = {-1..1, -1..1, -1..1};

      var MyChunk = new Chunk(Fluff=WithFluff, Real=MyLocDom, NeighDom=NeighDom);
      Grid[ijk] = MyChunk;

      // alias to save typing
      var DestSlice => MyChunk.DestSlice;
      var SrcSlice => MyChunk.SrcSlice;
      var Neighs => MyChunk.Neighs;

      // Create 3D slices representing an overlap
      forall (Src, Dest, neigh) in zip(SrcSlice, DestSlice, NeighDom) {
        // the number of bins (numNeed) we need in a direction (neigh)
        Dest = MyLocDom.exterior(neigh * numNeed);

        // in the case of many locales, most of the time source and destination
        // will share the same indices. The wrapping case is handled soon.
        Src = Dest;
      }

      // use so we can move atoms around more easily
      BinToLocale[MyLocDom] = ijk;

      // adjust our remote-translated slices and neighbor pointer
      // store for future use
      forall (Src, Dest, PosOffset, neigh) in
        zip(SrcSlice, DestSlice, MyChunk.PosOffset, NeighDom) {
        // no comms with ourself, can't 'continue' inside a forall
        if neigh != (0,0,0) {
          // Get the index of the neighboring Locale
          var neighbor = ijk + neigh;

          // some of our slices will overlap with 'real' cells, but slices
          // on the exterior will need their cells to be offset so we can index
          // into the neighboring locale's bins.
          var SrcOffset = (0,0,0);

          // Adjust the neighbor index so we're actually referencing an
          // existing locale. Also compute the offset for source slices,
          // if necessary.
          for i in 1..3 {
            if neighbor(i) < 0 {
              neighbor(i) = LocaleGridDom.high(i);
              SrcOffset(i) = binSpace.high(i) - Dest.high(i);
            }
            else if neighbor(i) > LocaleGridDom.high(i) {
              neighbor(i) = 0;
              SrcOffset(i) = -binSpace.high(i) + binSpace.low(i) - 1;
            }
          }
          Src = Src.translate(SrcOffset);
          Neighs[neigh] = neighbor;

          var mask = (1.0,1.0,1.0);
          for i in 1..3 do
            if Dest.low(i) == Src.low(i) then mask(i) = 0.0;

          // ghost's position offset
          PosOffset = (neigh) * box * mask;

          if debug
            then writeln(Src, " is the remote neighbor of ", Dest);
          }
      }
    } // end of on statement
  } // end of coforall
}

// Reads an input file
proc inputFile() {
  var err : syserr;
  var fchan = open(err, input_file, iomode.r);

  if err then {
    input_file = "none";
    return;
  }

  var r = fchan.reader();

  // skip first line
  r.readln();

  // get unit type ('lj' vs 'metal')
  units = r.readln(string);

  // use data file?
  data_file = r.readln(string);

  // get force type : 'lj' vs 'eam'
  force = r.readln(string);

  // 'size of problem'
  problemSize = r.readln(int,int,int);

  // # iterations
  numSteps = r.readln(int);

  // simulated time step size
  dt = r.readln(real);

  // starting temperature
  initialTemp = r.readln(real);

  // start density (overriden if data file), used in EAM
  density = r.readln(real);

  // recompute neighbors every N iterations
  neigh_every = r.readln(int);

  // force cutoff: used to determine significance of force computation
  // neighbor cutoff: used to determine if a nearby atom may be significant
  (force_cut,cutneigh) = r.readln(real, real);

  // do thermo compute every N iterations
  thermoEvery = r.readln(int);

  // cutneigh = skin + force_cutoff, used in neighboring algorithm
  cutneigh += force_cut;

  // cleanup
  r.close();
  fchan.close();
}

proc printHelp() {
  writeln("Commandline Options:");
  writeln("\n  Chapel help:");
  writeln("\t--chapel:                Show Chapel's command line options");
  writeln("\n  Execution configuration:");
  writeln("\n  Simulation setup:");
  writeln("\t--input_file <string>:   set input file to be used (default: in.lj.miniMD)");
  writeln("\t--numSteps <int>:        set number of timesteps for simulation");
  writeln("\t--size <int>:            set linear dimension of systembox");
  writeln("\t--neigh_bins <int>:      set linear dimension of neighbor bin grid");
  writeln("\t--units <string>:        set units (lj or metal), see LAMMPS documentation");
  writeln("\t--force <string>:        set interaction model (lj or eam)");
  writeln("\t--data_file <string>:    read configuration from LAMMPS data file");
  writeln("\n  Miscelaneous:");
  writeln("\t                                within rcut_neighbor (outer force cutoff)");
  writeln("\t--yaml_output <int>:     level of yaml output (default 1)");
  writeln("\t--yaml_screen:                write yaml output also to screen");
  writeln("\t--pHelp:                  display this help message");
  writeln("\t--------------------------------------------------");
  exit(0);
}

// Print simulation settings
proc printSim() {
  if !printOriginal then return;
  writeln("# Done .... ");
  writeln("# miniMD-Reference 0.1 (Chapel) output ...");
  writeln("# Systemparameters: ");
  writeln("\t# input_file: ", input_file);
  if data_file != "" then
    writeln("\t# datafile: ", data_file);
  else
    writeln("\t# datafile: none");
  writeln("\t# force: ", force);
  write("\t# units: ");
  if units == "metal" then writeln("METAL");
  else writeln("LJ");
  writeln("\t# atoms: ", numAtoms);
  write("\t# System size: ");
  writef("%.2dr %.2dr %.2dr (unit cells: %i %i %i)\n",
      box(1), box(2), box(3), problemSize(1), problemSize(2), problemSize(3));
  writef("\t%# density: %.6dr\n", density);
  writef("\t%# Force cutoff: %.6dr\n", force_cut);
  writef("\t%# neigh cutoff: %.6dr\n", cutneigh);
  writeln("\t# Neighbor bins: ", numBins(1), " ", numBins(2), " ", numBins(3));
  writeln("\t# neighbor frequency ", neigh_every);
  writef("\t%# timestep size: %.6dr\n", dt );
  writeln("\t# thermo frequency: ", thermoEvery);
}

proc create_atoms() {
  const lattice : real = (4.0 / density) ** (1.0 / 3.0);

  var lo, hi : v3int;

  for i in 1..3 {
    lo(i) = (boxlo(i) / (.5 * lattice)) : int;
    hi(i) = (boxhi(i) / (.5 * lattice)) : int;
  }

  for i in 1..3 {
    lo(i) = max(lo(i), 0);
    hi(i) = min(hi(i), 2 * problemSize(i) - 1) : int;
  }

  var temp, v : v3;
  var s, o : v3int;
  var m, n : int;
  var curCoord : v3int;
  var subboxdim = 8;
  var flag = false;

  while o(3) * subboxdim <= hi(3) {
    curCoord = (o * (subboxdim,subboxdim,subboxdim) + s);

    if flag then continue;

    var withinBounds = true;
    for i in 1..3 {
      withinBounds = withinBounds && curCoord(i) >= lo(i) && curCoord(i) <= hi(i);
    }

    if ((curCoord(1) + curCoord(2) + curCoord(3)) % 2 == 0) && withinBounds {
      for i in 1..3 do
        temp(i) = .5 * lattice * curCoord(i);

      withinBounds = true;
      for i in 1..3 {
        withinBounds = withinBounds && temp(i) >= boxlo(i) && temp(i) < boxhi(i);
      }

      if withinBounds {
        n = (curCoord(3) * (2 * problemSize(2)) * (2*problemSize(1)) + curCoord(2) * (2 * problemSize(1)) + curCoord(1) + 1) : int;
        for i in 1..3 {
          for m in 1..5 { pmrand(n); }
          v(i) = pmrand(n);
        }
        addatom(new atom(v), temp, coord2bin(temp));
      }
    }

    s(1) += 1;

    if s(1) == subboxdim {
      s(1) = 0;
      s(2) += 1;
    }

    if s(2) == subboxdim {
      s(2) = 0;
      s(3) += 1;
    }

    if s(3) == subboxdim {
      s(3) = 0;
      o(1) += 1;
    }

    if o(1) * subboxdim > hi(1) {
      o(1) = 0;
      o(2) += 1;
    }

    if o(2) * subboxdim > hi(2) {
      o(2) = 0;
      o(3) += 1;
    }
  }
}

proc create_velocity() {
  var vtot : v3;

  // find the total velocity
  var vtotDist : [LocaleGridDom] v3;
  coforall (ijk) in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      vtotDist[ijk] = + reduce forall bin in Me.Real do
      + reduce forall ind in 1..Me.Count[bin] do Me.Bins[bin][ind].v;
    }
  }
  vtot = + reduce vtotDist;

  // get the average
  vtot /= (numAtoms,numAtoms,numAtoms);

  // adjust using average
  coforall (ijk) in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      forall (bin,c) in zip(Me.Bins[Me.Real], Me.Count[Me.Real]) {
        for a in bin[1..c] {
          a.v -= vtot;
        }
      }
    }
  }

  const factor = sqrt(initialTemp / temperature());

  coforall (ijk) in LocaleGridDom {
    on LocaleGrid[ijk] {
      const fac = (factor,factor,factor);
      const Me = Grid[ijk];
      forall (bin,c) in zip(Me.Bins[Me.Real], Me.Count[Me.Real]) {
        for a in bin[1..c] {
          a.v *= fac;
        }
      }
    }
  }
}

// Park/Miller RNG w/out MASKING
proc pmrand(ref n : int) : real {
  var k : int;
  var ans : real;

  k = n / IQ;
  n = IA * (n - k * IQ) - IR * k;
  if n < 0 then n += IM;
  ans = AM * n;
  return ans;
}

proc updateFluff() {
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      forall (Src, Dest, Neighbor, Offset, neigh) in
        zip(Me.SrcSlice, Me.DestSlice, Grid[Me.Neighs], Me.PosOffset, Me.NeighDom) {
        // don't look at ourself
        if neigh != (0,0,0) {
          // copy over the bins' counts
          Me.Count[Dest] = Neighbor.Count[Src];

          // copy over fluff atoms' positions
          Me.Pos[Dest] = Neighbor.Pos[Src];

          // offset atoms if necessary
          if Offset != (0.0,0.0,0.0) {
            forall p in Me.Pos[Dest] do p += Offset;
          }
        }
      }
    }
  }
}

// if atoms moved outside the box, wrap them around
proc pbc() {
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      const b = box;
      forall (p,c) in zip(Me.Pos[Me.Real],Me.Count[Me.Real]) {
        for x in p[1..c] {
          for i in 1..3 {
            if x(i) < 0 then x(i) += b(i);
            else if x(i) >= b(i) then x(i) -= b(i);
          }
        }
      }
    }
  }
}

proc buildNeighbors() {
  if debug then writeln("starting to build...");

  var neighTimer : Timer;
  neighTimer.start();

  // enforce boundaries
  pbc();

  commTime += neighTimer.elapsed();
  neighTimer.stop();
  neighTimer.clear();
  neighTimer.start();

  // if any atoms moved outside a bin, relocate
  binAtoms();

  buildTime += neighTimer.elapsed();
  neighTimer.stop();
  neighTimer.clear();
  neighTimer.start();

  if debug then writeln("starting comms...");

  // grab ghost/overlapping atoms
  updateFluff();

  if debug then writeln("comms done...");

  commTime += neighTimer.elapsed();
  neighTimer.stop();
  neighTimer.clear();
  neighTimer.start();

  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      const Me = Grid[ijk];
      forall (bin,posBin,r,c) in zip(Me.Bins, Me.Pos[Me.Real],Me.Real, Me.Count[Me.Real]) {
        const existing = 1..c;
        for (a,pos,i) in zip(bin[existing],posBin[existing],existing) {
          // reset neighbor list
          a.ncount = 0;

          // for each surrounding bin
          for s in stencil {
            const o = r + s;
            const existing = 1..Me.Count[o];
            for (npos,x) in zip(Me.Pos[o][existing],existing) {
              // if we're looking at ourself
              if r == o && x == i then continue;

              // are we within range?
              const del = pos - npos;
              const rsq = dot(del,del);
              if rsq <= cutneighsq {
                a.ncount += 1;

                // resize neighbor list if necessary
                var h = a.nspace.high;
                while a.ncount > h {
                  h = ((a.nspace.high*1.2) : int);
                  a.nspace = {1..h};
                }

                // store atom's bin and index
                a.neighs[a.ncount] = (o,x);
              }
            }
          }
        }
      }
    }
  }

  buildTime += neighTimer.elapsed();

  if debug then writeln("building done...");
} // end of buildNeighbors

// add an atom 'a' to bin 'b'
// resize if necessary
inline proc addatom(a : atom, x : v3, b : v3int) {
  const Target = Grid[BinToLocale[b]];

  // increment bin's # of atoms
  Target.Count[b] += 1;
  const end = Target.Count[b];

  // resize bin storage if needed
  if end >= perBinSpace.high {
    var h = (perBinSpace.high * 1.5) : int;
    perBinSpace = {1..h};
  }

  // add to the end of the bin
  Target.Bins[b][end] = a;
  Target.Pos[b][end] = x;
}

// helper class for moving atoms around
class Mov {
  var Space : domain(1) = {1..50};
  var List : [Space] (atom, v3, v3int);
  var Count : int;
}

// TODO: reduce communication here
proc binAtoms() {
  var Moved : [LocaleGridDom] Mov;
  coforall ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      var ToMove = new Mov();
      Moved[ijk] = ToMove;
      const Me = Grid[ijk];
      for (bin, pos, r, c) in zip(Me.Bins, Me.Pos[Me.Real] ,Me.Real ,Me.Count[Me.Real]) {
        var cur = 1;

        // for each atom, check if moved
        // because we move from the end, this setup allows us to examine
        // the atom pulled from the end
        while(cur <= c) {
          const destBin = coord2bin(pos[cur]);

          // atom moved
          if destBin != r {
            ToMove.Count += 1;

            // resize storage as needed
            if ToMove.Count >= ToMove.Space.high then
              ToMove.Space = {1..ToMove.Space.high * 2};

            ToMove.List[ToMove.Count] = (bin[cur], pos[cur], destBin);

            // replace with atom at end of list, if one exists
            if cur < c {
              bin[cur] = bin[c];
              pos[cur] = pos[c];
            }

            // correct bin count
            c -= 1;
          } else cur += 1;
        }
      }
    }
  }

  // actually move the atoms
  for ijk in LocaleGridDom {
    on LocaleGrid[ijk] {
      for (a, x, b) in Moved[ijk].List[1..Moved[ijk].Count] {
        addatom(a,x,b);
      }
    }
  }
}

// compute atom's correct bin based on its physical position
proc coord2bin(x : v3){
  var cur : v3int;

  // create a simple mask on a per-dimension basis
  var mask : v3int;
  for i in 1..3 do
    mask(i) = if x(i) >= box(i) then 1 else 0;

  // if the position has drifted outside the physical space,
  // adjust. Divide position by the size of a bin, and add
  // (1,1,1) so we're starting at the lowest possible bin
  const temp = (x - box*mask) * bininv + (1,1,1);

  // can't cast from 3*real to 3*int (yet?)
  for i in 1..3 do
    cur(i) = temp(i) : int;

  return cur;
}


Fixed 2 race conditions in kernel 4.

Fixed 2 race conditions in kernel 4.

Fixed 2 race conditions in my kernel 4 implementation that were due to
replicating the level set work queues.  Both required a barrier to
insure that globally visible writes were completed before proceeding
to the next iteration.

This never showed up during nightly testing of the RMAT graph, because
the average degree is largish and the paths are shortish and so we got
lucky.  Torus graph testing hit one or both of the race conditions
every time, since the average degree is small and the paths can be
very long.

After a few nights of successful testing, I will promote this to the
top level version.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@19032 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  |  The equivalent of "ifdef" is provided by conditionals that are          |
//  |  evaluated at compile time.  Dead code is eliminated by the compiler.    |
//  +==========================================================================+

//  +==========================================================================+
//  |  Parallelism note -- March 2010                                          |
//  |                                                                          |
//  |  A common motif in all three kernels is a pair of loops of the form      |
//  |      forall vertices in some_set do                                      |
//  |         forall neighbors of each vertex do ....                          |
//  |  Even when this appears inside an enclosing embarrassingly parallel      |
//  |  outer loop, it will be important to parallelize this pair of loops.     |
//  |  In the most important cases, the size of the set of vertices varies     |
//  |  considerably during the execution of the kernel and there can be wide   |
//  |  variation in the number of neighbors.  So parallelization should        |
//  |  consider both loops together.                                           |
//  |                                                                          |
//  |  That said, currently Chapel cannot parallelize the innermost loops,     |
//  |  even when the innermost loop is a reduction.  Some or all of the        |
//  |  iterators that appear in this inner loop do not have parallel forms     |
//  |  at present; they use sparse or associative domains.  So the inner loops |
//  |  all appear in this code as "for" rather than "forall".  They are all    |
//  |  parallelizable and should become parallel loops eventually.             |
//  +==========================================================================+
{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();
      var heaviest_edge_weight$ : sync int = 0;

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      //       heaviest_edge_weight = max reduce [ s in G.vertices ]
      //        	                         [ w in G.edge_weight (s) ] w;

      forall s in G.vertices do
        forall w in G.edge_weight (s) do
	  heaviest_edge_weight$ = max ( w, heaviest_edge_weight$ );

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in ( G.Neighbors (s), G.edge_weight (s) ) do

	  // should be forall, requires a custom parallel iterator in the
	  // random graph case and zippering for associative domains may
	  // also present a problem

	  if w == heaviest_edge_weight$.readXX () then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ",
		  heaviest_edge_weight$ . readFF ());
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in (G.Neighbors (s), G.edge_weight (s) ) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
			       Heavy_Edge_List     : domain,
			       Heavy_Edge_Subgraph : [],
			       in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance$            : [vertex_domain] sync int = -1;

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance$ ( y ).writeFF (0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do { // eventually, will be forall

	      if min_distance$ (w).readXX () < 0 then {

		if min_distance$ (w).readFE () < 0 then {
		  Next_Level.add (w);
		  Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
		  min_distance$ (w).writeEF (path_length);
		}
		else
		  min_distance$ (w).writeEF (path_length);
	      }

	      // min_distance$ must have been set by some thread by now

	      if min_distance$ (w).readFF () == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  // ============================================================
  // generic class structure must be defined outside of
  // generic procedure.  used by Betweenness Centrality kernel 4.
  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
  }
  use ReplicatedDist;

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
					   Between_Cent : [] real,
					   out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Had to change declaration below
      //    type Sparse_Vertex_List = sparse subdomain ( G.vertices );
      // to accommodate block distribution of G.vertices

      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] sync real = 0.0;
      var Sum_Min_Dist$ : sync real = 0.0;

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      for/*all*/ s in starting_vertices do {

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

	// --------------------------------------------------
	// all locally declared variables become private data
	// for each instance of the parallel for loop
	// --------------------------------------------------

  	var min_distance$  : [vertex_domain] sync int       = -1; //why?
	var path_count$    : [vertex_domain] sync real (64) = 0.0;
	var depend         : [vertex_domain] real           = 0.0;
	var Lcl_Sum_Min_Dist: sync real                     = 0.0;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance$) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

	var Active_Level: [rcDomain] Level_Set (Sparse_Vertex_List);
        var Next_Level: [rcDomain] Level_Set (Sparse_Vertex_List);
        var Active_Remaining: [LocaleSpace] bool = true;

        coforall loc in Locales do on loc {
          rcLocal(Active_Level) = new Level_Set (Sparse_Vertex_List);
          rcLocal(Active_Level).previous = nil;
          rcLocal(Next_Level) = new Level_Set (Sparse_Vertex_List);
          rcLocal(Next_Level).previous = rcLocal(Active_Level);
        }

	var current_distance : int = 0;

	// establish the initial level sets for the
	// breadth-first traversal from s

        on s {
          rcLocal(Active_Level).Members.add ( s );
          rcLocal(Next_Level).Members.clear ();
          min_distance$ (s) . writeFF (0);
          path_count$ (s) . writeFF (1);
        }

	while || reduce Active_Remaining do {

	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;

            // barrier
            var count: sync int = numLocales;
            var barrier: single bool;

            // for remote value forwarding
            const current_distance_c = current_distance;
            coforall loc in Locales do on loc {
             forall u in rcLocal(Active_Level).Members do { // sparse

              forall (v, w) in ( G.Neighbors (u), G.edge_weight (u) ) do on v {

		// should be forall, requires a custom parallel iterator in the
		// random graph case and zippering for associative domains

		// --------------------------------------------
		// add any unmarked neighbors to the next level
		// --------------------------------------------

		if  ( FILTERING &&  w % 8 != 0 ) || !FILTERING  then
		  if  min_distance$ (v) . readXX () < 0  then
		    {
		      if  min_distance$ (v) . readFE () < 0  then
			{
			  min_distance$ (v).writeEF (current_distance_c);
			  rcLocal(Next_Level).Members.add (v);
			  if VALIDATE_BC then
			    Lcl_Sum_Min_Dist += current_distance_c;
			}
		      else
                        // could min_distance$(v) be < current_distance?
			min_distance$ (v) . writeEF (current_distance_c);
		    }

		// ------------------------------------------------
		// only neighbors of  u  that are in the next level
		// are on shortest paths from s through v.  Some
		// thread will have set  min_distance$ (v) by the
		// time this code is reached, whether  v  lies in
		// the previous, the current or the next level.
		// ------------------------------------------------

		if  min_distance$ (v).readFF () == current_distance_c
		  then
		    path_count$ (v) += path_count$ (u).readFF ();
	      }
	    };

            // barrier needed to insure all updates to Next_Level are complete
            var myc = count;
            if myc==1 {
              barrier=true;
            } else {
              count = myc-1;
              barrier;
            }

            rcLocal(Active_Level) = rcLocal(Next_Level);
            rcLocal(Next_Level)   = new Level_Set (Sparse_Vertex_List);

	    // rcLocal(Next_Level).Members.clear ();
	    rcLocal(Next_Level).previous = rcLocal(Active_Level);

            Active_Remaining[here.id] = rcLocal(Active_Level).Members.numIndices:bool;

            }

	  };  // end forward pass

	if VALIDATE_BC then
	  Sum_Min_Dist$ += Lcl_Sum_Min_Dist;

	// -------------------------------------------------------------
	// compute the dependencies recursively, traversing the vertices
	// of the graph in non-increasing order of distance (reverse
	// ordering from the initial traversal)
	// -------------------------------------------------------------

	var graph_diameter = current_distance - 1;

	if DEBUG_KERNEL4 then
	  writeln ( " graph diameter from starting node ", s,
		    "  is ", graph_diameter );

        // barrier
        var count: sync int = numLocales;
        var barrier: [2..graph_diameter] single bool;

        coforall loc in Locales do on loc {
          delete rcLocal(Next_Level);	               // it's empty
          rcLocal(Next_Level)   = rcLocal(Active_Level).previous;  // back up to last level
          delete rcLocal(Active_Level);
          rcLocal(Active_Level) = rcLocal(Next_Level);

          for current_distance in 2 .. graph_diameter by -1 do {

            rcLocal(Next_Level)   = rcLocal(Active_Level).previous;
            delete rcLocal(Active_Level);
            rcLocal(Active_Level) = rcLocal(Next_Level);

            // inner reduction should parallelize eventually; compiler
            // serializes it today (and warns us that it did)

            forall u in rcLocal(Active_Level).Members do
              {
                depend (u) = + reduce
                  [ (v, w)  in ( G.Neighbors (u), G.edge_weight (u) ) ]
                  if ( min_distance$ (v).readFF () == current_distance ) &&
                  ( ( FILTERING && w % 8 != 0 || !FILTERING ) )
                  then
                    ( path_count$ (u) . readFF () /
                      path_count$ (v) . readFF () )      *
                      ( 1.0 + depend (v) );

                // do not need conditional u != s

                Between_Cent$ (u) += depend (u);
              }
            // barrier needed to insure all updates to depend are completed
            var myc = count;
            if myc==1 {
              count = numLocales;
              barrier[current_distance] = true;
            } else {
              count = myc-1;
              barrier[current_distance];
            }
          };
          delete rcLocal(Active_Level);
        }

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	var n0            = + reduce [v in vertex_domain] (G.n_Neighbors (v)== 0);
	var n_edges       = + reduce [v in vertex_domain] G.n_Neighbors (v);
	var N_VERTICES    = vertex_domain.numIndices;
	var TEPS          = 7.0 * N_VERTICES * (N_VERTICES - n0) / K4_time;
	var Adjusted_TEPS = n_edges * (N_VERTICES - n0) / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
	Sum_Min_Dist = Sum_Min_Dist$;

      Between_Cent = Between_Cent$;

    } // end of Brandes' betweenness centrality calculation

}

module SSCA2_kernels

//  +==========================================================================+
//  |  Polymorphic Implementation of SSCA #2, Kernels 2-4                      |
//  |                                                                          |
//  |  Each kernel takes a graph argument which provides for each vertex       |
//  |  1.  an iterator for its set of neighbors                                |
//  |  2.  a parallel integer array of edge weights, which can be zipper       |
//  |      iterated with the set of neighbors                                  |
//  |  3.  the number of neighbors                                             |
//  |                                                                          |
//  |  These are the only requirements on the representation of the graph.     |
//  |                                                                          |
//  |  Filtering in Kernel 4 is turned on or off by a compilation time param.  |
//  |  The equivalent of "ifdef" is provided by conditionals that are          |
//  |  evaluated at compile time.  Dead code is eliminated by the compiler.    |
//  +==========================================================================+

//  +==========================================================================+
//  |  Parallelism note -- March 2010                                          |
//  |                                                                          |
//  |  A common motif in all three kernels is a pair of loops of the form      |
//  |      forall vertices in some_set do                                      |
//  |         forall neighbors of each vertex do ....                          |
//  |  Even when this appears inside an enclosing embarrassingly parallel      |
//  |  outer loop, it will be important to parallelize this pair of loops.     |
//  |  In the most important cases, the size of the set of vertices varies     |
//  |  considerably during the execution of the kernel and there can be wide   |
//  |  variation in the number of neighbors.  So parallelization should        |
//  |  consider both loops together.                                           |
//  |                                                                          |
//  |  That said, currently Chapel cannot parallelize the innermost loops,     |
//  |  even when the innermost loop is a reduction.  Some or all of the        |
//  |  iterators that appear in this inner loop do not have parallel forms     |
//  |  at present; they use sparse or associative domains.  So the inner loops |
//  |  all appear in this code as "for" rather than "forall".  They are all    |
//  |  parallelizable and should become parallel loops eventually.             |
//  +==========================================================================+
{
  use SSCA2_compilation_config_params, Time;

  var stopwatch : Timer;

  // ========================================================
  //                           KERNEL 2:
  // ========================================================
  // Find the edges with the largest edges.  Return a list of
  // edges, all of which have the largest weight.
  // ========================================================

  proc largest_edges ( G, heavy_edge_list :domain )

    // edge_weights can be either an array over an associative
    // domain or over a sparse domain.  the output  heavy_edge_list
    // can either kind of domain or something else purpose-built
    // for this task.
    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();
      var heaviest_edge_weight$ : sync int = 0;

      // ---------------------------------------------------------
      // find heaviest edge weight in a single pass over all edges
      // ---------------------------------------------------------

      //       heaviest_edge_weight = max reduce [ s in G.vertices ]
      //        	                         [ w in G.edge_weight (s) ] w;

      forall s in G.vertices do
        forall w in G.edge_weight (s) do
	  heaviest_edge_weight$ = max ( w, heaviest_edge_weight$ );

      // ---------------------------------------------
      // in a second pass over all edges, extract list
      // of all edges  matching the heaviest weight
      // ---------------------------------------------

      forall s in G.vertices do
        forall (t, w) in ( G.Neighbors (s), G.edge_weight (s) ) do

	  // should be forall, requires a custom parallel iterator in the
	  // random graph case and zippering for associative domains may
	  // also present a problem

	  if w == heaviest_edge_weight$.readXX () then {
	    heavy_edge_list.add ( (s,t) );
	  };

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 2: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }

      // ------------------------------------------------
      // should be able to write a user-defined "maxlocs"
      // reduction more efficiently than this scheme
      // ------------------------------------------------

      if DEBUG_KERNEL2 then {
	writeln ();
	writeln ( "Heaviest weight      : ",
		  heaviest_edge_weight$ . readFF ());
	writeln ( "Number of heavy edges:", heavy_edge_list.numIndices );
	writeln ();
	writeln ( "Edges with largest weight and other neighbors:" );
	for (s,t) in heavy_edge_list do {
	  writeln ("edge   ", (s,t));
	  for (v,w) in (G.Neighbors (s), G.edge_weight (s) ) do
	    writeln ("      ", v, " ", w);}
      }
    };

  // ===================================================================
  //                              KERNEL 3:
  // ===================================================================
  // For each root (heavy) edge, find the subgraph (vertices and edges)
  // defined by directed paths of length no greater than max_path_length
  // in which the first edge traversed is the root edge
  // ===================================================================

  proc rooted_heavy_subgraphs ( G,
			       Heavy_Edge_List     : domain,
			       Heavy_Edge_Subgraph : [],
			       in max_path_length  : int )

    // -------------------------------------------------------------------------
    // there is a classic space versus time tradeoff.  if the subgraphs expanded
    // by breadth first search are small, it would make sense to use a hash
    // table or an associative domain to represent the search.  If the subgraphs
    // are large, using a full length vector to represent the search is more
    // appropriate.  We expect small diameters for power law graphs, so we
    // expect large subgraphs.
    // -------------------------------------------------------------------------

    {
      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      const vertex_domain = G.vertices;

      forall ( x, y ) in Heavy_Edge_List do {
	var Active_Level, Next_Level : domain ( index (vertex_domain) );
	var min_distance$            : [vertex_domain] sync int = -1;

	if DEBUG_KERNEL3 then
	  writeln ( " Building heavy edge subgraph from pair:", (x,y) );
	Active_Level.add ( y );
	Next_Level.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).nodes.clear ();
	Heavy_Edge_Subgraph ( (x, y) ).edges.clear ();
	min_distance$ ( y ).writeFF (0);

	Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (x, y) );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( x );
	Heavy_Edge_Subgraph ( (x, y) ).nodes.add ( y );

	for path_length in 1 .. max_path_length do {

	  forall v in Active_Level do {

	    forall w in G.Neighbors (v) do { // eventually, will be forall

	      if min_distance$ (w).readXX () < 0 then {

		if min_distance$ (w).readFE () < 0 then {
		  Next_Level.add (w);
		  Heavy_Edge_Subgraph ( (x, y) ).nodes.add (w);
		  min_distance$ (w).writeEF (path_length);
		}
		else
		  min_distance$ (w).writeEF (path_length);
	      }

	      // min_distance$ must have been set by some thread by now

	      if min_distance$ (w).readFF () == path_length then {
		Heavy_Edge_Subgraph ( (x, y) ).edges.add ( (v, w) );
	      }
	    }
	  }

	  if path_length < max_path_length then {
	    Active_Level = Next_Level;
	    Next_Level.clear ();
	  }
	}
      }

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	writeln ( "Elapsed time for Kernel 3: ", stopwatch.elapsed (),
		  " seconds");
	stopwatch.clear ();
      }
    } // end of rooted_heavy_subgraphs

  // ============================================================
  // generic class structure must be defined outside of
  // generic procedure.  used by Betweenness Centrality kernel 4.
  // ------------------------------------------------------------
  // The set of vertices at a particular distance from s form a
  // level set.  The class allows the full set of vertices to be
  // partitioned into a linked list of level sets.  Each instance
  // of the outer loop in kernel 4 creates such a partitioning.
  // ============================================================

  class Level_Set {
    type Sparse_Vertex_List;
    var Members  : Sparse_Vertex_List;
    var previous : Level_Set (Sparse_Vertex_List);
  }
  use ReplicatedDist;

  // ==================================================================
  //                              KERNEL 4
  // ==================================================================
  // Calculate Betweenness Centrality for simple unweighted directed or
  // undirected graphs, using Madduri, et.al.'s modification of
  // Brandes's 2001 algorithm
  // ==================================================================

  proc approximate_betweenness_centrality ( G, starting_vertices,
					   Between_Cent : [] real,
					   out Sum_Min_Dist : real )

    // -----------------------------------------------------------------------
    // The betweenness centrality metric for a given node  v  is defined
    // as the double sum over s not equal to v and  t not equal to
    // either s or v of the ratio of the number of shortest paths from s to t
    // passing through v to the number of shortest paths from s to t.
    //
    // Brandes's algorithm decomposes the computation of this metric into,
    // first, separate sums for each vertex s, which can be computed
    // independently in parallel, and
    // two, a recursive, tree-based, calculation of the path counts for
    // any particular s.
    // The complexity of this algorithm is O ( |V||E| ) time for an unweighted
    // graph.  The algorithm requires O ( |V| ) temporary space for each
    // process that executes instances of the outermost loop.
    // -----------------------------------------------------------------------
    {
      const vertex_domain = G.vertices;

      // Had to change declaration below
      //    type Sparse_Vertex_List = sparse subdomain ( G.vertices );
      // to accommodate block distribution of G.vertices

      type Sparse_Vertex_List = domain(index(vertex_domain));

      var Between_Cent$ : [vertex_domain] sync real = 0.0;
      var Sum_Min_Dist$ : sync real = 0.0;

      // ------------------------------------------------------
      // Each iteration of the outer loop of Brandes's algorithm
      // computes the contribution (the "dependency" metric) for
      // one particular vertex  (s)  independently.
      // ------------------------------------------------------

      if PRINT_TIMING_STATISTICS then stopwatch.start ();

      for/*all*/ s in starting_vertices do {

	if DEBUG_KERNEL4 then writeln ( "expanding from starting node ", s );

	// --------------------------------------------------
	// all locally declared variables become private data
	// for each instance of the parallel for loop
	// --------------------------------------------------

  	var min_distance$  : [vertex_domain] sync int       = -1; //why?
	var path_count$    : [vertex_domain] sync real (64) = 0.0;
	var depend         : [vertex_domain] real           = 0.0;
	var Lcl_Sum_Min_Dist: sync real                     = 0.0;

	// The structure of the algorithm depends on a breadth-first
	// traversal. Each vertex will be marked by the length of
	// the shortest path (min_distance$) from s to it. The array
	// path_count$ will hold a count of the number of shortest
	// paths from s to this node.  The number of paths in moderate
	// sized tori exceeds 2**64.

	var Active_Level: [rcDomain] Level_Set (Sparse_Vertex_List);
        var Next_Level: [rcDomain] Level_Set (Sparse_Vertex_List);
        var Active_Remaining: [LocaleSpace] bool = true;

        coforall loc in Locales do on loc {
          rcLocal(Active_Level) = new Level_Set (Sparse_Vertex_List);
          rcLocal(Active_Level).previous = nil;
          rcLocal(Next_Level) = new Level_Set (Sparse_Vertex_List);
          rcLocal(Next_Level).previous = rcLocal(Active_Level);
        }

	var current_distance : int = 0;

	// establish the initial level sets for the
	// breadth-first traversal from s

        on s {
          rcLocal(Active_Level).Members.add ( s );
          rcLocal(Next_Level).Members.clear ();
          min_distance$ (s) . writeFF (0);
          path_count$ (s) . writeFF (1);
        }

	while || reduce Active_Remaining do {

	    // ------------------------------------------------
	    // expand the neighbor sets for all vertices at the
	    // current distance from the starting vertex  s
	    // ------------------------------------------------

	    current_distance += 1;
            // for remote value forwarding
            const current_distance_c = current_distance;
            coforall loc in Locales do on loc {
             forall u in rcLocal(Active_Level).Members do { // sparse

              forall (v, w) in ( G.Neighbors (u), G.edge_weight (u) ) do on v {

		// should be forall, requires a custom parallel iterator in the
		// random graph case and zippering for associative domains

		// --------------------------------------------
		// add any unmarked neighbors to the next level
		// --------------------------------------------

		if  ( FILTERING &&  w % 8 != 0 ) || !FILTERING  then
		  if  min_distance$ (v) . readXX () < 0  then
		    {
		      if  min_distance$ (v) . readFE () < 0  then
			{
			  min_distance$ (v).writeEF (current_distance_c);
			  rcLocal(Next_Level).Members.add (v);
			  if VALIDATE_BC then
			    Lcl_Sum_Min_Dist += current_distance_c;
			}
		      else
                        // could min_distance$(v) be < current_distance?
			min_distance$ (v) . writeEF (current_distance_c);
		    }

		// ------------------------------------------------
		// only neighbors of  u  that are in the next level
		// are on shortest paths from s through v.  Some
		// thread will have set  min_distance$ (v) by the
		// time this code is reached, whether  v  lies in
		// the previous, the current or the next level.
		// ------------------------------------------------

		if  min_distance$ (v).readFF () == current_distance_c
		  then
		    path_count$ (v) += path_count$ (u).readFF ();
	      }
	    };

            rcLocal(Active_Level) = rcLocal(Next_Level);
            rcLocal(Next_Level)   = new Level_Set (Sparse_Vertex_List);

	    // rcLocal(Next_Level).Members.clear ();
	    rcLocal(Next_Level).previous = rcLocal(Active_Level);

            Active_Remaining[here.id] = rcLocal(Active_Level).Members.numIndices:bool;

            }

	  };  // end forward pass

	if VALIDATE_BC then
	  Sum_Min_Dist$ += Lcl_Sum_Min_Dist;

	// -------------------------------------------------------------
	// compute the dependencies recursively, traversing the vertices
	// of the graph in non-increasing order of distance (reverse
	// ordering from the initial traversal)
	// -------------------------------------------------------------

	var graph_diameter = current_distance - 1;

	if DEBUG_KERNEL4 then
	  writeln ( " graph diameter from starting node ", s,
		    "  is ", graph_diameter );

        coforall loc in Locales do on loc {
          delete rcLocal(Next_Level);	               // it's empty
          rcLocal(Next_Level)   = rcLocal(Active_Level).previous;  // back up to last level
          delete rcLocal(Active_Level);
          rcLocal(Active_Level) = rcLocal(Next_Level);

          for current_distance in 2 .. graph_diameter by -1 do {

            rcLocal(Next_Level)   = rcLocal(Active_Level).previous;
            delete rcLocal(Active_Level);
            rcLocal(Active_Level) = rcLocal(Next_Level);

            // inner reduction should parallelize eventually; compiler
            // serializes it today (and warns us that it did)

            forall u in rcLocal(Active_Level).Members do
              {
                depend (u) = + reduce
                  [ (v, w)  in ( G.Neighbors (u), G.edge_weight (u) ) ]
                  if ( min_distance$ (v).readFF () == current_distance ) &&
                  ( ( FILTERING && w % 8 != 0 || !FILTERING ) )
                  then
                    ( path_count$ (u) . readFF () /
                      path_count$ (v) . readFF () )      *
                      ( 1.0 + depend (v) );

                // do not need conditional u != s

                Between_Cent$ (u) += depend (u);
              }
          };
          delete rcLocal(Active_Level);
        }

      }; // closure of outer embarassingly parallel forall

      if PRINT_TIMING_STATISTICS then {
	stopwatch.stop ();
	var K4_time = stopwatch.elapsed ();
	stopwatch.clear ();
	writeln ( "Elapsed time for Kernel 4: ", K4_time, " seconds");

	var n0            = + reduce [v in vertex_domain] (G.n_Neighbors (v)== 0);
	var n_edges       = + reduce [v in vertex_domain] G.n_Neighbors (v);
	var N_VERTICES    = vertex_domain.numIndices;
	var TEPS          = 7.0 * N_VERTICES * (N_VERTICES - n0) / K4_time;
	var Adjusted_TEPS = n_edges * (N_VERTICES - n0) / K4_time;

	writeln ( "                     TEPS: ", TEPS );
	writeln ( " edge count adjusted TEPS: ", Adjusted_TEPS );
      }

      if VALIDATE_BC then
	Sum_Min_Dist = Sum_Min_Dist$;

      Between_Cent = Between_Cent$;

    } // end of Brandes' betweenness centrality calculation

}