Small edits to BradsBlock1DPar

Small edits to BradsBlock1DPar

* Put in assertion that local distribution class is always
  constructed on correct locale
* Renamed locid to localeIdx for clarity
* Updated some comments

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14432 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: Label which routines are the compiler's interface vs. an
// internal interface.

// TODO: Label the linkage fields.

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist {
  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;
  //
  // NOTE: originally I wrote this as:
  //
  //   const targetLocs = Locales;
  //   const targetLocDom = targetLocs.domain;
  //
  // but this had two interesting problems:
  //
  // (1) the capture of the targetLocs array only makes a copy of the
  //     array, not of its domain, so if someone were to resize the
  //     LocaleSpace domain outside of the class, this would inadvertantly
  //     get resized as well.  We really want to make a copy of both the
  //     domain and array (which I now do in the constructor).
  //
  // (2) Initializing targetLocs with Locales prevents us from assigning
  //     anything other than a numLocales-ary array of locales to it
  //     which is stricter than I wanted -- I only wanted to restrict
  //     us to a 1D array of locales.  In our current model, this would
  //     require us to have some sort of [domain(1)] locale declaration
  //     to make it work.  Or we could think about whether a model in
  //     which only the array's static type information is inferred
  //     for a field?

  //
  // an array of local distribution class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  // WANTED:
  //
  /*
  const locDist: [loc in targetLocDom] LocBlock1DDist(idxType)
                   = on targetLocs(loc) do
                       new LocBlock1DDist(idxType, targetLocDom.order(loc), this);
  */
  //
  // but this doesn't work yet because an array forall initializer
  // apparently can't refer to a local member domain.
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);
  //
  // WORKAROUND: Initialize in the constructor instead
  //

  //
  // TODO: 0-base the target locales domain in the constructor rather
  // than whenever it's used for simplicity?
  //
  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [?targetLocalesDomain] locale = Locales) {
    boundingBox = bbox;
    targetLocDom = targetLocalesDomain;
    targetLocs = targetLocales;
    //
    // WANT TO DO:
    /*
    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);
    */
    //
    // BUT results in a _heapAlloc(type int(64)) call.
    // WORKAROUND:
    helpConstruct();
  }
  //
  // WORKAROUND CONTINUED:
  //
  def helpConstruct() {
    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);
  }
  //
  // END WORKAROUND
  //

  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newDomain(inds, type domIdxType = idxType) where domIdxType == idxType {
    return new Block1DDom(idxType, this, inds);
  }

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set

    // TODO: Should this be able to be written as myChunk[inds] ???
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: If targetLocsDom was 0-based, would this function be easier?
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.
  //
  def ind2locInd(ind: idxType) {
    const ind0 = ind - boundingBox.low;
    const loc0 = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    const locInd = loc0: index(targetLocs.domain) + targetLocs.domain.low;
    return locInd;
  }

  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {
  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);
  const localeIdx: int;
  const loc: locale;

  //
  // Compute what chunk of index(1) is owned by the current locale
  // Arguments:
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const localeIdx0 = dist.targetLocDom.order(localeIdx); // 0-based locale ID
    const blo = if (localeIdx0 == 0) then min(idxType)
                else procToData((numelems: real * localeIdx0) / numlocs, lo);
    const bhi = if (localeIdx0 == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx0+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom {
  //
  // The index types of the domain
  //
  type idxType;

  //
  // a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  //
  // an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType);

  def initialize() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocBlock1DDom(idxType, this,
                                           dist.getChunk(whole, localeIdx));
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));
  }

  //
  // the iterator for the domain -- currently sequential
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def newArray(type elemType) {
    return new Block1DArr(idxType, elemType, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }
}

//
// the local domain class
//
class LocBlock1DDom {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType);

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type elemType;

  //
  // the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType);

  //
  // an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, elemType);

  def initialize() {
    for localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, elemType, dom.locDoms(localeIdx));
  }

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see th
  // example for general notes on the approach
  //

  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {
    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type elemType;

  //
  // a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType);

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] elemType;

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: Label which routines are the compiler's interface vs. an
// internal interface.

// TODO: Label the linkage fields.

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist {
  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;
  //
  // NOTE: originally I wrote this as:
  //
  //   const targetLocs = Locales;
  //   const targetLocDom = targetLocs.domain;
  //
  // but this had two interesting problems:
  //
  // (1) the capture of the targetLocs array only makes a copy of the
  //     array, not of its domain, so if someone were to resize the
  //     LocaleSpace domain outside of the class, this would inadvertantly
  //     get resized as well.  We really want to make a copy of both the
  //     domain and array (which I now do in the constructor).
  //
  // (2) Initializing targetLocs with Locales prevents us from assigning
  //     anything other than a numLocales-ary array of locales to it
  //     which is stricter than I wanted -- I only wanted to restrict
  //     us to a 1D array of locales.  In our current model, this would
  //     require us to have some sort of [domain(1)] locale declaration
  //     to make it work.  Or we could think about whether a model in
  //     which only the array's static type information is inferred
  //     for a field?

  //
  // an array of local distribution class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  // WANTED:
  //
  /*
  const locDist: [loc in targetLocDom] LocBlock1DDist(idxType)
                   = on targetLocs(loc) do
                       new LocBlock1DDist(idxType, targetLocDom.order(loc), this);
  */
  //
  // but this doesn't work yet because an array forall initializer
  // apparently can't refer to a local member domain.
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);
  //
  // WORKAROUND: Initialize in the constructor instead
  //

  //
  // TODO: 0-base the target locales domain in the constructor rather
  // than whenever it's used for simplicity?
  //
  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [?targetLocalesDomain] locale = Locales) {
    boundingBox = bbox;
    targetLocDom = targetLocalesDomain;
    targetLocs = targetLocales;
    //
    // WANT TO DO:
    /*
    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);
    */
    //
    // BUT results in a _heapAlloc(type int(64)) call.
    // WORKAROUND:
    helpConstruct();
  }
  //
  // WORKAROUND CONTINUED:
  //
  def helpConstruct() {
    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);
  }
  //
  // END WORKAROUND
  //

  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What would we do if domIdxType did not match idxType?
  //
  def newDomain(inds, type domIdxType = idxType) where domIdxType == idxType {
    return new Block1DDom(idxType, this, inds);
  }

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set

    // TODO: Should this be able to be written as myChunk[inds] ???
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: If targetLocsDom was 0-based, would this function be easier?
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.
  //
  def ind2locInd(ind: idxType) {
    const ind0 = ind - boundingBox.low;
    const loc0 = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    const locInd = loc0: index(targetLocs.domain) + targetLocs.domain.low;
    return locInd;
  }

  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {
  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  // TODO: Rename locid -- it's not very clear
  //
  const myChunk: domain(1, idxType);
  const locid: int;
  const loc: locale;

  //
  // Compute what chunk of index(1) is owned by the current locale
  // Arguments:
  //
  def LocBlock1DDist(type idxType,
                     _locid: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    locid = _locid;
    loc = dist.targetLocs(locid);
    //
    // TODO -- assert that "loc == here" at this point for safety?
    //
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const locid0 = dist.targetLocDom.order(locid); // 0-based locale ID
    const blo = if (locid0 == 0) then min(idxType)
                else procToData((numelems: real * locid0) / numlocs, lo);
    const bhi = if (locid0 == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (locid0+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom {
  //
  // The index types of the domain
  //
  type idxType;

  //
  // a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  //
  // an array of local domain class descriptors -- set up in
  // initialize() below
  //
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType);

  def initialize() {
    for locid in dist.targetLocDom do
      on dist.targetLocs(locid) do
        locDoms(locid) = new LocBlock1DDom(idxType, this,
                                           dist.getChunk(whole, locid));
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));
  }

  //
  // the iterator for the domain -- currently sequential
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def newArray(type elemType) {
    return new Block1DArr(idxType, elemType, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }
}

//
// the local domain class
//
class LocBlock1DDom {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType);

  //
  // a local domain describing the indices owned by this locale
  //
  //
  // TODO: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  var myBlock: domain(1, idxType);

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: Are these used internally only?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type elemType;

  //
  // the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType);

  //
  // an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, elemType);

  def initialize() {
    for locid in dom.dist.targetLocDom do
      on dom.dist.targetLocs(locid) do
        locArr(locid) = new LocBlock1DArr(idxType, elemType, dom.locDoms(locid));
  }

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see th
  // example for general notes on the approach
  //

  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {
    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type elemType;

  //
  // a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType);

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] elemType;

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}


Turn off output if testing

Turn off output if testing

use Histogram;
use Random;
use Time;

// Test flags
config const isTest=false;
config const isPerf=false;
config const doBrute=false;

// Performance test parameters
config const nParticles=10000;

// Input/Output filenames
config const fn1 = "test.dat";
config const fn2 = "test.dat";
config const pairfn = "test-DD.dat";

// Dimension constanst
param NDIM=3;
param NEXT=2; // w, r2
param NTOT=NDIM+NEXT;
const W=NDIM;
const R2=NDIM+1;
const ParticleAttrib = 0.. #NTOT;
const DimSpace = 0.. #NDIM;

// Tree parameters
config const minpart=500;

// Histogram parameters
config const smax=200.0;
config const smax2=smax**2;
config const nmubins=5;
config const nsbins=5;

// Global variables
var gtime1 : Timer;

proc main() {
  doPairs();
}

class Particle3D {
  var npart : int = 0;
  var Darr : domain(2);
  var Dpart : domain(1);
  var arr : [Darr] real;

  var _tmp : [Dpart] real;
  var _n1, _ndx : [Dpart] int;

  proc Particle3D(npart1 : int, random : bool = false) {
    npart = npart1;
    Darr = {ParticleAttrib, 0.. #npart};
    Dpart = {0.. #npart};
    if random {
      var rng = new RandomStream();
      var x, y, z : real;
      for ii in Dpart {
        x = rng.getNext()*1000.0; y = rng.getNext()*1000.0; z = rng.getNext()*1000.0;
        arr[0,ii] = x; arr[1, ii] = y; arr[2, ii] = z;
        arr[3,ii] = 1.0;
        arr[4,ii] = x**2 + y**2 + z**2;
      }
      delete rng;
    }
  }

  proc this(ii : int) : NTOT*real {
    // ??? Hardcoded hack!
    return (arr[0,ii],arr[1,ii],arr[2,ii],arr[3,ii],arr[4,ii]);
  }

  // Reorder all the arrays according to _ndx (we assume you have set this)
  proc reorder(idom : domain(1)) {

    for kk in ParticleAttrib {
      forall ii in idom {
        _tmp[_ndx[ii]] = arr[kk,ii];
      }
      forall ii in idom {
        arr[kk,ii] = _tmp[ii];
      }
    }
  }

  // Shuffles the elements of the array...
  // Note that the random number generator cannot cover the space
  // of all permutations, but that's fine for what we need.
  proc shuffle() {
    // Fill in _ndx
    forall ii in Dpart {
      _ndx[ii] = ii;
    }

    // Set the random number generator
    var rng = new RandomStream(41);
    var jj : int;
    for ii in 0..(npart-2) {
      jj = (rng.getNext()*(npart-ii)):int + ii;
      _ndx[jj] <=> _ndx[ii];
    }
    delete rng;

    reorder(Dpart);
  }

}

proc countLines(fn : string) : int {
  var ff = open(fn, iomode.r);
  var ipart = 0;
  for iff in ff.lines() do ipart +=1;
  ff.close();
  return ipart;
}

proc readFile(fn : string) : Particle3D  {
  var npart = countLines(fn);
  var pp = new Particle3D(npart);

  var ff = openreader(fn);
  var ipart = 0;
  var x,y,z,w,r2 : real;
  while (ff.read(x,y,z,w)) {
    r2 = x**2 + y**2 + z**2;
    pp.arr[..,ipart] = [x,y,z,w,r2];
    ipart += 1;
  }
  ff.close();

  return pp;
}

proc splitOn(pp : Particle3D, idom : domain(1), splitDim : int, xsplit : real) : int {
  // Setup for a prefix scan
  forall ii in idom {
    if (pp.arr[splitDim,ii] < xsplit) {
      pp._n1[ii] = 1;
    } else {
      pp._n1[ii] = 0;
    }
  }

  var lnpart : int = 0;
  for ii in idom do lnpart += pp._n1[ii];
  var li = idom.low;
  var ri = li+lnpart;
  for ii in idom {
    if (pp._n1[ii] == 1) {
      pp._ndx[ii] = li;
      li += 1;
    } else {
      pp._ndx[ii] = ri;
      ri += 1;
    }
  }

  pp.reorder(idom);

  return lnpart;
}

class KDNode {
  var lo, hi,npart,id : int;
  var dom : domain(1);
  var xcen : [DimSpace]real;
  var rcell : real;
  var left, right : KDNode;

  proc isLeaf() : bool {
    return (left==nil) && (right==nil);
  }

  proc ~KDNode() {
    if left then delete left;
    if right then delete right;
  }

}

proc BuildTree(pp : Particle3D, lo : int, hi : int, id : int) : KDNode  {
  var me : KDNode = new KDNode();
  me.lo = lo;
  me.hi = hi;
  me.dom = {lo..hi};
  me.id = id;
  me.npart = (me.hi-me.lo)+1;

  //  work out xcen and vantage point radius
  var pmin, pmax : [DimSpace] real;
  for idim in DimSpace {
    pmin[idim] = min reduce pp.arr[idim,lo..hi];
    pmax[idim] = max reduce pp.arr[idim,lo..hi];
  }
  me.xcen = (pmax+pmin)/2.0;
  me.rcell = 0.0;
  var r1 : real;
  for ii in lo..hi {
    r1 = 0.0;
    for idim in DimSpace {
      r1 += (pp.arr[idim,ii]-me.xcen[idim])**2;
    }
    r1 = sqrt(r1);
    if (r1 > me.rcell) then me.rcell = r1;
  }

  // Continue to split
  if (me.npart <= minpart) then return me; // Don't split further.

  // Find dimension to split on
  var dx : [DimSpace] real;
  dx = pmax - pmin;
  var splitDim = 0;
  for idim in DimSpace {
    if (dx[idim] > dx[splitDim]) then splitDim=idim;
  }

  // Split
  var lnpart = splitOn(pp,me.dom,splitDim, me.xcen[splitDim]);
  me.left  = BuildTree(pp, lo, lo+lnpart-1, 2*id+1);
  me.right = BuildTree(pp, lo+lnpart, hi,2*id+2);
  return me;
}

proc TreeAccumulate(hh : UniformBins, p1, p2 : Particle3D, node1, node2 : KDNode) {
  // Compute the distance between node1 and node2
  var rr = sqrt (+ reduce(node1.xcen - node2.xcen)**2);
  var rmin = rr - (node1.rcell+node2.rcell);

  // If distance is greater than all cases
  if (rmin > smax) then return;

  // If both nodes are leaves
  if (node1.isLeaf() & node2.isLeaf()) {
    begin smuAccumulate(hh, p1, p2, node1.dom, node2.dom,1);
    return;
  }

  // If one node is a leaf
  if (node1.isLeaf()) {
    TreeAccumulate(hh, p1, p2, node1, node2.left);
    TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }
  if (node2.isLeaf()) {
    TreeAccumulate(hh, p1, p2, node1.left, node2);
    TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  }

  // Split the larger case;
  if (node1.npart > node2.npart) {
    TreeAccumulate(hh, p1, p2, node1.left, node2);
    TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  } else {
    TreeAccumulate(hh, p1, p2, node1, node2.left);
    TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }

}

// The basic pair counter
proc smuAccumulate(hh : UniformBins, p1,p2 : Particle3D, d1,d2 : domain(1), scale : real) {
  forall ii in d1 { // Loop over first set of particles

    var x1,y1,z1,w1,r2 : real;
    var sl, s2, l1, s1, l2, mu, wprod : real;
    (x1,y1,z1,w1,r2) = p1[ii];

    for jj in d2 { // Second set of particles
      mu=2*(p2.arr[0,jj]*x1 + p2.arr[1,jj]*y1 + p2.arr[2,jj]*z1);
      sl = r2 - p2.arr[R2,jj];
      l1 = r2 + p2.arr[R2,jj];
      s2 = l1 - mu;
      l2 = l1 + mu;
      if ((s2 >= smax2) || (s2 < 1.0e-20)) then continue;
      wprod = scale * w1 * p2.arr[W,jj];
      s1 = sqrt(s2);
      mu = sl/(s1*sqrt(l2));
      if (mu < 0) then mu = -mu;

      hh.add((s1,mu),wprod);
    }
  }
}

proc doPairs() {
  var tt : Timer;

  // Read in the file
  tt.clear(); tt.start();
  var pp1, pp2 : Particle3D;
  if isPerf {
    pp1 = new Particle3D(nParticles, true);
    pp2 = new Particle3D(nParticles, true);
  } else {
    pp1 = readFile(fn1);
    pp2 = readFile(fn2);
    if !isTest {
      writef("Read in %i lines from file %s \n", pp1.npart, fn1);
      writef("Read in %i lines from file %s \n", pp2.npart, fn2);
    }
  }
  tt.stop();
  if !isTest {
    writef("Time to read : %r \n", tt.elapsed());
  }

  // Shuffle the particles -- this is not used here, but will be usef for the multilocale version
  tt.clear(); tt.start();
  pp1.shuffle();
  pp2.shuffle();
  tt.stop();
  if !isTest then writef("Time to shuffle : %r \n",tt.elapsed());

  // Build the tree
  tt.clear(); tt.start(); gtime1.clear();
  var root1 = BuildTree(pp1, 0, pp1.npart-1, 0);
  var root2 = BuildTree(pp2, 0, pp2.npart-1, 0);
  tt.stop();
  if !isTest then writef("Time to build trees : %r \n", tt.elapsed());
  //writef("Time in splitOn : %r \n", gtime1.elapsed());

  // Set up the histogram
  var hh = new UniformBins(2,(nsbins,nmubins), ((0.0,smax),(0.0,1.0+1.e-10)));

  // Do the paircounts with a tree
  hh.reset();
  tt.clear(); tt.start();
  sync TreeAccumulate(hh, pp1,pp2, root1, root2);
  tt.stop();
  if (!isTest) {
    writef("Time to tree paircount : %r \n", tt.elapsed());
    if !isPerf {
      var ff = openwriter("%s.tree".format(pairfn));
      writeHist(ff,hh);
      ff.close();
    }
  } else {
    hh.set((0,0),0.0);
    writeHist(stdout,hh,"%20.5er ");
  }

  //
  // clean up
  //
  delete pp1;
  delete pp2;
  delete root1;
  delete root2;
  delete hh;
}

use Histogram;
use Random;
use Time;

// Test flags
config const isTest=false;
config const isPerf=false;
config const doBrute=false;

// Performance test parameters
config const nParticles=10000;

// Input/Output filenames
config const fn1 = "test.dat";
config const fn2 = "test.dat";
config const pairfn = "test-DD.dat";

// Dimension constanst
param NDIM=3;
param NEXT=2; // w, r2
param NTOT=NDIM+NEXT;
const W=NDIM;
const R2=NDIM+1;
const ParticleAttrib = 0.. #NTOT;
const DimSpace = 0.. #NDIM;

// Tree parameters
config const minpart=500;

// Histogram parameters
config const smax=200.0;
config const smax2=smax**2;
config const nmubins=5;
config const nsbins=5;

// Global variables
var gtime1 : Timer;

proc main() {
  doPairs();
}

class Particle3D {
  var npart : int = 0;
  var Darr : domain(2);
  var Dpart : domain(1);
  var arr : [Darr] real;

  var _tmp : [Dpart] real;
  var _n1, _ndx : [Dpart] int;

  proc Particle3D(npart1 : int, random : bool = false) {
    npart = npart1;
    Darr = {ParticleAttrib, 0.. #npart};
    Dpart = {0.. #npart};
    if random {
      var rng = new RandomStream();
      var x, y, z : real;
      for ii in Dpart {
        x = rng.getNext()*1000.0; y = rng.getNext()*1000.0; z = rng.getNext()*1000.0;
        arr[0,ii] = x; arr[1, ii] = y; arr[2, ii] = z;
        arr[3,ii] = 1.0;
        arr[4,ii] = x**2 + y**2 + z**2;
      }
      delete rng;
    }
  }

  proc this(ii : int) : NTOT*real {
    // ??? Hardcoded hack!
    return (arr[0,ii],arr[1,ii],arr[2,ii],arr[3,ii],arr[4,ii]);
  }

  // Reorder all the arrays according to _ndx (we assume you have set this)
  proc reorder(idom : domain(1)) {

    for kk in ParticleAttrib {
      forall ii in idom {
        _tmp[_ndx[ii]] = arr[kk,ii];
      }
      forall ii in idom {
        arr[kk,ii] = _tmp[ii];
      }
    }
  }

  // Shuffles the elements of the array...
  // Note that the random number generator cannot cover the space
  // of all permutations, but that's fine for what we need.
  proc shuffle() {
    // Fill in _ndx
    forall ii in Dpart {
      _ndx[ii] = ii;
    }

    // Set the random number generator
    var rng = new RandomStream(41);
    var jj : int;
    for ii in 0..(npart-2) {
      jj = (rng.getNext()*(npart-ii)):int + ii;
      _ndx[jj] <=> _ndx[ii];
    }
    delete rng;

    reorder(Dpart);
  }

}

proc countLines(fn : string) : int {
  var ff = open(fn, iomode.r);
  var ipart = 0;
  for iff in ff.lines() do ipart +=1;
  ff.close();
  return ipart;
}

proc readFile(fn : string) : Particle3D  {
  var npart = countLines(fn);
  var pp = new Particle3D(npart);

  var ff = openreader(fn);
  var ipart = 0;
  var x,y,z,w,r2 : real;
  while (ff.read(x,y,z,w)) {
    r2 = x**2 + y**2 + z**2;
    pp.arr[..,ipart] = [x,y,z,w,r2];
    ipart += 1;
  }
  ff.close();

  return pp;
}

proc splitOn(pp : Particle3D, idom : domain(1), splitDim : int, xsplit : real) : int {
  // Setup for a prefix scan
  forall ii in idom {
    if (pp.arr[splitDim,ii] < xsplit) {
      pp._n1[ii] = 1;
    } else {
      pp._n1[ii] = 0;
    }
  }

  var lnpart : int = 0;
  for ii in idom do lnpart += pp._n1[ii];
  var li = idom.low;
  var ri = li+lnpart;
  for ii in idom {
    if (pp._n1[ii] == 1) {
      pp._ndx[ii] = li;
      li += 1;
    } else {
      pp._ndx[ii] = ri;
      ri += 1;
    }
  }

  pp.reorder(idom);

  return lnpart;
}

class KDNode {
  var lo, hi,npart,id : int;
  var dom : domain(1);
  var xcen : [DimSpace]real;
  var rcell : real;
  var left, right : KDNode;

  proc isLeaf() : bool {
    return (left==nil) && (right==nil);
  }

  proc ~KDNode() {
    if left then delete left;
    if right then delete right;
  }

}

proc BuildTree(pp : Particle3D, lo : int, hi : int, id : int) : KDNode  {
  var me : KDNode = new KDNode();
  me.lo = lo;
  me.hi = hi;
  me.dom = {lo..hi};
  me.id = id;
  me.npart = (me.hi-me.lo)+1;

  //  work out xcen and vantage point radius
  var pmin, pmax : [DimSpace] real;
  for idim in DimSpace {
    pmin[idim] = min reduce pp.arr[idim,lo..hi];
    pmax[idim] = max reduce pp.arr[idim,lo..hi];
  }
  me.xcen = (pmax+pmin)/2.0;
  me.rcell = 0.0;
  var r1 : real;
  for ii in lo..hi {
    r1 = 0.0;
    for idim in DimSpace {
      r1 += (pp.arr[idim,ii]-me.xcen[idim])**2;
    }
    r1 = sqrt(r1);
    if (r1 > me.rcell) then me.rcell = r1;
  }

  // Continue to split
  if (me.npart <= minpart) then return me; // Don't split further.

  // Find dimension to split on
  var dx : [DimSpace] real;
  dx = pmax - pmin;
  var splitDim = 0;
  for idim in DimSpace {
    if (dx[idim] > dx[splitDim]) then splitDim=idim;
  }

  // Split
  var lnpart = splitOn(pp,me.dom,splitDim, me.xcen[splitDim]);
  me.left  = BuildTree(pp, lo, lo+lnpart-1, 2*id+1);
  me.right = BuildTree(pp, lo+lnpart, hi,2*id+2);
  return me;
}

proc TreeAccumulate(hh : UniformBins, p1, p2 : Particle3D, node1, node2 : KDNode) {
  // Compute the distance between node1 and node2
  var rr = sqrt (+ reduce(node1.xcen - node2.xcen)**2);
  var rmin = rr - (node1.rcell+node2.rcell);

  // If distance is greater than all cases
  if (rmin > smax) then return;

  // If both nodes are leaves
  if (node1.isLeaf() & node2.isLeaf()) {
    begin smuAccumulate(hh, p1, p2, node1.dom, node2.dom,1);
    return;
  }

  // If one node is a leaf
  if (node1.isLeaf()) {
    TreeAccumulate(hh, p1, p2, node1, node2.left);
    TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }
  if (node2.isLeaf()) {
    TreeAccumulate(hh, p1, p2, node1.left, node2);
    TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  }

  // Split the larger case;
  if (node1.npart > node2.npart) {
    TreeAccumulate(hh, p1, p2, node1.left, node2);
    TreeAccumulate(hh, p1, p2, node1.right, node2);
    return;
  } else {
    TreeAccumulate(hh, p1, p2, node1, node2.left);
    TreeAccumulate(hh, p1, p2, node1, node2.right);
    return;
  }

}

// The basic pair counter
proc smuAccumulate(hh : UniformBins, p1,p2 : Particle3D, d1,d2 : domain(1), scale : real) {
  forall ii in d1 { // Loop over first set of particles

    var x1,y1,z1,w1,r2 : real;
    var sl, s2, l1, s1, l2, mu, wprod : real;
    (x1,y1,z1,w1,r2) = p1[ii];

    for jj in d2 { // Second set of particles
      mu=2*(p2.arr[0,jj]*x1 + p2.arr[1,jj]*y1 + p2.arr[2,jj]*z1);
      sl = r2 - p2.arr[R2,jj];
      l1 = r2 + p2.arr[R2,jj];
      s2 = l1 - mu;
      l2 = l1 + mu;
      if ((s2 >= smax2) || (s2 < 1.0e-20)) then continue;
      wprod = scale * w1 * p2.arr[W,jj];
      s1 = sqrt(s2);
      mu = sl/(s1*sqrt(l2));
      if (mu < 0) then mu = -mu;

      hh.add((s1,mu),wprod);
    }
  }
}

proc doPairs() {
  var tt : Timer;

  // Read in the file
  tt.clear(); tt.start();
  var pp1, pp2 : Particle3D;
  if isPerf {
    pp1 = new Particle3D(nParticles, true);
    pp2 = new Particle3D(nParticles, true);
  } else {
    pp1 = readFile(fn1);
    pp2 = readFile(fn2);
    writef("Read in %i lines from file %s \n", pp1.npart, fn1);
    writef("Read in %i lines from file %s \n", pp2.npart, fn2);
  }
  tt.stop();
  if !isTest {
    writef("Time to read : %r \n", tt.elapsed());
  }

  // Shuffle the particles -- this is not used here, but will be usef for the multilocale version
  tt.clear(); tt.start();
  pp1.shuffle();
  pp2.shuffle();
  tt.stop();
  if !isTest then writef("Time to shuffle : %r \n",tt.elapsed());

  // Build the tree
  tt.clear(); tt.start(); gtime1.clear();
  var root1 = BuildTree(pp1, 0, pp1.npart-1, 0);
  var root2 = BuildTree(pp2, 0, pp2.npart-1, 0);
  tt.stop();
  if !isTest then writef("Time to build trees : %r \n", tt.elapsed());
  //writef("Time in splitOn : %r \n", gtime1.elapsed());

  // Set up the histogram
  var hh = new UniformBins(2,(nsbins,nmubins), ((0.0,smax),(0.0,1.0+1.e-10)));

  // Do the paircounts with a tree
  hh.reset();
  tt.clear(); tt.start();
  sync TreeAccumulate(hh, pp1,pp2, root1, root2);
  tt.stop();
  if (!isTest) {
    writef("Time to tree paircount : %r \n", tt.elapsed());
    if !isPerf {
      var ff = openwriter("%s.tree".format(pairfn));
      writeHist(ff,hh);
      ff.close();
    }
  } else {
    hh.set((0,0),0.0);
    writeHist(stdout,hh,"%20.5er ");
  }

  //
  // clean up
  //
  delete pp1;
  delete pp2;
  delete root1;
  delete root2;
  delete hh;
}


Set the startup locale to Locales[0] and don't return the root locale in ID lookup.

git-svn-id: 5aabc979e906cfa1cd4a03d781c193b99fe6e46f@21401 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// DefaultArchitecture.chpl
//
// Provides a default architectural description.
// This architectural description is backward compatible with
// the architecture implicitly provided by releases 1.6 and preceding.
//
pragma "no use ChapelStandard"
module DefaultArchitecture {

  use ChapelLocale;
  use DefaultRectangular;
  use ChapelNumLocales;
  use RootLocale;
  use Sys;

  // We would really like a class-static storage class.(C++ nomenclature)
  var doneCreatingLocales: bool = false;

  //
  // A concrete class representing the nodes in this architecture.
  //
  class DefaultNode : locale {
    const callStackSize: int;
    const _node_id : int;
    const local_name : string;

    // This constructor must be invoked "on" the node
    // that it is intended to represent.  This trick is used
    // to establish the equivalence the "locale" field of the locale object
    // and the node ID portion of any wide pointer referring to it.
    proc DefaultNode() {
      if doneCreatingLocales {
        halt("Cannot create additional DefaultNode instances");
      }
      init;
    }

    proc DefaultNode(parent_loc : locale) {
      parent = parent_loc;
      init;
    }

    // chpl_nodeID is the node ID associated with the running image.
    proc chpl_id() return _node_id;
    proc chpl_name() return local_name;

    proc readWriteThis(f) {
      // Most classes will define it like this:
//      f <~> name;
      // but here it is defined thus for backward compatibility.
      f <~> new ioLiteral("LOCALE") <~> _node_id;
    }

    proc getChildSpace() return emptyLocaleSpace;

    proc getChildCount() return 0;

    iter getChildIndices() : int {
      for idx in emptyLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) : locale {
      // This is a temporary implementation, where an index of zero is forced
      // to mean "here".  A better solution is on the way.
      // TODO: Run index lookup through rootLocale.findLocale(lid:chpl_localeID_t);
      if idx == 0 then return this;
      else
        if boundsChecking then
          halt("requesting a child from a DefaultNode locale");
      return nil;
    }

    iter getChldren() : locale  {
      for loc in emptyLocales do
        yield loc;
    }

    proc getChildArray() {
      return emptyLocales;
    }

    // Part of the public interface required by the compiler
    // These are dynamically dispatched, so cannot be inlined.
    proc taskInit() {}
    proc taskExit() {}

    //------------------------------------------------------------------------{
    //- Implementation (private)
    //-
    proc init {
      _node_id = __primitive("cast", int(64), __primitive("chpl_nodeID"));

      // chpl_nodeName is defined in chplsys.c.
      // It supplies a node name obtained by running uname(3) on the current node.
      // For this reason (as well), the constructor (or at least this init method)
      // must be run on the node it is intended to describe.
//      extern proc getenv(s:string) : string;
      var comm, spawnfn : string;
      extern proc chpl_nodeName() : string;
      // sys_getenv returns zero on success.
      if sys_getenv("CHPL_COMM", comm) == 0 && comm == "gasnet" &&
        sys_getenv("GASNET_SPAWNFN", spawnfn) == 0 && spawnfn == "L"
      then local_name = chpl_nodeName() + "-" + _node_id : string;
      else local_name = chpl_nodeName();

      extern proc chpl_task_getCallStackSize(): int;
      callStackSize = chpl_task_getCallStackSize();

      extern proc chpl_numCoresOnThisLocale(): int;
      numCores = chpl_numCoresOnThisLocale();
    }
    //------------------------------------------------------------------------}
  }

  //
  // An instance of this class is the default contents 'rootLocale'.
  //
  // In the current implementation a platform-specific architectural description
  // may overwrite this instance or any of its children to establish a more customized
  // representation of the system resources.
  //
  class DefaultRootLocale : RootLocale {

    // Would like to make myLocaleSpace distributed with one index per node.
    const myLocaleSpace: domain(1) = {0..numLocales-1};
    const myLocales: [myLocaleSpace] locale;

    proc DefaultRootLocale()
    {
      parent = nil;
      numCores = 0;

      // We cannot use a forall here because the default leader iterator will
      // access 'Locales' and 'here', which are not yet initialized.
      for locIdx in myLocaleSpace
      {
        var locID : chpl_localeID_t;
        locID.node = locIdx : chpl_nodeID_t;
        on __primitive("chpl_on_locale_num", locID)
        {
          // chpl_on_locale_num sets the localeID portion of "here", but
          // leaves the addr portion as a NULL pointer.
          const node = new DefaultNode(this);

          // So immediately after creating a new locale to represent here, we
          // have to call this primitive to insert it into task-private storage.
          __primitive("_task_set_here", node);

          myLocales[locIdx] = node;
          numCores += node.numCores;
        }
      }
      doneCreatingLocales = true;

      // Sanity check.
      // Ensure that the locale representing node x has a nodeID of x.
      // At the top level, this lets us look up the locale representing the node on which
      // an object o lives by executing
      //  myLocales[__primitive("_wide_get_node", o)].
      for locIdx in myLocaleSpace {
        var loc = myLocales[locIdx];
        if __primitive("_wide_get_node", loc) != locIdx then
          halt("In this architecture, we expect the locale whose index is x to live on node x.");
      }

      // Programs traditionally expect the startup process to run on Locales[0], so
      // this is how we mimic that behavior.
      // Note that this means we have to ask for here.parent or rootLocale to get
      // the root locale of the default architecture.
      var loc = myLocales[0];
      __primitive("_task_set_here", loc);
      var locID : chpl_localeID_t; // Init'd to {0,0}.
      __primitive("_task_set_locale", locID);
    }

    // Has to be globally unique and not equal to a node ID.
    // We return numLocales for now, since we expect nodes to be numbered less than this.
    // -1 is used in the abstract locale class to specify an invalid node ID.
    proc chpl_id() return numLocales;
    proc chpl_name() return local_name();
    proc local_name() return "rootLocale";

    proc readWriteThis(f) {
      f <~> name;
    }

    proc getChildCount() return this.myLocaleSpace.numIndices;

    proc getChildSpace() return this.myLocaleSpace;

    iter getChildIndices() : int {
      for idx in this.myLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) return this.myLocales[idx];

    iter getChldren() : locale  {
      for loc in this.myLocales do
        yield loc;
    }

    proc getDefaultLocaleSpace() return this.myLocaleSpace;
    proc getDefaultLocaleArray() return myLocales;

    proc localeIDtoLocale(id : chpl_localeID_t) {
      // In the default architecture, there are only nodes and no sublocales.
      // What is more, the nodeID portion of a wide pointer is the same as
      // the index into myLocales that yields the locale representing that node.
      return myLocales[id.node];
    }
  }

  if (rootLocale == nil) then
    rootLocale = new DefaultRootLocale();

  // Expose the underlying locales array (and its domain)
  // for user convenience and backward compatibility.
  // The downcase is because we cannot move the domain and array return types
  // into the base (abstract) locale class.
  // That would make locales depend on arrays which depend on locales....
  // If we had a way to express domain and array return types abstractly,
  // that problem would go away. <hilde>
  const Locales => (rootLocale:RootLocale).getDefaultLocaleArray();
  const LocaleSpace = Locales.domain;
}

// DefaultArchitecture.chpl
//
// Provides a default architectural description.
// This architectural description is backward compatible with
// the architecture implicitly provided by releases 1.6 and preceding.
//
pragma "no use ChapelStandard"
module DefaultArchitecture {

  use ChapelLocale;
  use DefaultRectangular;
  use ChapelNumLocales;
  use RootLocale;
  use Sys;

  // We would really like a class-static storage class.(C++ nomenclature)
  var doneCreatingLocales: bool = false;

  //
  // A concrete class representing the nodes in this architecture.
  //
  class DefaultNode : locale {
    const callStackSize: int;
    const _node_id : int;
    const local_name : string;

    // This constructor must be invoked "on" the node
    // that it is intended to represent.  This trick is used
    // to establish the equivalence the "locale" field of the locale object
    // and the node ID portion of any wide pointer referring to it.
    proc DefaultNode() {
      if doneCreatingLocales {
        halt("Cannot create additional DefaultNode instances");
      }
      init;
    }

    proc DefaultNode(parent_loc : locale) {
      parent = parent_loc;
      init;
    }

    // chpl_nodeID is the node ID associated with the running image.
    proc chpl_id() return _node_id;
    proc chpl_name() return local_name;

    proc readWriteThis(f) {
      // Most classes will define it like this:
//      f <~> name;
      // but here it is defined thus for backward compatibility.
      f <~> new ioLiteral("LOCALE") <~> _node_id;
    }

    proc getChildSpace() return emptyLocaleSpace;

    proc getChildCount() return 0;

    iter getChildIndices() : int {
      for idx in emptyLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) : locale {
      // This is a temporary implementation, where an index of zero is forced
      // to mean "here".  A better solution is on the way.
      // TODO: Run index lookup through rootLocale.findLocale(lid:chpl_localeID_t);
      if idx == 0 then return this;
      else
        if boundsChecking then
          halt("requesting a child from a DefaultNode locale");
      return nil;
    }

    iter getChldren() : locale  {
      for loc in emptyLocales do
        yield loc;
    }

    proc getChildArray() {
      return emptyLocales;
    }

    // Part of the public interface required by the compiler
    // These are dynamically dispatched, so cannot be inlined.
    proc taskInit() {}
    proc taskExit() {}

    //------------------------------------------------------------------------{
    //- Implementation (private)
    //-
    proc init {
      _node_id = __primitive("cast", int(64), __primitive("chpl_nodeID"));

      // chpl_nodeName is defined in chplsys.c.
      // It supplies a node name obtained by running uname(3) on the current node.
      // For this reason (as well), the constructor (or at least this init method)
      // must be run on the node it is intended to describe.
//      extern proc getenv(s:string) : string;
      var comm, spawnfn : string;
      extern proc chpl_nodeName() : string;
      // sys_getenv returns zero on success.
      if sys_getenv("CHPL_COMM", comm) == 0 && comm == "gasnet" &&
        sys_getenv("GASNET_SPAWNFN", spawnfn) == 0 && spawnfn == "L"
      then local_name = chpl_nodeName() + "-" + _node_id : string;
      else local_name = chpl_nodeName();

      extern proc chpl_task_getCallStackSize(): int;
      callStackSize = chpl_task_getCallStackSize();

      extern proc chpl_numCoresOnThisLocale(): int;
      numCores = chpl_numCoresOnThisLocale();
    }
    //------------------------------------------------------------------------}
  }

  //
  // An instance of this class is the default contents 'rootLocale'.
  //
  // In the current implementation a platform-specific architectural description
  // may overwrite this instance or any of its children to establish a more customized
  // representation of the system resources.
  //
  class DefaultRootLocale : RootLocale {

    // Would like to make myLocaleSpace distributed with one index per node.
    const myLocaleSpace: domain(1) = {0..numLocales-1};
    const myLocales: [myLocaleSpace] locale;

    proc DefaultRootLocale()
    {
      parent = nil;
      numCores = 0;

      // We cannot use a forall here because the default leader iterator will
      // access 'Locales' and 'here', which are not yet initialized.
      for locIdx in myLocaleSpace
      {
        var locID : chpl_localeID_t;
        locID.node = locIdx : chpl_nodeID_t;
        on __primitive("chpl_on_locale_num", locID)
        {
          // chpl_on_locale_num sets the localeID portion of "here", but
          // leaves the addr portion as a NULL pointer.
          const node = new DefaultNode(this);

          // So immediately after creating a new locale to represent here, we
          // have to call this primitive to insert it into task-private storage.
          __primitive("_task_set_here", node);

          myLocales[locIdx] = node;
          numCores += node.numCores;
        }
      }
      doneCreatingLocales = true;

      // Sanity check.
      // Ensure that the locale representing node x has a nodeID of x.
      // At the top level, this lets us look up the locale representing the node on which
      // an object o lives by executing
      //  myLocales[__primitive("_wide_get_node", o)].
      for locIdx in myLocaleSpace {
        var loc = myLocales[locIdx];
        if __primitive("_wide_get_node", loc) != locIdx then
          halt("In this architecture, we expect the locale whose index is x to live on node x.");
      }
    }

    // Has to be globally unique and not equal to a node ID.
    // We return numLocales for now, since we expect nodes to be numbered less than this.
    // -1 is used in the abstract locale class to specify an invalid node ID.
    proc chpl_id() return numLocales;
    proc chpl_name() return local_name();
    proc local_name() return "rootLocale";

    proc readWriteThis(f) {
      f <~> name;
    }

    proc getChildCount() return this.myLocaleSpace.numIndices;

    proc getChildSpace() return this.myLocaleSpace;

    iter getChildIndices() : int {
      for idx in this.myLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) return this.myLocales[idx];

    iter getChldren() : locale  {
      for loc in this.myLocales do
        yield loc;
    }

    proc getDefaultLocaleSpace() return this.myLocaleSpace;
    proc getDefaultLocaleArray() return myLocales;

    proc localeIDtoLocale(id : chpl_localeID_t) {
      // In the default architecture, there are only nodes and no sublocales.
      // What is more, the nodeID portion of a wide pointer is the same as the index into
      // myLocales that yields the locale representing that node.
      if id.node == __primitive("chpl_nodeID") then
        return here;
      else
        return myLocales[id.node];
    }
  }

  if (rootLocale == nil) then
    rootLocale = new DefaultRootLocale();

  // Expose the underlying locales array (and its domain)
  // for user convenience and backward compatibility.
  // The downcase is because we cannot move the domain and array return types
  // into the base (abstract) locale class.
  // That would make locales depend on arrays which depend on locales....
  // If we had a way to express domain and array return types abstractly,
  // that problem would go away. <hilde>
  const Locales => (rootLocale:RootLocale).getDefaultLocaleArray();
  const LocaleSpace = Locales.domain;
}


Added a reference to the chpldoc README.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@23118 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
  chpldoc Primer

  This primer covers the use of chpldoc to document source code. For further
  information, please see $CHPL_HOME/docs/technotes/README.chpldoc

*/

// Creating and accessing the documentation:

// To generate the documentation for this and any source file, you may either
// compile using the command 'chpldoc <sourceName.chpl>' for documentation only
// or using the command 'chpl --docs <sourceName.chpl>' to create both
// documentation and an executable.

// The documentation files will be found in a special folder created by the
// chpldoc tool.  If you are compiling a Chapel file that is deeper than your
// current directory, this heirarchy will be reflected within the documentation
// folder.
// (i.e. foo/bar.chpl will generate documentation within docs/foo/)

// Documentation within the code:

/*
  Multiline comments found before a method are associated with that method
  as long as there are no code blocks between them.
*/

proc commented(val: int): string {
  var totalNum = val + 4;
  var str = "hello agent " + totalNum;
  return str;
}

/*
  The method can be a stub and still output its comment
*/
proc stub(val): bool {
  /* Comments within the method body are ignored */
}

// Single line comments are also ignored.  However, this does not prevent
// the display of the method itself.
proc uncommented() {
  /*
    Longer comments within the method body are also ignored
  */
}

/*
  If the source code defines a module, it can also have a comment associated
  with it
*/
module Defined {

  /*
    And classes can display their comments, too
  */
  class Foo {
    /*
      Including the comments associated with their fields
    */
    var a: int;

    /*
      And class specific methods
    */
    proc getA(): int {
      return a;
    }
  }

  /*
    If a class inherits from another class, those fields, methods, and their
    comments are output in both the super and the subclass
  */
  class Bar : Foo {

    proc setA(newVal: int) {
      a = newVal;
    }
  }
}

/*
  chpldoc Primer

  This primer covers the use of chpldoc to document source code

*/

// Creating and accessing the documentation:

// To generate the documentation for this and any source file, you may either
// compile using the command 'chpldoc <sourceName.chpl>' for documentation only
// or using the command 'chpl --docs <sourceName.chpl>' to create both
// documentation and an executable.

// The documentation files will be found in a special folder created by the
// chpldoc tool.  If you are compiling a Chapel file that is deeper than your
// current directory, this heirarchy will be reflected within the documentation
// folder.
// (i.e. foo/bar.chpl will generate documentation within docs/foo/)

// Documentation within the code:

/*
  Multiline comments found before a method are associated with that method
  as long as there are no code blocks between them.
*/

proc commented(val: int): string {
  var totalNum = val + 4;
  var str = "hello agent " + totalNum;
  return str;
}

/*
  The method can be a stub and still output its comment
*/
proc stub(val): bool {
  /* Comments within the method body are ignored */
}

// Single line comments are also ignored.  However, this does not prevent
// the display of the method itself.
proc uncommented() {
  /*
    Longer comments within the method body are also ignored
  */
}

/*
  If the source code defines a module, it can also have a comment associated
  with it
*/
module Defined {

  /*
    And classes can display their comments, too
  */
  class Foo {
    /*
      Including the comments associated with their fields
    */
    var a: int;

    /*
      And class specific methods
    */
    proc getA(): int {
      return a;
    }
  }

  /*
    If a class inherits from another class, those fields, methods, and their
    comments are output in both the super and the subclass
  */
  class Bar : Foo {

    proc setA(newVal: int) {
      a = newVal;
    }
  }
}


Adding a comment about domain literals, since we don't seem to talk
about them directly

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20604 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
 * Domains primer
 *
 * This primer showcases Chapel domains.  For other uses of domains
 * see the Arrays primer (arrays.chpl) and the Sparse arrays primer
 * (sparse.chpl).
 *
 */

config var n = 10;
//
// A domain is a first-class representation of an index set used to
// specify iteration spaces, define arrays, and aggregate operations
// such as slicing.
//

//
// Chapel provides two base domain types: rectangular and associative.
//
// Rectangular domains are used to represent rectangular index sets.
// Each dimesion of a rectangular domain is specified by a range and
// thus can take on the shape of any range.  See the Ranges primer
// (ranges.chpl) for more information.
//
// Rectangular domains support a literal syntax that is a
// comma-separated list of range expressions enclosed in curly braces.
//
// RD is an n by n by n domain.
//
var RD: domain(3) = {1..n, 1..n, 1..n};
writeln(RD);

//
// Rectangular domains have a set of methods that enable convenient
// reuse of existing domains.
//
// The expand method returns a new domain that is expanded or
// contracted depending on the sign of the offset argument.
//
var RDbigger = RD.expand((1,1,1));
writeln(RDbigger);
var RDsmaller = RD.expand((-1,-1,-1));
writeln(RDsmaller);

//
// The exterior method returns a new domain that is the exterior
// portion of the current domain.  A positive offset specifies that
// the exterior should be taken from the high bound; a negative
// offset, the low bound.
//
var RDext_p = RD.exterior((1,1,1));
writeln(RDext_p);
var RDext_n = RD.exterior((-1,-1,-1));
writeln(RDext_n);

//
// The interior method returns a new domain that is the interior
// portion of the current domain.  The sign of the offset implies
// using the high or low bound as in the exterior case.
//
var RDint_p = RD.interior((1,1,1));
writeln(RDint_p);
var RDint_n = RD.interior((-1,-1,-1));
writeln(RDint_n);

//
// The translate method returns a new domain that is the current domain
// translated by the offset.
//
var RDtrans_p = RD.translate((1,1,1));
writeln(RDtrans_p);
var RDtrans_n = RD.translate((-1,-1,-1));
writeln(RDtrans_n);

//
// Associative domains represent an arbitrary index set of any type.
//
// AD is an associative array indexed by strings.  Associative domains
// start out empty.
//
var AD: domain(string);
writeln(AD);

//
// The '+' operator is used to add indices to an associative domain.
//
AD += "John";
AD += "Paul";
AD += "Stuart";
AD += "George";
writeln(AD);

//
// The '-' operator is used to remove indices to an associative domain.
//
AD -= "Stuart";
AD += "Ringo";
writeln(AD);

//
// A subdomain is a domain that is declared in terms of a parent
// domain, causing it to have the same type as their parent.  A
// subdomain represents a subset of its parent domain's index set,
// though this constraint is not currently enforced by the
// implementation.

//
// Create rectangular subdomains.
//
var RSD1, RSD2 : subdomain(RD);

// A subdomain is initially empty.
writeln("RSD1:", RSD1);
writeln("RSD2:", RSD2);

// We can select parts of the original rectangular domain using slicing.
RSD1 = RD[..n/2, .., ..]; // This gives half of the domain
RSD2 = RD[n/2+1.., .., ..]; // And this the other half.

writeln("RSD1:", RSD1);
writeln("RSD2:", RSD2);

// Notes:
// - Subdomains of rectangular domains are regular unless they are explicitly
// declared to be sparse.
// - At present, range checking to ensure that a subdomain fits within its parent
// domain is unimplemented.

//
// Create a sparse subdomain of a regular domain.
//

var SSD: sparse subdomain(RD);

writeln("SSD:", SSD); // Initially empty.

// Add some indices to the sparse subdomain.

SSD += (1,2,3);
SSD += (4,5,6);
SSD += (7,8,9);
SSD += (9,10,1);

writeln("SSD:", SSD); // Now contains an unordered set of indices.

// Note: Checks to ensure that sparse subdomain indices lie within the parent domain
// have not been implemented.

//
// Create an associative subdomain.
//

var ASD : subdomain(AD);

// The subdomain is initially empty.

writeln("ASD:", ASD);

ASD += "Stuart";

writeln("ASD:", ASD);

// Note: Checks to ensure that ASD is a subset of AD have not been implemented.

/*
 * For more information on domains, see the Domains chapter of the
 * Chapel Language Spec.
 */

/*
 * Domains primer
 *
 * This primer showcases Chapel domains.  For other uses of domains
 * see the Arrays primer (arrays.chpl) and the Sparse arrays primer
 * (sparse.chpl).
 *
 */

config var n = 10;
//
// A domain is a first-class representation of an index set used to
// specify iteration spaces, define arrays, and aggregate operations
// such as slicing.
//

//
// Chapel provides two base domain types: rectangular and associative.
//
// Rectangular domains are used to represent rectangular index sets.
// Each dimesion of a rectangular domain is specified by a range and
// thus can take on the shape of any range.  See the Ranges primer
// (ranges.chpl) for more information.
//
// RD is an n by n by n domain.
//
var RD: domain(3) = {1..n, 1..n, 1..n};
writeln(RD);

//
// Rectangular domains have a set of methods that enable convenient
// reuse of existing domains.
//
// The expand method returns a new domain that is expanded or
// contracted depending on the sign of the offset argument.
//
var RDbigger = RD.expand((1,1,1));
writeln(RDbigger);
var RDsmaller = RD.expand((-1,-1,-1));
writeln(RDsmaller);

//
// The exterior method returns a new domain that is the exterior
// portion of the current domain.  A positive offset specifies that
// the exterior should be taken from the high bound; a negative
// offset, the low bound.
//
var RDext_p = RD.exterior((1,1,1));
writeln(RDext_p);
var RDext_n = RD.exterior((-1,-1,-1));
writeln(RDext_n);

//
// The interior method returns a new domain that is the interior
// portion of the current domain.  The sign of the offset implies
// using the high or low bound as in the exterior case.
//
var RDint_p = RD.interior((1,1,1));
writeln(RDint_p);
var RDint_n = RD.interior((-1,-1,-1));
writeln(RDint_n);

//
// The translate method returns a new domain that is the current domain
// translated by the offset.
//
var RDtrans_p = RD.translate((1,1,1));
writeln(RDtrans_p);
var RDtrans_n = RD.translate((-1,-1,-1));
writeln(RDtrans_n);

//
// Associative domains represent an arbitrary index set of any type.
//
// AD is an associative array indexed by strings.  Associative domains
// start out empty.
//
var AD: domain(string);
writeln(AD);

//
// The '+' operator is used to add indices to an associative domain.
//
AD += "John";
AD += "Paul";
AD += "Stuart";
AD += "George";
writeln(AD);

//
// The '-' operator is used to remove indices to an associative domain.
//
AD -= "Stuart";
AD += "Ringo";
writeln(AD);

//
// A subdomain is a domain that is declared in terms of a parent
// domain, causing it to have the same type as their parent.  A
// subdomain represents a subset of its parent domain's index set,
// though this constraint is not currently enforced by the
// implementation.

//
// Create rectangular subdomains.
//
var RSD1, RSD2 : subdomain(RD);

// A subdomain is initially empty.
writeln("RSD1:", RSD1);
writeln("RSD2:", RSD2);

// We can select parts of the original rectangular domain using slicing.
RSD1 = RD[..n/2, .., ..]; // This gives half of the domain
RSD2 = RD[n/2+1.., .., ..]; // And this the other half.

writeln("RSD1:", RSD1);
writeln("RSD2:", RSD2);

// Notes:
// - Subdomains of rectangular domains are regular unless they are explicitly
// declared to be sparse.
// - At present, range checking to ensure that a subdomain fits within its parent
// domain is unimplemented.

//
// Create a sparse subdomain of a regular domain.
//

var SSD: sparse subdomain(RD);

writeln("SSD:", SSD); // Initially empty.

// Add some indices to the sparse subdomain.

SSD += (1,2,3);
SSD += (4,5,6);
SSD += (7,8,9);
SSD += (9,10,1);

writeln("SSD:", SSD); // Now contains an unordered set of indices.

// Note: Checks to ensure that sparse subdomain indices lie within the parent domain
// have not been implemented.

//
// Create an associative subdomain.
//

var ASD : subdomain(AD);

// The subdomain is initially empty.

writeln("ASD:", ASD);

ASD += "Stuart";

writeln("ASD:", ASD);

// Note: Checks to ensure that ASD is a subset of AD have not been implemented.

/*
 * For more information on domains, see the Domains chapter of the
 * Chapel Language Spec.
 */


Fixed GB/s computation in stream-ep.

Fixed GB/s computation in stream-ep.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@16400 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Embarassingly Parallel Implementation of STREAM Triad
//
// This version of the stream benchmark is not as elegant as
// stream.chpl.  It is a per-locale code with no communication in the
// actual computation.  It highlights the ability of a Chapel
// programmer to escape the global-view programming model and write
// codes with a fragmented, per-locale model.
//
// Comments marked with '***' point out differences with the global
// version of this benchmark in stream.chpl.
//

//
// *** Since this benchmark is written on a per-locale basis, there is no
// *** need to use the Block distribution.  The following use omits
// *** BlockDist, included in stream.chpl.
//
// Use standard modules for Timing routines, Type utility functions,
// and Random numbers
//
use Time, Types, Random;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of vectors and element type of those vectors
//
const numVectors = 3;
type elemType = real(64);

//
// *** To ensure a local problem size, we spoof the number of vectors
// *** passed to the computeProblemSize function to be the number of
// *** vectors times the number of locales.
//
// Configuration constants to set the problem size (m) and the scalar
// multiplier, alpha
//
config const m = computeProblemSize(numVectors*numLocales, elemType),
             alpha = 3.0;

//
// Configuration constants to set the number of trials to run and the
// amount of error to permit in the verification
//
config const numTrials = 10,
             epsilon = 0.0;

//
// *** There isn't (yet) a way to set the number of tasks to use for
// *** implementing a forall loop over a default array.  When there is
// *** such a way, we will want to set it via this configuration
// *** constant to get functionality like we have with stream.chpl.
//
// config const tasksPerLocale = here.numCores;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

//
// *** To ensure determinism of output, there is no more printing of the
// *** arrays in initVectors and verifyResults.
//
//
// Configuration constants to control what's printed -- benchmark
// parameters and/or statistics
//
config const printParams = true,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // *** Aggregates for collecting per-locale results for the minimum
  // *** execution time per trial, and whether verification passed
  //
  var minTimes: [LocaleSpace] real;
  var validAnswers: [LocaleSpace] bool;

  //
  // *** Fragment control so that we have a single task running on
  // *** every locale.
  //
  coforall loc in Locales do on loc {

    //
    // *** We declare these variables outside of the local block since
    // *** we'll need to access them when we write back to the global
    // *** aggregates declared above.
    //
    var validAnswer: bool;
    var execTime: [1..numTrials] real;

    //
    // *** Indicates that all of the code in this block is local to
    // *** this locale.  There is no communication.  A violation will
    // *** result in an error, though error checking is disabled with
    // *** --fast or --no-checks.
    //
    local {

      //
      // *** A, B, and C are the three local vectors
      //
      var A, B, C: [1..m] elemType;

      initVectors(B, C);    // Initialize the input vectors, B and C

      for trial in 1..numTrials {                        // loop over the trials
        const startTime = getCurrentTime();              // capture the start time

        //
        // *** The main loop looks identical to stream.chpl.  However,
        // *** in this version we are iterating over arrays that are
        // *** not distributed.
        //
        forall (a, b, c) in (A, B, C) do
          a = b + alpha * c;

        execTime(trial) = getCurrentTime() - startTime;  // store the elapsed time
      }

      validAnswer = verifyResults(A, B, C);              // verify...
    }

    //
    // *** Write times and verification result into aggregates
    // *** declared above.  These are declared over LocaleSpace so we
    // *** can write to them in parallel.
    //
    minTimes[here.id] = min reduce execTime;
    validAnswers[here.id] = validAnswer;
  }

  //
  // *** Pass minimum, average, and maximum times to printResults
  //
  printResults(&& reduce validAnswers, minTimes);
}

//
// Print the problem size and number of trials
//
def printConfiguration() {
  if (printParams) {
    //
    // *** Here we multiply m by the number of locales so that we can
    // *** print out the global problem size.
    //
    printProblemSize(elemType, numVectors, m * numLocales);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

//
// *** Both initVectors and verifyResults are almost identical to
// *** stream.chpl even though they are called with arrays that are
// *** not distributed.  For initialization, the same random stream is
// *** used on each locale.  In the global version, a single logical
// *** stream of random numbers is used across all of the locales.
// ***
// *** In this version, we've omitted a way to print the arrays.  This
// *** ensures determinism of output.  Printing the arrays also
// *** violates the locality constraint imposed by the local block
// *** from which these functions are called.
//
// Initialize vectors B and C using a random stream of values
//
def initVectors(B, C) {
  var randlist = new RandomStream(seed);

  randlist.fillRandom(B);
  randlist.fillRandom(C);

  delete randlist;
}

//
// Verify that the computation is correct
//
def verifyResults(A, B, C) {

  //
  // recompute the computation, destructively storing into B to save space
  //
  forall (b, c) in (B, C) do
    b += alpha *c;

  //
  // Compute the infinity-norm by computing the maximum reduction of the
  // absolute value of A's elements minus the new result computed in B.
  // "[i in I]" represents an expression-level loop: "forall i in I"
  //
  const infNorm = max reduce [(a,b) in (A,B)] abs(a - b);

  return (infNorm <= epsilon);    // return whether the error is acceptable
}

//
// Print out success/failure, the timings, and the GB/s value.
//
// *** Here we report maximum, average, and minimum times instead of
// *** total, average, and minimum.
//
def printResults(successful, minTimes) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    const maxTime = max reduce minTimes,
          avgTime = + reduce minTimes / numLocales,
          minTime = min reduce minTimes;
    writeln("Execution time:");
    writeln("  max = ", maxTime);
    writeln("  avg = ", avgTime);
    writeln("  min = ", minTime);

    const GBPerSec = numVectors * numBytes(elemType) * (numLocales * m / minTime) * 1e-9;
    writeln("Performance (GB/s) = ", GBPerSec);
  }
}

//
// Embarassingly Parallel Implementation of STREAM Triad
//
// This version of the stream benchmark is not as elegant as
// stream.chpl.  It is a per-locale code with no communication in the
// actual computation.  It highlights the ability of a Chapel
// programmer to escape the global-view programming model and write
// codes with a fragmented, per-locale model.
//
// Comments marked with '***' point out differences with the global
// version of this benchmark in stream.chpl.
//

//
// *** Since this benchmark is written on a per-locale basis, there is no
// *** need to use the Block distribution.  The following use omits
// *** BlockDist, included in stream.chpl.
//
// Use standard modules for Timing routines, Type utility functions,
// and Random numbers
//
use Time, Types, Random;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of vectors and element type of those vectors
//
const numVectors = 3;
type elemType = real(64);

//
// *** To ensure a local problem size, we spoof the number of vectors
// *** passed to the computeProblemSize function to be the number of
// *** vectors times the number of locales.
//
// Configuration constants to set the problem size (m) and the scalar
// multiplier, alpha
//
config const m = computeProblemSize(numVectors*numLocales, elemType),
             alpha = 3.0;

//
// Configuration constants to set the number of trials to run and the
// amount of error to permit in the verification
//
config const numTrials = 10,
             epsilon = 0.0;

//
// *** There isn't (yet) a way to set the number of tasks to use for
// *** implementing a forall loop over a default array.  When there is
// *** such a way, we will want to set it via this configuration
// *** constant to get functionality like we have with stream.chpl.
//
// config const tasksPerLocale = here.numCores;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

//
// *** To ensure determinism of output, there is no more printing of the
// *** arrays in initVectors and verifyResults.
//
//
// Configuration constants to control what's printed -- benchmark
// parameters and/or statistics
//
config const printParams = true,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // *** Aggregates for collecting per-locale results for the minimum
  // *** execution time per trial, and whether verification passed
  //
  var minTimes: [LocaleSpace] real;
  var validAnswers: [LocaleSpace] bool;

  //
  // *** Fragment control so that we have a single task running on
  // *** every locale.
  //
  coforall loc in Locales do on loc {

    //
    // *** We declare these variables outside of the local block since
    // *** we'll need to access them when we write back to the global
    // *** aggregates declared above.
    //
    var validAnswer: bool;
    var execTime: [1..numTrials] real;

    //
    // *** Indicates that all of the code in this block is local to
    // *** this locale.  There is no communication.  A violation will
    // *** result in an error, though error checking is disabled with
    // *** --fast or --no-checks.
    //
    local {

      //
      // *** A, B, and C are the three local vectors
      //
      var A, B, C: [1..m] elemType;

      initVectors(B, C);    // Initialize the input vectors, B and C

      for trial in 1..numTrials {                        // loop over the trials
        const startTime = getCurrentTime();              // capture the start time

        //
        // *** The main loop looks identical to stream.chpl.  However,
        // *** in this version we are iterating over arrays that are
        // *** not distributed.
        //
        forall (a, b, c) in (A, B, C) do
          a = b + alpha * c;

        execTime(trial) = getCurrentTime() - startTime;  // store the elapsed time
      }

      validAnswer = verifyResults(A, B, C);              // verify...
    }

    //
    // *** Write times and verification result into aggregates
    // *** declared above.  These are declared over LocaleSpace so we
    // *** can write to them in parallel.
    //
    minTimes[here.id] = min reduce execTime;
    validAnswers[here.id] = validAnswer;
  }

  //
  // *** Pass minimum, average, and maximum times to printResults
  //
  printResults(&& reduce validAnswers, minTimes);
}

//
// Print the problem size and number of trials
//
def printConfiguration() {
  if (printParams) {
    //
    // *** Here we multiply m by the number of locales so that we can
    // *** print out the global problem size.
    //
    printProblemSize(elemType, numVectors, m * numLocales);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

//
// *** Both initVectors and verifyResults are almost identical to
// *** stream.chpl even though they are called with arrays that are
// *** not distributed.  For initialization, the same random stream is
// *** used on each locale.  In the global version, a single logical
// *** stream of random numbers is used across all of the locales.
// ***
// *** In this version, we've omitted a way to print the arrays.  This
// *** ensures determinism of output.  Printing the arrays also
// *** violates the locality constraint imposed by the local block
// *** from which these functions are called.
//
// Initialize vectors B and C using a random stream of values
//
def initVectors(B, C) {
  var randlist = new RandomStream(seed);

  randlist.fillRandom(B);
  randlist.fillRandom(C);

  delete randlist;
}

//
// Verify that the computation is correct
//
def verifyResults(A, B, C) {

  //
  // recompute the computation, destructively storing into B to save space
  //
  forall (b, c) in (B, C) do
    b += alpha *c;

  //
  // Compute the infinity-norm by computing the maximum reduction of the
  // absolute value of A's elements minus the new result computed in B.
  // "[i in I]" represents an expression-level loop: "forall i in I"
  //
  const infNorm = max reduce [(a,b) in (A,B)] abs(a - b);

  return (infNorm <= epsilon);    // return whether the error is acceptable
}

//
// Print out success/failure, the timings, and the GB/s value.
//
// *** Here we report maximum, average, and minimum times instead of
// *** total, average, and minimum.
//
def printResults(successful, minTimes) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    const maxTime = max reduce minTimes,
          avgTime = + reduce minTimes / numLocales,
          minTime = min reduce minTimes;
    writeln("Execution time:");
    writeln("  max = ", maxTime);
    writeln("  avg = ", avgTime);
    writeln("  min = ", minTime);

    const GBPerSec = numVectors * numBytes(elemType) * (m / minTime) * 1e-9;
    writeln("Performance (GB/s) = ", GBPerSec);
  }
}