Fix the multilocale distribution regressions for distString=block.

When rank change slicing a Block domain's bounding box, use the collapsed
dimension values plus the newRank sized index into targetLocDom to create the
index into the bigger targetLocDom.

When creating an index into the bounding box out of slice arguments, use the
bounding box low or high if the argument is out of range of the bounding box.

Added some comments to help explain what the code in the block rank change
functions is doing.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@16719 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Block Distribution
//
//      Block       BlockDom     BlockArr
//
//   LocBlock    LocBlockDom  LocBlockArr
//

use DSIUtil;
use ChapelUtil;

//
// Limitations
//
// 1. Changes to Block.tasksPerLocale are not made in privatized
//    copies of this distribution.
//

config param debugBlockDist = false; // internal development flag (debugging)
config param sanityCheckDistribution = false; // ditto; should promote to compiler flag

////////////////////////////////////////////////////////////////////////////////
// Block Distribution Class
//
class Block : BaseDist {
  param rank: int;
  type idxType = int;

  const boundingBox: domain(rank, idxType);
  const targetLocDom: domain(rank);
  const targetLocs: [targetLocDom] locale;
  const locDist: [targetLocDom] LocBlock(rank, idxType);

  var tasksPerLocale: int; // tasks per locale for forall iteration

  var pid: int = -1; // privatized object id

  def Block(param rank: int,
            type idxType = int,
            bbox: domain(rank, idxType),
            targetLocales: [] locale = thisRealm.Locales,
            tasksPerLocale = 0) {
    boundingBox = bbox;
    if rank != 1 && targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank do
        ranges(i) = 0..#factors(i);
      targetLocDom = [(...ranges)];
      targetLocs = reshape(targetLocales, targetLocDom);
    } else {
      if targetLocales.rank != rank then
	compilerError("targetLocales rank must equal distribution rank or one");

      var ranges: rank*range;
      for param i in 1..rank do
	ranges(i) = 0..#targetLocales.domain.dim(i).length;

      targetLocDom = [(...ranges)];
      targetLocs = targetLocales;
    }

    if debugBlockDist {
      writeln(targetLocDom);
      writeln(targetLocs);
    }

    const boundingBoxDims = boundingBox.dims();
    const targetLocDomDims = targetLocDom.dims();
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) =  new LocBlock(rank, idxType, locid, boundingBoxDims,
                                       targetLocDomDims);

    if tasksPerLocale == 0 then
      this.tasksPerLocale = min reduce targetLocs.numCores;
    else
      this.tasksPerLocale = tasksPerLocale;

    if debugBlockDist then
      for loc in locDist do writeln(loc);
  }

  // copy constructor for privatization
  def Block(param rank: int, type idxType, other: Block(rank, idxType), privateData) {
    boundingBox = [(...privateData(1))];
    targetLocDom = [(...privateData(2))];
    tasksPerLocale = privateData(3);
    for i in targetLocDom {
      targetLocs(i) = other.targetLocs(i);
      locDist(i) = other.locDist(i);
    }
  }

  def dsiClone() {
    return new Block(rank, idxType, boundingBox, targetLocs, tasksPerLocale);
  }

  def dsiDestroyDistClass() {
    coforall ld in locDist do {
      on ld do
        delete ld;
    }
  }
}

def Block.dsiDisplayRepresentation() {
  writeln("boundingBox = ", boundingBox);
  writeln("targetLocDom = ", targetLocDom);
  writeln("targetLocs = ", for tl in targetLocs do tl.id);
  writeln("tasksPerLocale = ", tasksPerLocale);
  for tli in targetLocDom do
    writeln("locDist[", tli, "].myChunk = ", locDist[tli].myChunk);
}

def Block.dsiSupportsPrivatization() param return true;

def Block.dsiGetPrivatizeData() {
  return (boundingBox.dims(), targetLocDom.dims(), tasksPerLocale);
}

def Block.dsiPrivatize(privatizeData) {
  return new Block(rank=rank, idxType=idxType, this, privatizeData);
}

def Block.dsiReprivatize(other) {
  boundingBox = other.boundingBox;
  targetLocDom = other.targetLocDom;
  targetLocs = other.targetLocs;
  locDist = other.locDist;
  tasksPerLocale = other.tasksPerLocale;
}

//
// create a new arithmetic domain over this distribution
//
def Block.dsiNewArithmeticDom(param rank: int, type idxType,
                           param stridable: bool) {
  if idxType != this.idxType then
    compilerError("Block domain index type does not match distribution's");
  if rank != this.rank then
    compilerError("Block domain rank does not match distribution's");

  var dom = new BlockDom(rank=rank, idxType=idxType, dist=this, stridable=stridable);
  dom.setup();
  return dom;
}

//
// output distribution
//
def Block.writeThis(x:Writer) {
  x.writeln("Block");
  x.writeln("-------");
  x.writeln("distributes: ", boundingBox);
  x.writeln("across locales: ", targetLocs);
  x.writeln("indexed via: ", targetLocDom);
  x.writeln("resulting in: ");
  for locid in targetLocDom do
    x.writeln("  [", locid, "] ", locDist(locid));
}

//
// convert an index into a locale value
//
// dsiIndexLocale
//  note - we only need the second one
def Block.ind2loc(ind: idxType) where rank == 1 {
  return targetLocs(ind2locInd(ind));
}

def Block.ind2loc(ind: rank*idxType) {
  return targetLocs(ind2locInd(ind));
}

//
// compute what chunk of inds is owned by a given locale -- assumes
// it's being called on the locale in question
//
def Block.getChunk(inds, locid) {
  // use domain slicing to get the intersection between what the
  // locale owns and the domain's index set
  //
  // TODO: Should this be able to be written as myChunk[inds] ???
  //
  // TODO: Does using David's detupling trick work here?
  //
  const chunk = locDist(locid).myChunk((...inds.getIndices()));
  if sanityCheckDistribution then
    if chunk.numIndices > 0 {
      if ind2locInd(chunk.low) != locid then
        writeln("[", here.id, "] ", chunk.low, " is in my chunk but maps to ",
                ind2locInd(chunk.low));
      if ind2locInd(chunk.high) != locid then
        writeln("[", here.id, "] ", chunk.high, " is in my chunk but maps to ",
                ind2locInd(chunk.high));
    }
  return chunk;
}

//
// determine which locale owns a particular index
//
// TODO: I jotted down a note during the code review asking whether
// targetLocs.numElements and boundingbox.numIndices should be
// captured locally, or captured in the default dom/array implementation
// or inlined.  Not sure what that point was anymore, though.  Maybe
// someone else can help me remember it (since it was probably someone
// else's suggestion).
//
def Block.ind2locInd(ind: idxType) where rank == 1 {
  const ind0 = ind - boundingBox.low;
  const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
  return max(0, min(locInd:int, (targetLocDom.dim(1).length-1):int));
}

def Block.ind2locInd(ind: rank*idxType) where rank == 1 {
  return ind2locInd(ind(1));
}

def Block.ind2locInd(ind: rank*idxType) where rank != 1 {
  var locInd: rank*int;
  for param i in 1..rank {
    const ind0 = ind(i) - boundingBox.low(i);
    const dimLocInd = (ind0 * targetLocDom.dim(i).length:idxType) / boundingBox.dim(i).length;
    locInd(i) = max(0, min(dimLocInd:int, (targetLocDom.dim(i).length-1):int));
  }
  return locInd;
}

def Block.dsiCreateReindexDist(newSpace, oldSpace) {
  // Should this error be in ChapelArray or not an error at all?
  if newSpace(1).eltType != oldSpace(1).eltType then
    compilerError("index type of reindex domain must match that of original domain");
  /* To shift the bounding box, we must perform the following operation for
   *  each dimension:
   *
   *   bbox(r).low - (oldSpace(r).low - newSpace(r).low)
   *   bbox(r).high - (oldSpace(r).low - newSpace(r).low)
   *
   * The following is guaranteed on entry:
   *
   *   oldSpace(r).low-newSpace(r).low = oldSpace(r).high-newSpace(r).high
   *
   * We need to be able to do this without going out of range of the index
   *  type.  The approach we take is to check if there is a way to perform
   *  the calculation without having any of the intermediate results go out
   *  of range.
   *
   *    newBblow = bbLow - oldLow + newLow
   *
   * Can be performed as:
   *
   *    t = bbLow-oldLow;
   *    newBbLow = t+newLow;
   * or
   *    t = oldLow-newLow;
   *    newBbLow = t+bbLow;
   * or
   *    t = bbLow+newLow;
   *    newBbLow = t-oldLow;
   *
   */
  def adjustBound(bbound, oldBound, newBound) {
    var t: bbound.type;
    if !subUnderflowsType(bbound, oldBound) {
      t = bbound-oldBound;
      if !addOverflowsType(t, newBound) {
        return (t+newBound, true);
      }
    }
    if !subUnderflowsType(oldBound, newBound) {
      t = oldBound-newBound;
      if !addOverflowsType(t, bbound) {
        return (t+bbound, true);
      }
    }
    if !addOverflowsType(bbound, newBound) {
      t = bbound+newBound;
      if !subUnderflowsType(t, oldBound) {
        return(t-oldBound, true);
      }
    }
    return (bbound, false);
  }

  var myNewBbox = boundingBox.dims();
  for param r in 1..rank {
    var oldLow = oldSpace(r)._low;
    var newLow = newSpace(r)._low;
    var oldHigh = oldSpace(r)._high;
    var newHigh = newSpace(r)._high;
    var valid: bool;
    if oldLow != newLow {
      (myNewBbox(r)._low,valid) = adjustBound(myNewBbox(r).low,oldLow,newLow);
      if !valid then // try with high
        (myNewBbox(r)._low,valid) = adjustBound(myNewBbox(r).low,oldHigh,newHigh);
      if !valid then
        halt("invalid reindex for Block: distribution bounding box (low) out of range in dimension ", r);

      (myNewBbox(r)._high,valid) = adjustBound(myNewBbox(r).high,oldHigh,newHigh);
      if !valid then
        (myNewBbox(r)._high,valid) = adjustBound(myNewBbox(r).high,oldLow,newLow);
      if !valid then // try with low
        halt("invalid reindex for Block: distribution bounding box (high) out of range in dimension ", r);
    }
  }
  var d = [(...myNewBbox)];
  var newDist = new Block(rank, idxType, d, targetLocs, tasksPerLocale);
  return newDist;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Distribution Class
//
class LocBlock {
  param rank: int;
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(rank, idxType);

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock(param rank: int,
               type idxType,
               locid,   // the locale index from the target domain
               boundingBox: rank*range(idxType),
               targetLocBox: rank*range) {
    if rank == 1 {
      const lo = boundingBox(1).low;
      const hi = boundingBox(1).high;
      const numelems = hi - lo + 1;
      const numlocs = targetLocBox(1).length;
      const (blo, bhi) = _computeBlock(numelems, numlocs, locid,
                                       max(idxType), min(idxType), lo);
      myChunk = [blo..bhi];
    } else {
      var tuple: rank*range(idxType);
      for param i in 1..rank {
        const lo = boundingBox(i).low;
        const hi = boundingBox(i).high;
        const numelems = hi - lo + 1;
        const numlocs = targetLocBox(i).length;
        const (blo, bhi) = _computeBlock(numelems, numlocs, locid(i),
                                         max(idxType), min(idxType), lo);
        tuple(i) = blo..bhi;
      }
      myChunk = [(...tuple)];
    }
  }
}

def LocBlock.writeThis(x:Writer) {
  var localeid: int;
  on this {
    localeid = here.uid;
  }
  x.write("locale ", localeid, " owns chunk: ", myChunk);
}

////////////////////////////////////////////////////////////////////////////////
// Block Domain Class
//
class BlockDom: BaseArithmeticDom {
  param rank: int;
  type idxType;
  param stridable: bool;
  //
  // LEFT LINK: a pointer to the parent distribution
  //
  const dist: Block(rank, idxType);

  //
  // DOWN LINK: an array of local domain class descriptors -- set up in
  // setup() below
  //
  var locDoms: [dist.targetLocDom] LocBlockDom(rank, idxType, stridable);

  //
  // a domain describing the complete domain
  //
  const whole: domain(rank=rank, idxType=idxType, stridable=stridable);

  var pid: int = -1; // privatized object id

  def getBaseDist() return dist;
}

def BlockDom.dsiDisplayRepresentation() {
  writeln("whole = ", whole);
  for tli in dist.targetLocDom do
    writeln("locDoms[", tli, "].myBlock = ", locDoms[tli].myBlock);
}

def BlockDom.dsiDims() return whole.dims();

def BlockDom.dsiDim(d: int) return whole.dim(d);

// stopgap to avoid accessing locDoms field (and returning an array)
def BlockDom.getLocDom(localeIdx) return locDoms(localeIdx);

//
// Given a tuple of scalars of type t or range(t) match the shape but
// using types rangeType and scalarType e.g. the call:
// _matchArgsShape(range(int), int, (1:int(64), 1:int(64)..5, 1:int(64)..5))
// returns the type: (int, range(int), range(int))
//
def _matchArgsShape(type rangeType, type scalarType, args) type {
  def tuple(type t ...) type return t;
  def helper(param i: int) type {
    if i == args.size {
      if isCollapsedDimension(args(i)) then
        return tuple(scalarType);
      else
        return tuple(rangeType);
    } else {
      if isCollapsedDimension(args(i)) then
        return (scalarType, (... helper(i+1)));
      else
        return (rangeType, (... helper(i+1)));
    }
  }
  return helper(1);
}

def Block.dsiCreateRankChangeDist(param newRank: int, args) {
  var collapsedDimLocs: rank*idxType;

  for param i in 1..rank {
    if isCollapsedDimension(args(i)) {
      collapsedDimLocs(i) = args(i);
    } else {
      collapsedDimLocs(i) = 0;
    }
  }
  const collapsedLocInd = ind2locInd(collapsedDimLocs);
  var collapsedBbox: _matchArgsShape(range(eltType=idxType), idxType, args);
  var collapsedLocs: _matchArgsShape(range(eltType=int), int, args);

  for param i in 1..rank {
    if isCollapsedDimension(args(i)) {
      // set indicies that are out of bounds to the bounding box low or high.
      collapsedBbox(i) = if args(i) < boundingBox.dim(i).low then boundingBox.dim(i).low else if args(i) > boundingBox.dim(i).high then boundingBox.dim(i).high else args(i);
      collapsedLocs(i) = collapsedLocInd(i);
    } else {
      collapsedBbox(i) = boundingBox.dim(i);
      collapsedLocs(i) = targetLocDom.dim(i);
    }
  }

  const newBbox = boundingBox[(...collapsedBbox)];
  const newTargetLocs = targetLocs((...collapsedLocs));
  return new Block(rank=newRank, idxType=idxType, bbox=newBbox,
                   targetLocales=newTargetLocs,
                   tasksPerLocale=tasksPerLocale);
}

def BlockDom.these() {
  for i in whole do
    yield i;
}

def BlockDom.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dist.tasksPerLocale;
  const precomputedWholeLow = whole.low;
  coforall locDom in locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;

    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }
    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).length, numTasks, taskid,
                                      locBlock(1).high,
                                      locBlock(1).low, locBlock(1).low);
        tuple(1) = lo..hi;
        yield tuple;
      }
    }
  }
}

//
// TODO: Abstract the addition of low into a function?
// Note relationship between this operation and the
// order/position functions -- any chance for creating similar
// support? (esp. given how frequent this seems likely to be?)
//
// TODO: Is there some clever way to invoke the leader/follower
// iterator on the local blocks in here such that the per-core
// parallelism is expressed at that level?  Seems like a nice
// natural composition and might help with my fears about how
// stencil communication will be done on a per-locale basis.
//
def BlockDom.these(param tag: iterator, follower) where tag == iterator.follower {
  var t: rank*range(idxType, stridable=stridable);
  for param i in 1..rank {
    var stride = whole.dim(i).stride: idxType;
    var low = stride * follower(i).low;
    var high = stride * follower(i).high;
    t(i) = (low..high by stride:int) + whole.dim(i).low;
  }
  for i in [(...t)] {
    yield i;
  }
}

//
// output domain
//
def BlockDom.dsiSerialWrite(x:Writer) {
  x.write(whole);
}

//
// how to allocate a new array over this domain
//
def BlockDom.dsiBuildArray(type eltType) {
  var arr = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this);
  arr.setup();
  return arr;
}

def BlockDom.dsiNumIndices return whole.numIndices;
def BlockDom.dsiLow return whole.low;
def BlockDom.dsiHigh return whole.high;

//
// INTERFACE NOTES: Could we make dsiSetIndices() for an arithmetic
// domain take a domain rather than something else?
//
def BlockDom.dsiSetIndices(x: domain) {
  if x.rank != rank then
    compilerError("rank mismatch in domain assignment");
  if x._value.idxType != idxType then
    compilerError("index type mismatch in domain assignment");
  whole = x;
  setup();
}

def BlockDom.dsiSetIndices(x) {
  if x.size != rank then
    compilerError("rank mismatch in domain assignment");
  if x(1).eltType != idxType then
    compilerError("index type mismatch in domain assignment");
  //
  // TODO: This seems weird:
  //
  whole.setIndices(x);
  setup();
}

def BlockDom.dsiGetIndices() {
  return whole.getIndices();
}

// remove all instances of getDist
def BlockDom.getDist(): Block(idxType) {
  return dist;
}

// dsiLocalSlice
def BlockDom.localSlice(param stridable: bool, ranges) {
  return whole((...ranges));
}

def BlockDom.setup() {
  if locDoms(dist.targetLocDom.low) == nil {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocBlockDom(rank, idxType, stridable, this,
                                             dist.getChunk(whole, localeIdx));
    }
  } else {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    }
  }
  if debugBlockDist then
    for loc in dist.targetLocDom do writeln(loc, " owns ", locDoms(loc));
}

def BlockDom.dsiSupportsPrivatization() param return true;

def BlockDom.dsiGetPrivatizeData() return (dist.pid, whole.dims());

def BlockDom.dsiPrivatize(privatizeData) {
  var distpid = privatizeData(1);
  var thisdist = dist;
  var privdist = __primitive("chpl_getPrivatizedClass", thisdist, distpid);
  var c = new BlockDom(rank=rank, idxType=idxType, stridable=stridable, dist=privdist);
  for i in c.dist.targetLocDom do
    c.locDoms(i) = locDoms(i);
  c.whole = [(...privatizeData(2))];
  return c;
}

def BlockDom.dsiGetReprivatizeData() return whole.dims();

def BlockDom.dsiReprivatize(other, reprivatizeData) {
  for i in dist.targetLocDom do
    locDoms(i) = other.locDoms(i);
  whole = [(...reprivatizeData)];
}

def BlockDom.dsiMember(i) {
  return whole.member(i);
}

def BlockDom.dsiIndexOrder(i) {
  return whole.indexOrder(i);
}

//
// build a new arithmetic domain using the given range
//
def BlockDom.dsiBuildArithmeticDom(param rank: int, type idxType,
                                   param stridable: bool,
                                   ranges: rank*range(idxType,
                                                      BoundedRangeType.bounded,
                                                      stridable)) {
  if idxType != dist.idxType then
    compilerError("Block domain index type does not match distribution's");
  if rank != dist.rank then
    compilerError("Block domain rank does not match distribution's");

  var dom = new BlockDom(rank=rank, idxType=idxType,
                         dist=dist, stridable=stridable);
  dom.dsiSetIndices(ranges);
  return dom;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Domain Class
//
class LocBlockDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // UP LINK: a reference to the parent global domain class
  //
  const wholeDom: BlockDom(rank, idxType, stridable);

  //
  // a local domain describing the indices owned by this locale
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(rank, idxType, stridable);
}

//
// output local domain piece
//
def LocBlockDom.writeThis(x:Writer) {
  x.write(myBlock);
}

//
// Added as a performance stopgap to avoid returning a domain
//
def LocBlockDom.member(i) return myBlock.member(i);

////////////////////////////////////////////////////////////////////////////////
// Block Array Class
//
class BlockArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: the global domain descriptor for this array
  //
  var dom: BlockDom(rank, idxType, stridable);

  //
  // DOWN LINK: an array of local array classes
  //
  var locArr: [dom.dist.targetLocDom] LocBlockArr(eltType, rank, idxType, stridable);

  //
  // optimized reference to a local LocBlockArr instance (or nil)
  //
  var myLocArr: LocBlockArr(eltType, rank, idxType, stridable);

  var pid: int = -1; // privatized object id
}

def BlockArr.dsiDisplayRepresentation() {
  for tli in dom.dist.targetLocDom do
    writeln("locArr[", tli, "].myElems = ", for e in locArr[tli].myElems do e);
}

def BlockArr.dsiGetBaseDom() return dom;

def BlockArr.setup() {
  var thisid = this.locale.uid;
  coforall localeIdx in dom.dist.targetLocDom {
    on dom.dist.targetLocs(localeIdx) {
      const locDom = dom.getLocDom(localeIdx);
      locArr(localeIdx) = new LocBlockArr(eltType, rank, idxType, stridable, locDom);
      if thisid == here.uid then
        myLocArr = locArr(localeIdx);
    }
  }
}

def BlockArr.dsiSupportsPrivatization() param return true;

def BlockArr.dsiGetPrivatizeData() return dom.pid;

def BlockArr.dsiPrivatize(privatizeData) {
  var dompid = privatizeData;
  var thisdom = dom;
  var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
  var c = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
  for localeIdx in c.dom.dist.targetLocDom {
    c.locArr(localeIdx) = locArr(localeIdx);
    if c.locArr(localeIdx).locale.uid == here.uid then
      c.myLocArr = c.locArr(localeIdx);
  }
  return c;
}

//
// the global accessor for the array
//
// TODO: Do we need a global bounds check here or in ind2locind?
//
def BlockArr.dsiAccess(i: idxType) var where rank == 1 {
  if myLocArr then local {
    if myLocArr.locDom.member(i) then
      return myLocArr.this(i);
  }
  return locArr(dom.dist.ind2locInd(i))(i);
}

def BlockArr.dsiAccess(i: rank*idxType) var {
  if rank == 1 {
    return dsiAccess(i(1));
  } else {
    if myLocArr then local {
      if myLocArr.locDom.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }
}

def BlockArr.these() var {
  for i in dom do
    yield dsiAccess(i);
}

//
// TODO: Rewrite this to reuse more of the global domain iterator
// logic?  (e.g., can we forward the forall to the global domain
// somehow?
//
def BlockArr.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dom.dist.tasksPerLocale;
  const precomputedWholeLow = dom.whole.low;
  coforall locDom in dom.locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;
    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }

    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).length, numTasks, taskid,
                                      locBlock(1).high,
                                      locBlock(1).low, locBlock(1).low);

        tuple(1) = lo..hi;
        yield tuple;

      }
    }
  }
}

def BlockArr.dsiSupportsAlignedFollower() param return true;

def BlockArr.these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
  var followThis: rank*range(eltType=idxType, stridable=stridable);
  var lowIdx: rank*idxType;

  for param i in 1..rank {
    var stride = dom.whole.dim(i).stride;
    var low = follower(i).low * stride;
    var high = follower(i).high * stride;
    followThis(i) = (low..high by stride) + dom.whole.dim(i).low;
    lowIdx(i) = followThis(i).low;
  }
  const followThisDom = [(...followThis)];

  //
  // TODO: The following is a buggy hack that will only work when we're
  // distributing across the entire Locales array.  I still think the
  // locArr/locDoms arrays should be associative over locale values.
  //
  var arrSection = locArr(dom.dist.ind2locInd(lowIdx));
  if aligned {
    //
    // if arrSection is not local and we're aligned, it means that
    // followThisDom is empty; make arrSection local so that we can
    // use the local block below
    //
    if arrSection.locale.uid != here.uid then
      arrSection = myLocArr;
    local {
      for e in arrSection.myElems(followThisDom) do
        yield e;
    }
  } else {
    //
    // we don't own all the elements we're following
    //
    def accessHelper(i) var {
      local {
        if myLocArr.locDom.member(i) then
          return myLocArr.this(i);
      }
      return dsiAccess(i);
    }
    for i in followThisDom {
      yield accessHelper(i);
    }
  }
}

//
// output array
//
def BlockArr.dsiSerialWrite(f: Writer) {
  if dom.dsiNumIndices == 0 then return;
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dsiDim(dim)._low;
  label next while true {
    f.write(dsiAccess(i));
    if i(rank) <= (dom.dsiDim(rank)._high - dom.dsiDim(rank)._stride:idxType) {
      f.write(" ");
      i(rank) += dom.dsiDim(rank)._stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dsiDim(dim)._high - dom.dsiDim(dim)._stride:idxType) {
          i(dim) += dom.dsiDim(dim)._stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dsiDim(dim2)._low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

def BlockArr.dsiSlice(d: BlockDom) {
  var alias = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, dom=d, pid=pid);
  var thisid = this.locale.uid;
  coforall i in d.dist.targetLocDom {
    on d.dist.targetLocs(i) {
      alias.locArr[i] = new LocBlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, locDom=d.locDoms[i]);
      alias.locArr[i].myElems => locArr[i].myElems[alias.locArr[i].locDom.myBlock];
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }

  return alias;
}

def BlockArr.localSlice(ranges) {
  var low: rank*idxType;
  for param i in 1..rank {
    low(i) = ranges(i).low;
  }
  var A => locArr(dom.dist.ind2locInd(low)).myElems((...ranges));
  return A;
}

def _extendTuple(type t, idx: _tuple, args) {
  var tup: args.size*t;
  var j: int = 1;

  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) then
      tup(i) = args(i);
    else {
      tup(i) = idx(j);
      j += 1;
    }
  }
  return tup;
}

def _extendTuple(type t, idx, args) {
  var tup: args.size*t;
  var idxTup = tuple(idx);
  var j: int = 1;

  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) then
      tup(i) = args(i);
    else {
      tup(i) = idxTup(j);
      j += 1;
    }
  }
  return tup;
}

def BlockArr.dsiRankChange(d, param newRank: int, param stridable: bool, args) {
  var alias = new BlockArr(eltType=eltType, rank=newRank, idxType=idxType, stridable=stridable, dom=d);
  var thisid = this.locale.uid;
  coforall ind in d.dist.targetLocDom {
    on d.dist.targetLocs(ind) {
      const locDom = d.getLocDom(ind);
      // locSlice is a tuple of ranges and scalars. It will match the basic
      // shape of the args argument.
      var locSlice: _matchArgsShape(range(eltType=idxType, stridable=stridable), idxType, args);
      // collapsedDims stores the value any collapsed dimension is down to.
      // For any non-collapsed dimension, that position is ignored.
      // This tuple is then passed to the ind2LocInd function to build up a
      // partial index into this.targetLocDom with correct values set for all
      // collapsed dimensions. The rest of the dimensions get their values from
      // ind - an index into the new rank changed targetLocDom.
      var collapsedDims: rank*idxType;
      var locArrInd: rank*int;

      var j = 1;
      for param i in 1..args.size {
        if isCollapsedDimension(args(i)) {
          locSlice(i) = args(i);
          collapsedDims(i) = args(i);
        } else {
          locSlice(i) = locDom.myBlock.dim(j)(args(i));
          j += 1;
        }
      }
      locArrInd = dom.dist.ind2locInd(collapsedDims);
      j = 1;
      // Now that the locArrInd values are known for the collapsed dimensions
      // Pull the rest of the dimensions values from ind
      for param i in 1..args.size {
        if !isCollapsedDimension(args(i)) {
          if newRank > 1 then
            locArrInd(i) = ind(j);
          else
            locArrInd(i) = ind;
          j += 1;
        }
      }

      alias.locArr[ind] = new LocBlockArr(eltType=eltType, rank=newRank, idxType=d.idxType, stridable=d.stridable, locDom=locDom);
      // Set the local portion of the new array to alias to a rank changed slice
      // of the local portion of this
      alias.locArr[ind].myElems =>
        locArr[(...locArrInd)].myElems[(...locSlice)];

      if thisid == here.uid then
        alias.myLocArr = alias.locArr[ind];
    }
  }
  return alias;
}

def BlockArr.dsiReindex(d: BlockDom) {
  var alias = new BlockArr(eltType=eltType, rank=d.rank, idxType=d.idxType,
                           stridable=d.stridable, dom=d);

  var thisid = this.locale.uid;
  coforall i in d.dist.targetLocDom {
    on d.dist.targetLocs(i) {
      const locDom = d.getLocDom(i);
      alias.locArr[i] = new LocBlockArr(eltType=eltType,
                                        rank=rank, idxType=d.idxType,
                                        stridable=d.stridable,
                                        locDom=locDom);
      alias.locArr[i].myElems => locArr[i].myElems;
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }

  return alias;
}

def BlockArr.dsiReallocate(d: domain) {
  //
  // For the default arithmetic array, this function changes the data
  // vector in the array class so that it is setup once the default
  // arithmetic domain is changed.  For this distributed array class,
  // we don't need to do anything, because changing the domain will
  // change the domain in the local array class which will change the
  // data in the local array class.  This will only work if the domain
  // we are reallocating to has the same distribution, but domain
  // assignment is defined so that only the indices are transferred.
  // The distribution remains unchanged.
  //
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Array Class
//
class LocBlockArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlockDom(rank, idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;
}

//
// the accessor for the local array -- assumes the index is local
//
def LocBlockArr.this(i) var {
  return myElems(i);
}

//
// output local array piece
//
def LocBlockArr.writeThis(x: Writer) {
  // note on this fails; see writeThisUsingOn.chpl
  x.write(myElems);
}

//
// Block Distribution
//
//      Block       BlockDom     BlockArr
//
//   LocBlock    LocBlockDom  LocBlockArr
//

use DSIUtil;
use ChapelUtil;

//
// Limitations
//
// 1. Changes to Block.tasksPerLocale are not made in privatized
//    copies of this distribution.
//

config param debugBlockDist = false; // internal development flag (debugging)
config param sanityCheckDistribution = false; // ditto; should promote to compiler flag

////////////////////////////////////////////////////////////////////////////////
// Block Distribution Class
//
class Block : BaseDist {
  param rank: int;
  type idxType = int;

  const boundingBox: domain(rank, idxType);
  const targetLocDom: domain(rank);
  const targetLocs: [targetLocDom] locale;
  const locDist: [targetLocDom] LocBlock(rank, idxType);

  var tasksPerLocale: int; // tasks per locale for forall iteration

  var pid: int = -1; // privatized object id

  def Block(param rank: int,
            type idxType = int,
            bbox: domain(rank, idxType),
            targetLocales: [] locale = thisRealm.Locales,
            tasksPerLocale = 0) {
    boundingBox = bbox;
    if rank != 1 && targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank do
        ranges(i) = 0..#factors(i);
      targetLocDom = [(...ranges)];
      targetLocs = reshape(targetLocales, targetLocDom);
    } else {
      if targetLocales.rank != rank then
	compilerError("targetLocales rank must equal distribution rank or one");

      var ranges: rank*range;
      for param i in 1..rank do
	ranges(i) = 0..#targetLocales.domain.dim(i).length;

      targetLocDom = [(...ranges)];
      targetLocs = targetLocales;
    }

    if debugBlockDist {
      writeln(targetLocDom);
      writeln(targetLocs);
    }

    const boundingBoxDims = boundingBox.dims();
    const targetLocDomDims = targetLocDom.dims();
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) =  new LocBlock(rank, idxType, locid, boundingBoxDims,
                                       targetLocDomDims);

    if tasksPerLocale == 0 then
      this.tasksPerLocale = min reduce targetLocs.numCores;
    else
      this.tasksPerLocale = tasksPerLocale;

    if debugBlockDist then
      for loc in locDist do writeln(loc);
  }

  // copy constructor for privatization
  def Block(param rank: int, type idxType, other: Block(rank, idxType), privateData) {
    boundingBox = [(...privateData(1))];
    targetLocDom = [(...privateData(2))];
    tasksPerLocale = privateData(3);
    for i in targetLocDom {
      targetLocs(i) = other.targetLocs(i);
      locDist(i) = other.locDist(i);
    }
  }

  def dsiClone() {
    return new Block(rank, idxType, boundingBox, targetLocs, tasksPerLocale);
  }

  def dsiDestroyDistClass() {
    coforall ld in locDist do {
      on ld do
        delete ld;
    }
  }
}

def Block.dsiDisplayRepresentation() {
  writeln("boundingBox = ", boundingBox);
  writeln("targetLocDom = ", targetLocDom);
  writeln("targetLocs = ", for tl in targetLocs do tl.id);
  writeln("tasksPerLocale = ", tasksPerLocale);
  for tli in targetLocDom do
    writeln("locDist[", tli, "].myChunk = ", locDist[tli].myChunk);
}

def Block.dsiSupportsPrivatization() param return true;

def Block.dsiGetPrivatizeData() {
  return (boundingBox.dims(), targetLocDom.dims(), tasksPerLocale);
}

def Block.dsiPrivatize(privatizeData) {
  return new Block(rank=rank, idxType=idxType, this, privatizeData);
}

def Block.dsiReprivatize(other) {
  boundingBox = other.boundingBox;
  targetLocDom = other.targetLocDom;
  targetLocs = other.targetLocs;
  locDist = other.locDist;
  tasksPerLocale = other.tasksPerLocale;
}

//
// create a new arithmetic domain over this distribution
//
def Block.dsiNewArithmeticDom(param rank: int, type idxType,
                           param stridable: bool) {
  if idxType != this.idxType then
    compilerError("Block domain index type does not match distribution's");
  if rank != this.rank then
    compilerError("Block domain rank does not match distribution's");

  var dom = new BlockDom(rank=rank, idxType=idxType, dist=this, stridable=stridable);
  dom.setup();
  return dom;
}

//
// output distribution
//
def Block.writeThis(x:Writer) {
  x.writeln("Block");
  x.writeln("-------");
  x.writeln("distributes: ", boundingBox);
  x.writeln("across locales: ", targetLocs);
  x.writeln("indexed via: ", targetLocDom);
  x.writeln("resulting in: ");
  for locid in targetLocDom do
    x.writeln("  [", locid, "] ", locDist(locid));
}

//
// convert an index into a locale value
//
// dsiIndexLocale
//  note - we only need the second one
def Block.ind2loc(ind: idxType) where rank == 1 {
  return targetLocs(ind2locInd(ind));
}

def Block.ind2loc(ind: rank*idxType) {
  return targetLocs(ind2locInd(ind));
}

//
// compute what chunk of inds is owned by a given locale -- assumes
// it's being called on the locale in question
//
def Block.getChunk(inds, locid) {
  // use domain slicing to get the intersection between what the
  // locale owns and the domain's index set
  //
  // TODO: Should this be able to be written as myChunk[inds] ???
  //
  // TODO: Does using David's detupling trick work here?
  //
  const chunk = locDist(locid).myChunk((...inds.getIndices()));
  if sanityCheckDistribution then
    if chunk.numIndices > 0 {
      if ind2locInd(chunk.low) != locid then
        writeln("[", here.id, "] ", chunk.low, " is in my chunk but maps to ",
                ind2locInd(chunk.low));
      if ind2locInd(chunk.high) != locid then
        writeln("[", here.id, "] ", chunk.high, " is in my chunk but maps to ",
                ind2locInd(chunk.high));
    }
  return chunk;
}

//
// determine which locale owns a particular index
//
// TODO: I jotted down a note during the code review asking whether
// targetLocs.numElements and boundingbox.numIndices should be
// captured locally, or captured in the default dom/array implementation
// or inlined.  Not sure what that point was anymore, though.  Maybe
// someone else can help me remember it (since it was probably someone
// else's suggestion).
//
def Block.ind2locInd(ind: idxType) where rank == 1 {
  const ind0 = ind - boundingBox.low;
  const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
  return max(0, min(locInd:int, (targetLocDom.dim(1).length-1):int));
}

def Block.ind2locInd(ind: rank*idxType) where rank == 1 {
  return ind2locInd(ind(1));
}

def Block.ind2locInd(ind: rank*idxType) where rank != 1 {
  var locInd: rank*int;
  for param i in 1..rank {
    const ind0 = ind(i) - boundingBox.low(i);
    const dimLocInd = (ind0 * targetLocDom.dim(i).length:idxType) / boundingBox.dim(i).length;
    locInd(i) = max(0, min(dimLocInd:int, (targetLocDom.dim(i).length-1):int));
  }
  return locInd;
}

def Block.dsiCreateReindexDist(newSpace, oldSpace) {
  // Should this error be in ChapelArray or not an error at all?
  if newSpace(1).eltType != oldSpace(1).eltType then
    compilerError("index type of reindex domain must match that of original domain");
  /* To shift the bounding box, we must perform the following operation for
   *  each dimension:
   *
   *   bbox(r).low - (oldSpace(r).low - newSpace(r).low)
   *   bbox(r).high - (oldSpace(r).low - newSpace(r).low)
   *
   * The following is guaranteed on entry:
   *
   *   oldSpace(r).low-newSpace(r).low = oldSpace(r).high-newSpace(r).high
   *
   * We need to be able to do this without going out of range of the index
   *  type.  The approach we take is to check if there is a way to perform
   *  the calculation without having any of the intermediate results go out
   *  of range.
   *
   *    newBblow = bbLow - oldLow + newLow
   *
   * Can be performed as:
   *
   *    t = bbLow-oldLow;
   *    newBbLow = t+newLow;
   * or
   *    t = oldLow-newLow;
   *    newBbLow = t+bbLow;
   * or
   *    t = bbLow+newLow;
   *    newBbLow = t-oldLow;
   *
   */
  def adjustBound(bbound, oldBound, newBound) {
    var t: bbound.type;
    if !subUnderflowsType(bbound, oldBound) {
      t = bbound-oldBound;
      if !addOverflowsType(t, newBound) {
        return (t+newBound, true);
      }
    }
    if !subUnderflowsType(oldBound, newBound) {
      t = oldBound-newBound;
      if !addOverflowsType(t, bbound) {
        return (t+bbound, true);
      }
    }
    if !addOverflowsType(bbound, newBound) {
      t = bbound+newBound;
      if !subUnderflowsType(t, oldBound) {
        return(t-oldBound, true);
      }
    }
    return (bbound, false);
  }

  var myNewBbox = boundingBox.dims();
  for param r in 1..rank {
    var oldLow = oldSpace(r)._low;
    var newLow = newSpace(r)._low;
    var oldHigh = oldSpace(r)._high;
    var newHigh = newSpace(r)._high;
    var valid: bool;
    if oldLow != newLow {
      (myNewBbox(r)._low,valid) = adjustBound(myNewBbox(r).low,oldLow,newLow);
      if !valid then // try with high
        (myNewBbox(r)._low,valid) = adjustBound(myNewBbox(r).low,oldHigh,newHigh);
      if !valid then
        halt("invalid reindex for Block: distribution bounding box (low) out of range in dimension ", r);

      (myNewBbox(r)._high,valid) = adjustBound(myNewBbox(r).high,oldHigh,newHigh);
      if !valid then
        (myNewBbox(r)._high,valid) = adjustBound(myNewBbox(r).high,oldLow,newLow);
      if !valid then // try with low
        halt("invalid reindex for Block: distribution bounding box (high) out of range in dimension ", r);
    }
  }
  var d = [(...myNewBbox)];
  var newDist = new Block(rank, idxType, d, targetLocs, tasksPerLocale);
  return newDist;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Distribution Class
//
class LocBlock {
  param rank: int;
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(rank, idxType);

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock(param rank: int,
               type idxType,
               locid,   // the locale index from the target domain
               boundingBox: rank*range(idxType),
               targetLocBox: rank*range) {
    if rank == 1 {
      const lo = boundingBox(1).low;
      const hi = boundingBox(1).high;
      const numelems = hi - lo + 1;
      const numlocs = targetLocBox(1).length;
      const (blo, bhi) = _computeBlock(numelems, numlocs, locid,
                                       max(idxType), min(idxType), lo);
      myChunk = [blo..bhi];
    } else {
      var tuple: rank*range(idxType);
      for param i in 1..rank {
        const lo = boundingBox(i).low;
        const hi = boundingBox(i).high;
        const numelems = hi - lo + 1;
        const numlocs = targetLocBox(i).length;
        const (blo, bhi) = _computeBlock(numelems, numlocs, locid(i),
                                         max(idxType), min(idxType), lo);
        tuple(i) = blo..bhi;
      }
      myChunk = [(...tuple)];
    }
  }
}

def LocBlock.writeThis(x:Writer) {
  var localeid: int;
  on this {
    localeid = here.uid;
  }
  x.write("locale ", localeid, " owns chunk: ", myChunk);
}

////////////////////////////////////////////////////////////////////////////////
// Block Domain Class
//
class BlockDom: BaseArithmeticDom {
  param rank: int;
  type idxType;
  param stridable: bool;
  //
  // LEFT LINK: a pointer to the parent distribution
  //
  const dist: Block(rank, idxType);

  //
  // DOWN LINK: an array of local domain class descriptors -- set up in
  // setup() below
  //
  var locDoms: [dist.targetLocDom] LocBlockDom(rank, idxType, stridable);

  //
  // a domain describing the complete domain
  //
  const whole: domain(rank=rank, idxType=idxType, stridable=stridable);

  var pid: int = -1; // privatized object id

  def getBaseDist() return dist;
}

def BlockDom.dsiDisplayRepresentation() {
  writeln("whole = ", whole);
  for tli in dist.targetLocDom do
    writeln("locDoms[", tli, "].myBlock = ", locDoms[tli].myBlock);
}

def BlockDom.dsiDims() return whole.dims();

def BlockDom.dsiDim(d: int) return whole.dim(d);

// stopgap to avoid accessing locDoms field (and returning an array)
def BlockDom.getLocDom(localeIdx) return locDoms(localeIdx);

//
// Given a tuple of scalars of type t or range(t) match the shape but
// using types rangeType and scalarType e.g. the call:
// _matchArgsShape(range(int), int, (1:int(64), 1:int(64)..5, 1:int(64)..5))
// returns the type: (int, range(int), range(int))
//
def _matchArgsShape(type rangeType, type scalarType, args) type {
  def tuple(type t ...) type return t;
  def helper(param i: int) type {
    if i == args.size {
      if isCollapsedDimension(args(i)) then
        return tuple(scalarType);
      else
        return tuple(rangeType);
    } else {
      if isCollapsedDimension(args(i)) then
        return (scalarType, (... helper(i+1)));
      else
        return (rangeType, (... helper(i+1)));
    }
  }
  return helper(1);
}

def Block.dsiCreateRankChangeDist(param newRank: int, args) {
  var collapsedDimLocs: rank*idxType;

  for param i in 1..rank {
    if isCollapsedDimension(args(i)) {
      collapsedDimLocs(i) = args(i);
    } else {
      collapsedDimLocs(i) = 0;
    }
  }
  const collapsedLocInd = ind2locInd(collapsedDimLocs);
  var collapsedBbox: _matchArgsShape(range(eltType=idxType), idxType, args);
  var collapsedLocs: _matchArgsShape(range(eltType=int), int, args);

  for param i in 1..rank {
    if isCollapsedDimension(args(i)) {
      collapsedBbox(i) = args(i);
      collapsedLocs(i) = collapsedLocInd(i);
    } else {
      collapsedBbox(i) = boundingBox.dim(i);
      collapsedLocs(i) = targetLocDom.dim(i);
    }
  }

  const newBbox = boundingBox((...collapsedBbox));
  const newTargetLocs = targetLocs((...collapsedLocs));
  return new Block(rank=newRank, idxType=idxType, bbox=newBbox,
                   targetLocales=newTargetLocs,
                   tasksPerLocale=tasksPerLocale);
}

def BlockDom.these() {
  for i in whole do
    yield i;
}

def BlockDom.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dist.tasksPerLocale;
  const precomputedWholeLow = whole.low;
  coforall locDom in locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;

    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }
    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).length, numTasks, taskid,
                                      locBlock(1).high,
                                      locBlock(1).low, locBlock(1).low);
        tuple(1) = lo..hi;
        yield tuple;
      }
    }
  }
}

//
// TODO: Abstract the addition of low into a function?
// Note relationship between this operation and the
// order/position functions -- any chance for creating similar
// support? (esp. given how frequent this seems likely to be?)
//
// TODO: Is there some clever way to invoke the leader/follower
// iterator on the local blocks in here such that the per-core
// parallelism is expressed at that level?  Seems like a nice
// natural composition and might help with my fears about how
// stencil communication will be done on a per-locale basis.
//
def BlockDom.these(param tag: iterator, follower) where tag == iterator.follower {
  var t: rank*range(idxType, stridable=stridable);
  for param i in 1..rank {
    var stride = whole.dim(i).stride: idxType;
    var low = stride * follower(i).low;
    var high = stride * follower(i).high;
    t(i) = (low..high by stride:int) + whole.dim(i).low;
  }
  for i in [(...t)] {
    yield i;
  }
}

//
// output domain
//
def BlockDom.dsiSerialWrite(x:Writer) {
  x.write(whole);
}

//
// how to allocate a new array over this domain
//
def BlockDom.dsiBuildArray(type eltType) {
  var arr = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this);
  arr.setup();
  return arr;
}

def BlockDom.dsiNumIndices return whole.numIndices;
def BlockDom.dsiLow return whole.low;
def BlockDom.dsiHigh return whole.high;

//
// INTERFACE NOTES: Could we make dsiSetIndices() for an arithmetic
// domain take a domain rather than something else?
//
def BlockDom.dsiSetIndices(x: domain) {
  if x.rank != rank then
    compilerError("rank mismatch in domain assignment");
  if x._value.idxType != idxType then
    compilerError("index type mismatch in domain assignment");
  whole = x;
  setup();
}

def BlockDom.dsiSetIndices(x) {
  if x.size != rank then
    compilerError("rank mismatch in domain assignment");
  if x(1).eltType != idxType then
    compilerError("index type mismatch in domain assignment");
  //
  // TODO: This seems weird:
  //
  whole.setIndices(x);
  setup();
}

def BlockDom.dsiGetIndices() {
  return whole.getIndices();
}

// remove all instances of getDist
def BlockDom.getDist(): Block(idxType) {
  return dist;
}

// dsiLocalSlice
def BlockDom.localSlice(param stridable: bool, ranges) {
  return whole((...ranges));
}

def BlockDom.setup() {
  if locDoms(dist.targetLocDom.low) == nil {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocBlockDom(rank, idxType, stridable, this,
                                             dist.getChunk(whole, localeIdx));
    }
  } else {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    }
  }
  if debugBlockDist then
    for loc in dist.targetLocDom do writeln(loc, " owns ", locDoms(loc));
}

def BlockDom.dsiSupportsPrivatization() param return true;

def BlockDom.dsiGetPrivatizeData() return (dist.pid, whole.dims());

def BlockDom.dsiPrivatize(privatizeData) {
  var distpid = privatizeData(1);
  var thisdist = dist;
  var privdist = __primitive("chpl_getPrivatizedClass", thisdist, distpid);
  var c = new BlockDom(rank=rank, idxType=idxType, stridable=stridable, dist=privdist);
  for i in c.dist.targetLocDom do
    c.locDoms(i) = locDoms(i);
  c.whole = [(...privatizeData(2))];
  return c;
}

def BlockDom.dsiGetReprivatizeData() return whole.dims();

def BlockDom.dsiReprivatize(other, reprivatizeData) {
  for i in dist.targetLocDom do
    locDoms(i) = other.locDoms(i);
  whole = [(...reprivatizeData)];
}

def BlockDom.dsiMember(i) {
  return whole.member(i);
}

def BlockDom.dsiIndexOrder(i) {
  return whole.indexOrder(i);
}

//
// build a new arithmetic domain using the given range
//
def BlockDom.dsiBuildArithmeticDom(param rank: int, type idxType,
                                   param stridable: bool,
                                   ranges: rank*range(idxType,
                                                      BoundedRangeType.bounded,
                                                      stridable)) {
  if idxType != dist.idxType then
    compilerError("Block domain index type does not match distribution's");
  if rank != dist.rank then
    compilerError("Block domain rank does not match distribution's");

  var dom = new BlockDom(rank=rank, idxType=idxType,
                         dist=dist, stridable=stridable);
  dom.dsiSetIndices(ranges);
  return dom;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Domain Class
//
class LocBlockDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // UP LINK: a reference to the parent global domain class
  //
  const wholeDom: BlockDom(rank, idxType, stridable);

  //
  // a local domain describing the indices owned by this locale
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(rank, idxType, stridable);
}

//
// output local domain piece
//
def LocBlockDom.writeThis(x:Writer) {
  x.write(myBlock);
}

//
// Added as a performance stopgap to avoid returning a domain
//
def LocBlockDom.member(i) return myBlock.member(i);

////////////////////////////////////////////////////////////////////////////////
// Block Array Class
//
class BlockArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: the global domain descriptor for this array
  //
  var dom: BlockDom(rank, idxType, stridable);

  //
  // DOWN LINK: an array of local array classes
  //
  var locArr: [dom.dist.targetLocDom] LocBlockArr(eltType, rank, idxType, stridable);

  //
  // optimized reference to a local LocBlockArr instance (or nil)
  //
  var myLocArr: LocBlockArr(eltType, rank, idxType, stridable);

  var pid: int = -1; // privatized object id
}

def BlockArr.dsiDisplayRepresentation() {
  for tli in dom.dist.targetLocDom do
    writeln("locArr[", tli, "].myElems = ", for e in locArr[tli].myElems do e);
}

def BlockArr.dsiGetBaseDom() return dom;

def BlockArr.setup() {
  var thisid = this.locale.uid;
  coforall localeIdx in dom.dist.targetLocDom {
    on dom.dist.targetLocs(localeIdx) {
      const locDom = dom.getLocDom(localeIdx);
      locArr(localeIdx) = new LocBlockArr(eltType, rank, idxType, stridable, locDom);
      if thisid == here.uid then
        myLocArr = locArr(localeIdx);
    }
  }
}

def BlockArr.dsiSupportsPrivatization() param return true;

def BlockArr.dsiGetPrivatizeData() return dom.pid;

def BlockArr.dsiPrivatize(privatizeData) {
  var dompid = privatizeData;
  var thisdom = dom;
  var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
  var c = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
  for localeIdx in c.dom.dist.targetLocDom {
    c.locArr(localeIdx) = locArr(localeIdx);
    if c.locArr(localeIdx).locale.uid == here.uid then
      c.myLocArr = c.locArr(localeIdx);
  }
  return c;
}

//
// the global accessor for the array
//
// TODO: Do we need a global bounds check here or in ind2locind?
//
def BlockArr.dsiAccess(i: idxType) var where rank == 1 {
  if myLocArr then local {
    if myLocArr.locDom.member(i) then
      return myLocArr.this(i);
  }
  return locArr(dom.dist.ind2locInd(i))(i);
}

def BlockArr.dsiAccess(i: rank*idxType) var {
  if rank == 1 {
    return dsiAccess(i(1));
  } else {
    if myLocArr then local {
      if myLocArr.locDom.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }
}

def BlockArr.these() var {
  for i in dom do
    yield dsiAccess(i);
}

//
// TODO: Rewrite this to reuse more of the global domain iterator
// logic?  (e.g., can we forward the forall to the global domain
// somehow?
//
def BlockArr.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dom.dist.tasksPerLocale;
  const precomputedWholeLow = dom.whole.low;
  coforall locDom in dom.locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;
    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }

    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).length, numTasks, taskid,
                                      locBlock(1).high,
                                      locBlock(1).low, locBlock(1).low);

        tuple(1) = lo..hi;
        yield tuple;

      }
    }
  }
}

def BlockArr.dsiSupportsAlignedFollower() param return true;

def BlockArr.these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
  var followThis: rank*range(eltType=idxType, stridable=stridable);
  var lowIdx: rank*idxType;

  for param i in 1..rank {
    var stride = dom.whole.dim(i).stride;
    var low = follower(i).low * stride;
    var high = follower(i).high * stride;
    followThis(i) = (low..high by stride) + dom.whole.dim(i).low;
    lowIdx(i) = followThis(i).low;
  }
  const followThisDom = [(...followThis)];

  //
  // TODO: The following is a buggy hack that will only work when we're
  // distributing across the entire Locales array.  I still think the
  // locArr/locDoms arrays should be associative over locale values.
  //
  var arrSection = locArr(dom.dist.ind2locInd(lowIdx));
  if aligned {
    //
    // if arrSection is not local and we're aligned, it means that
    // followThisDom is empty; make arrSection local so that we can
    // use the local block below
    //
    if arrSection.locale.uid != here.uid then
      arrSection = myLocArr;
    local {
      for e in arrSection.myElems(followThisDom) do
        yield e;
    }
  } else {
    //
    // we don't own all the elements we're following
    //
    def accessHelper(i) var {
      local {
        if myLocArr.locDom.member(i) then
          return myLocArr.this(i);
      }
      return dsiAccess(i);
    }
    for i in followThisDom {
      yield accessHelper(i);
    }
  }
}

//
// output array
//
def BlockArr.dsiSerialWrite(f: Writer) {
  if dom.dsiNumIndices == 0 then return;
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dsiDim(dim)._low;
  label next while true {
    f.write(dsiAccess(i));
    if i(rank) <= (dom.dsiDim(rank)._high - dom.dsiDim(rank)._stride:idxType) {
      f.write(" ");
      i(rank) += dom.dsiDim(rank)._stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dsiDim(dim)._high - dom.dsiDim(dim)._stride:idxType) {
          i(dim) += dom.dsiDim(dim)._stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dsiDim(dim2)._low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

def BlockArr.dsiSlice(d: BlockDom) {
  var alias = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, dom=d, pid=pid);
  var thisid = this.locale.uid;
  coforall i in d.dist.targetLocDom {
    on d.dist.targetLocs(i) {
      alias.locArr[i] = new LocBlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, locDom=d.locDoms[i]);
      alias.locArr[i].myElems => locArr[i].myElems[alias.locArr[i].locDom.myBlock];
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }

  return alias;
}

def BlockArr.localSlice(ranges) {
  var low: rank*idxType;
  for param i in 1..rank {
    low(i) = ranges(i).low;
  }
  var A => locArr(dom.dist.ind2locInd(low)).myElems((...ranges));
  return A;
}

def _extendTuple(type t, idx: _tuple, args) {
  var tup: args.size*t;
  var j: int = 1;

  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) then
      tup(i) = args(i);
    else {
      tup(i) = idx(j);
      j += 1;
    }
  }
  return tup;
}

def _extendTuple(type t, idx, args) {
  var tup: args.size*t;
  var idxTup = tuple(idx);
  var j: int = 1;

  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) then
      tup(i) = args(i);
    else {
      tup(i) = idxTup(j);
      j += 1;
    }
  }
  return tup;
}

def BlockArr.dsiRankChange(d, param newRank: int, param stridable: bool, args) {
  var alias = new BlockArr(eltType=eltType, rank=newRank, idxType=idxType, stridable=stridable, dom=d);
  var thisid = this.locale.uid;
  coforall ind in d.dist.targetLocDom {
    on d.dist.targetLocs(ind) {
      const locDom = d.getLocDom(ind);
      var locSlice = args;
      var j = 1;
      var locArrInd: rank*int;
      var collapsedDims: rank*idxType;

      for param i in 1..args.size {
        if isCollapsedDimension(args(i)) {
          locSlice(i) = args(i);
          collapsedDims(i) = args(i);
        } else {
          locSlice(i) = locDom.myBlock.dim(j)(args(i));
          collapsedDims(i) = 0;
          j += 1;
        }
      }
      locArrInd = dom.dist.ind2locInd(collapsedDims);
      alias.locArr[ind] = new LocBlockArr(eltType=eltType, rank=newRank, idxType=d.idxType, stridable=d.stridable, locDom=locDom);

      alias.locArr[ind].myElems =>
        locArr[(...locArrInd)].
        myElems[(...locSlice)];
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[ind];
    }
  }
  return alias;
}

def BlockArr.dsiReindex(d: BlockDom) {
  var alias = new BlockArr(eltType=eltType, rank=d.rank, idxType=d.idxType,
                           stridable=d.stridable, dom=d);

  var thisid = this.locale.uid;
  coforall i in d.dist.targetLocDom {
    on d.dist.targetLocs(i) {
      const locDom = d.getLocDom(i);
      alias.locArr[i] = new LocBlockArr(eltType=eltType,
                                        rank=rank, idxType=d.idxType,
                                        stridable=d.stridable,
                                        locDom=locDom);
      alias.locArr[i].myElems => locArr[i].myElems;
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }

  return alias;
}

def BlockArr.dsiReallocate(d: domain) {
  //
  // For the default arithmetic array, this function changes the data
  // vector in the array class so that it is setup once the default
  // arithmetic domain is changed.  For this distributed array class,
  // we don't need to do anything, because changing the domain will
  // change the domain in the local array class which will change the
  // data in the local array class.  This will only work if the domain
  // we are reallocating to has the same distribution, but domain
  // assignment is defined so that only the indices are transferred.
  // The distribution remains unchanged.
  //
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Array Class
//
class LocBlockArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlockDom(rank, idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;
}

//
// the accessor for the local array -- assumes the index is local
//
def LocBlockArr.this(i) var {
  return myElems(i);
}

//
// output local array piece
//
def LocBlockArr.writeThis(x: Writer) {
  // note on this fails; see writeThisUsingOn.chpl
  x.write(myElems);
}


Make error arguments consistent with other modules

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use Error;

/* Returns the canonical path specified, eliminating symbolic links.  If an
   error occurred, it will be returned via the out parameter.
   err: a syserr used to indicate if an error occurred during this operation.
   name: a string representing a valid path.
*/
proc realPath(out error: syserr, name: string): string {
  extern proc chpl_fs_realpath(path: c_string, ref shortened: c_string_copy): syserr;

  var res: c_string_copy;
  error = chpl_fs_realpath(name.c_str(), res);
  return toString(res);
}

/* Returns the canonical path specified, eliminating symbolic links.  May
   halt with an error message.
   name: a string representing a valid path.
*/
proc realPath(name: string): string {
  var err: syserr = ENOERR;
  var ret = realPath(err, name);
  if err != ENOERR then ioerror(err, "in realPath of", name);
  return ret;
}

/* Returns the canonical path referred to by the file record, eliminating
   symbolic links.  If an error occurred, it will be returned via the out
   parameter.
   err: a syserr used to indicate if an error occurred during this operation.
*/
proc file.realPath(out error: syserr): string {
  extern proc chpl_fs_realpath_file(path: qio_file_ptr_t, ref shortened: c_string_copy): syserr;

  var res: c_string_copy;

  if (is_c_nil(_file_internal)) {
    // This file is referencing a null file.  We'll get a segfault if we
    // continue.
    error = EBADF;
    return "";
  }
  error = chpl_fs_realpath_file(_file_internal, res);
  return toString(res);
}

/* Returns the canonical path referred to by the file record, eliminating
   symbolic links.
*/
proc file.realPath(): string {
  var err: syserr = ENOERR;
  var ret = realPath(err);
  if err != ENOERR then ioerror(err, "in file.realPath");
  return ret;
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use Error;

/* Returns the canonical path specified, eliminating symbolic links.  If an
   error occurred, it will be returned via the out parameter.
   err: a syserr used to indicate if an error occurred during this operation.
   name: a string representing a valid path.
*/
proc realPath(out err: syserr, name: string): string {
  extern proc chpl_fs_realpath(path: c_string, ref shortened: c_string_copy): syserr;

  var res: c_string_copy;
  err = chpl_fs_realpath(name.c_str(), res);
  return toString(res);
}

/* Returns the canonical path specified, eliminating symbolic links.  May
   halt with an error message.
   name: a string representing a valid path.
*/
proc realPath(name: string): string {
  var err: syserr = ENOERR;
  var ret = realPath(err, name);
  if err != ENOERR then ioerror(err, "in realPath of", name);
  return ret;
}

/* Returns the canonical path referred to by the file record, eliminating
   symbolic links.  If an error occurred, it will be returned via the out
   parameter.
   err: a syserr used to indicate if an error occurred during this operation.
*/
proc file.realPath(out err: syserr): string {
  extern proc chpl_fs_realpath_file(path: qio_file_ptr_t, ref shortened: c_string_copy): syserr;

  var res: c_string_copy;

  if (is_c_nil(_file_internal)) {
    // This file is referencing a null file.  We'll get a segfault if we
    // continue.
    err = EBADF;
    return "";
  }
  err = chpl_fs_realpath_file(_file_internal, res);
  return toString(res);
}

/* Returns the canonical path referred to by the file record, eliminating
   symbolic links.
*/
proc file.realPath(): string {
  var err: syserr = ENOERR;
  var ret = realPath(err);
  if err != ENOERR then ioerror(err, "in file.realPath");
  return ret;
}


Update a title I missed

Update a title I missed

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
   Iterators for distributed iteration over Hadoop Distributed Filesystem

   Iterators that can iterate over distributed data in an HDFS filesystem
   in a distributed manner. See :mod:`HDFS`.
 */
module HDFSiterator {

use HDFS, UtilReplicatedVar, RecordParser;

/* Iterate through an HDFS file (available in the default configured
   HDFS server) and yield records matching a regular expression.

   Serial and leader-follower versions of this iterator are available.

   :arg path: the path to the file within the HDFS server
   :arg rec: the type of the records to return
   :arg regexp: a regular expression with the same number of captures as
                the number of fields in the record type ``rec``

   */
iter HDFSiter(path: string, type rec, regex: string) {

  var hdfs = hdfs_chapel_connect("default", 0);
  var fl = hdfs.hdfs_chapel_open(path, iomode.r);

  var rr = fl.reader();
  var N  = new RecordReader(rec, rr, regex);
  // Make this migrate around to various locales based upon where the blocks reside
  // Reason we haven't done this is because we can't have a yield inside an on clause in a
  // serial iterator
  for n in N.stream() {
    yield n;

  }

  fl.close();
  hdfs.hdfs_chapel_disconnect();
}

// Lame use of a L/F, so it won't support zippering (so we would really like
// to have a standalone parallel iterator for this).
// Parallel IO and zippering will probably not play nice with each, due to not
// having a priori knowledge of how records map to positions in the file.
pragma "no doc"
iter HDFSiter(param tag: iterKind, path: string, type rec, regex: string)
  where tag == iterKind.leader {

    // Represents (offset_in_file, length_of_block)
    type hdfsInfo = 2*int(64);

    // Create a work queue for each locale
    // Not actually a FIFO. (could be an unordered set)
    var workQueue: [LocaleSpace] domain(hdfsInfo);

    // ----- Create a file  and filesystem pointer per locale ----
    var hdfs = hdfsChapelConnect("default", 0);
    var globalFile   = hdfs.hdfsOpen(path, iomode.r);

    // -------- Get locales for blocks ------
    // Get the local file
    var localFile = globalFile.getLocal();

    // Get the mapping of host for blocks for this file along
    // with how many blocks  there are
    var (hosts, numBlocksForFile) = getHosts(localFile);

    // Populate the work queue with work
    for block in 0..numBlocksForFile-1 {
      var h = getLocaleBytes(hosts, block);
      var r: hdfsInfo;
      r(1) = h.start_byte;
      r(2) = h.len;
      workQueue[h.locale_id] += r;
    }

    // NOTE: Second component on HDFs tuples are all the same except for the last
    // block. We could code around this to make it more compact (by specializing for
    // the last block).
    coforall loc in Locales {
      on loc {
        // Rip though the work in parallel on that locale
        forall byteRange in workQueue[loc.id] {
          var rr = globalFile.hdfsReader(start=byteRange(1));
          var N  = new RecordReader(rec, rr, regex);
          // Dont have a L/F for this so we yield serially.
          for n in N.stream_num(byteRange(1), byteRange(2)) {
            yield n;
          }
        }
      }
    }

    globalFile.hdfsClose();
    hdfs.hdfsChapelDisconnect();
  }

pragma "no doc"
iter HDFSiter(param tag: iterKind, path: string, type rec, regex: string, followThis)
  where tag == iterKind.follower {
    yield followThis;
  }

/* How hdfsOpen (and hdfsChapelConnect) works:

        +-----------------------+
        | hdfsOpen (globalFile) |
        +-----------------------+
                 /|\
                / | \
               /  |  \
              /   |   \
             /    |    \
     +--------+-------+------
     | File 1 | File2 | ...       File numLocales
     +--------+-------+------
   */

} /* end of module */

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
   Iterators that can iterate over distributed data in an HDFS filesystem
   in a distributed manner. See :mod:`HDFS`.
 */
module HDFSiterator {

use HDFS, UtilReplicatedVar, RecordParser;

/* Iterate through an HDFS file (available in the default configured
   HDFS server) and yield records matching a regular expression.

   Serial and leader-follower versions of this iterator are available.

   :arg path: the path to the file within the HDFS server
   :arg rec: the type of the records to return
   :arg regexp: a regular expression with the same number of captures as
                the number of fields in the record type ``rec``

   */
iter HDFSiter(path: string, type rec, regex: string) {

  var hdfs = hdfs_chapel_connect("default", 0);
  var fl = hdfs.hdfs_chapel_open(path, iomode.r);

  var rr = fl.reader();
  var N  = new RecordReader(rec, rr, regex);
  // Make this migrate around to various locales based upon where the blocks reside
  // Reason we haven't done this is because we can't have a yield inside an on clause in a
  // serial iterator
  for n in N.stream() {
    yield n;

  }

  fl.close();
  hdfs.hdfs_chapel_disconnect();
}

// Lame use of a L/F, so it won't support zippering (so we would really like
// to have a standalone parallel iterator for this).
// Parallel IO and zippering will probably not play nice with each, due to not
// having a priori knowledge of how records map to positions in the file.
pragma "no doc"
iter HDFSiter(param tag: iterKind, path: string, type rec, regex: string)
  where tag == iterKind.leader {

    // Represents (offset_in_file, length_of_block)
    type hdfsInfo = 2*int(64);

    // Create a work queue for each locale
    // Not actually a FIFO. (could be an unordered set)
    var workQueue: [LocaleSpace] domain(hdfsInfo);

    // ----- Create a file  and filesystem pointer per locale ----
    var hdfs = hdfsChapelConnect("default", 0);
    var globalFile   = hdfs.hdfsOpen(path, iomode.r);

    // -------- Get locales for blocks ------
    // Get the local file
    var localFile = globalFile.getLocal();

    // Get the mapping of host for blocks for this file along
    // with how many blocks  there are
    var (hosts, numBlocksForFile) = getHosts(localFile);

    // Populate the work queue with work
    for block in 0..numBlocksForFile-1 {
      var h = getLocaleBytes(hosts, block);
      var r: hdfsInfo;
      r(1) = h.start_byte;
      r(2) = h.len;
      workQueue[h.locale_id] += r;
    }

    // NOTE: Second component on HDFs tuples are all the same except for the last
    // block. We could code around this to make it more compact (by specializing for
    // the last block).
    coforall loc in Locales {
      on loc {
        // Rip though the work in parallel on that locale
        forall byteRange in workQueue[loc.id] {
          var rr = globalFile.hdfsReader(start=byteRange(1));
          var N  = new RecordReader(rec, rr, regex);
          // Dont have a L/F for this so we yield serially.
          for n in N.stream_num(byteRange(1), byteRange(2)) {
            yield n;
          }
        }
      }
    }

    globalFile.hdfsClose();
    hdfs.hdfsChapelDisconnect();
  }

pragma "no doc"
iter HDFSiter(param tag: iterKind, path: string, type rec, regex: string, followThis)
  where tag == iterKind.follower {
    yield followThis;
  }

/* How hdfsOpen (and hdfsChapelConnect) works:

        +-----------------------+
        | hdfsOpen (globalFile) |
        +-----------------------+
                 /|\
                / | \
               /  |  \
              /   |   \
             /    |    \
     +--------+-------+------
     | File 1 | File2 | ...       File numLocales
     +--------+-------+------
   */

} /* end of module */


Adapt the CommDiagnostics standard module for nonblocking comm changes.

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// CommDiagnostics.chpl
//
pragma "no use ChapelStandard"
module CommDiagnostics
{
  //
  // multi-locale diagnostics/debugging support
  //

  // There should be a type like this declared in chpl-comm.h with a single
  // function that returns the C struct.  We're not doing it that way yet
  // due to some shortcomings in our extern records implementation.
  // Once that gets sorted out, we can turn this into an extern record,
  // and remove the 8 or so individual functions below that return the
  // various counters.
  extern record chpl_commDiagnostics {
    var get: uint(64);
    var get_nb: uint(64);
    var put: uint(64);
    var put_nb: uint(64);
    var test_nb: uint(64);
    var wait_nb: uint(64);
    var try_nb: uint(64);
    var fork: uint(64);
    var fork_fast: uint(64);
    var fork_nb: uint(64);
  };

  type commDiagnostics = chpl_commDiagnostics;

  extern proc chpl_startVerboseComm();
  extern proc chpl_stopVerboseComm();
  extern proc chpl_startVerboseCommHere();
  extern proc chpl_stopVerboseCommHere();
  extern proc chpl_gen_startCommDiagnostics();
  extern proc chpl_gen_stopCommDiagnostics();
  extern proc chpl_gen_startCommDiagnosticsHere();
  extern proc chpl_gen_stopCommDiagnosticsHere();
  extern proc chpl_resetCommDiagnosticsHere();
  extern proc chpl_getCommDiagnosticsHere(out cd: commDiagnostics);

  proc startVerboseComm() { chpl_startVerboseComm(); }
  proc stopVerboseComm() { chpl_stopVerboseComm(); }
  proc startVerboseCommHere() { chpl_startVerboseCommHere(); }
  proc stopVerboseCommHere() { chpl_stopVerboseCommHere(); }

  proc startCommDiagnostics() {
    chpl_gen_startCommDiagnostics();
  }
  proc stopCommDiagnostics() {
    chpl_gen_stopCommDiagnostics();
  }
  proc startCommDiagnosticsHere() {
    chpl_gen_startCommDiagnosticsHere();
  }
  proc stopCommDiagnosticsHere() {
    chpl_gen_stopCommDiagnosticsHere();
  }

  proc resetCommDiagnostics() {
    for loc in Locales do on loc do
      resetCommDiagnosticsHere();
  }

  inline proc resetCommDiagnosticsHere() {
    chpl_resetCommDiagnosticsHere();
  }

  // See note above regarding extern records
  extern proc chpl_numCommGets(): uint(64);
  extern proc chpl_numCommNBGets(): uint(64);
  extern proc chpl_numCommPuts(): uint(64);
  extern proc chpl_numCommNBPuts(): uint(64);
  extern proc chpl_numCommTestNB(): uint(64);
  extern proc chpl_numCommWaitNB(): uint(64);
  extern proc chpl_numCommTryNB(): uint(64);
  extern proc chpl_numCommForks(): uint(64);
  extern proc chpl_numCommFastForks(): uint(64);
  extern proc chpl_numCommNBForks(): uint(64);

  proc getCommDiagnostics() {
    var D: [LocaleSpace] commDiagnostics;
    for loc in Locales do on loc {
      // See note above regarding extern records
      D(loc.id).get = chpl_numCommGets();
      D(loc.id).get_nb = chpl_numCommNBGets();
      D(loc.id).put = chpl_numCommPuts();
      D(loc.id).put_nb = chpl_numCommNBPuts();
      D(loc.id).test_nb = chpl_numCommTestNB();
      D(loc.id).wait_nb = chpl_numCommWaitNB();
      D(loc.id).try_nb = chpl_numCommTryNB();
      D(loc.id).fork = chpl_numCommForks();
      D(loc.id).fork_fast = chpl_numCommFastForks();
      D(loc.id).fork_nb = chpl_numCommNBForks();
    }
    return D;
  }

  proc getCommDiagnosticsHere() {
    var cd: commDiagnostics;
    cd.get = chpl_numCommGets();
    cd.get_nb = chpl_numCommNBGets();
    cd.put = chpl_numCommPuts();
    cd.put_nb = chpl_numCommNBPuts();
    cd.test_nb = chpl_numCommTestNB();
    cd.wait_nb = chpl_numCommWaitNB();
    cd.try_nb = chpl_numCommTryNB();
    cd.fork = chpl_numCommForks();
    cd.fork_fast = chpl_numCommFastForks();
    cd.fork_nb = chpl_numCommNBForks();
    return cd;
  }

}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// CommDiagnostics.chpl
//
pragma "no use ChapelStandard"
module CommDiagnostics
{
  //
  // multi-locale diagnostics/debugging support
  //

  // There should be a type like this declared in chpl-comm.h with a single
  // function that returns the C struct.  We're not doing it that way yet
  // due to some shortcomings in our extern records implementation.
  // Once that gets sorted out, we can turn this into an extern record,
  // and remove the 8 or so individual functions below that return the
  // various counters.
  extern record chpl_commDiagnostics {
    var get: uint(64);
    var get_nb: uint(64);
    var get_nb_test: uint(64);
    var get_nb_wait: uint(64);
    var put: uint(64);
    var fork: uint(64);
    var fork_fast: uint(64);
    var fork_nb: uint(64);
  };

  type commDiagnostics = chpl_commDiagnostics;

  extern proc chpl_startVerboseComm();
  extern proc chpl_stopVerboseComm();
  extern proc chpl_startVerboseCommHere();
  extern proc chpl_stopVerboseCommHere();
  extern proc chpl_gen_startCommDiagnostics();
  extern proc chpl_gen_stopCommDiagnostics();
  extern proc chpl_gen_startCommDiagnosticsHere();
  extern proc chpl_gen_stopCommDiagnosticsHere();
  extern proc chpl_resetCommDiagnosticsHere();
  extern proc chpl_getCommDiagnosticsHere(out cd: commDiagnostics);

  proc startVerboseComm() { chpl_startVerboseComm(); }
  proc stopVerboseComm() { chpl_stopVerboseComm(); }
  proc startVerboseCommHere() { chpl_startVerboseCommHere(); }
  proc stopVerboseCommHere() { chpl_stopVerboseCommHere(); }

  proc startCommDiagnostics() {
    chpl_gen_startCommDiagnostics();
  }
  proc stopCommDiagnostics() {
    chpl_gen_stopCommDiagnostics();
  }
  proc startCommDiagnosticsHere() {
    chpl_gen_startCommDiagnosticsHere();
  }
  proc stopCommDiagnosticsHere() {
    chpl_gen_stopCommDiagnosticsHere();
  }

  proc resetCommDiagnostics() {
    for loc in Locales do on loc do
      resetCommDiagnosticsHere();
  }

  inline proc resetCommDiagnosticsHere() {
    chpl_resetCommDiagnosticsHere();
  }

  // See note above regarding extern records
  extern proc chpl_numCommGets(): uint(64);
  extern proc chpl_numCommNBGets(): uint(64);
  extern proc chpl_numCommTestNBGets(): uint(64);
  extern proc chpl_numCommWaitNBGets(): uint(64);
  extern proc chpl_numCommPuts(): uint(64);
  extern proc chpl_numCommForks(): uint(64);
  extern proc chpl_numCommFastForks(): uint(64);
  extern proc chpl_numCommNBForks(): uint(64);

  proc getCommDiagnostics() {
    var D: [LocaleSpace] commDiagnostics;
    for loc in Locales do on loc {
      // See note above regarding extern records
      D(loc.id).get = chpl_numCommGets();
      D(loc.id).put = chpl_numCommPuts();
      D(loc.id).fork = chpl_numCommForks();
      D(loc.id).fork_fast = chpl_numCommFastForks();
      D(loc.id).fork_nb = chpl_numCommNBForks();
      D(loc.id).get_nb = chpl_numCommNBGets();
      D(loc.id).get_nb_test = chpl_numCommTestNBGets();
      D(loc.id).get_nb_wait = chpl_numCommWaitNBGets();
    }
    return D;
  }

  proc getCommDiagnosticsHere() {
    var cd: commDiagnostics;
    cd.get = chpl_numCommGets();
    cd.put = chpl_numCommPuts();
    cd.fork = chpl_numCommForks();
    cd.fork_fast = chpl_numCommFastForks();
    cd.fork_nb = chpl_numCommNBForks();
    cd.get_nb = chpl_numCommNBGets();
    cd.get_nb_test = chpl_numCommTestNBGets();
    cd.get_nb_wait = chpl_numCommWaitNBGets();
    return cd;
  }

}