Cleaned up re-privatization code so that 'this' is the only object
that is re-privatized.  Made it more uniform so that the compiler
could easily insert this code.  However, the main question now is in
which functions should it be inserted?  A simple approach is to just
re-privatize 'this' in leaders that have on-statements and do so
within the on-statement, and also re-privatize 'this' in the
respective followers.  That would duplicate what I've done in this
distribution's leader and follower.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15197 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBlock1D = false;

//
// The distribution class
//
class Block1D : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  var tasksPerLoc: int;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1D(type idxType = int(64), bbox: domain(1, idxType),
              targetLocales: [] locale = Locales,
	      tasksPerLocale = 0) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    tasksPerLoc = tasksPerLocale;
    if (tasksPerLoc == 0) then tasksPerLoc = min reduce targetLocs.numCores;
    //    writeln("Using ", tasksPerLoc, " tasks per locale");

    //    writeln("Created a Block1D:\n", this);
  }

  //
  // builds up a privatized (replicated copy)
  //
  def Block1D(type idxType = int(64),
              other: Block1D(idxType)) {
    boundingBox = other.boundingBox;
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    locDist = other.locDist;
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDomain(param rank: int, type idxType, param stridable: bool) {
    var dom = new Block1DDom(idxType=idxType, dist=this, stridable=stridable);
    dom.setup();
    return dom;
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("Block1D");
    x.writeln("-------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    //    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1D(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const (blo, bhi) = chpl_computeBlock(min(idxType), numelems, lo,
                                         max(idxType), numlocs, localeIdx);

    myChunk = [blo..bhi];
    if debugBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseArithmeticDomain {
  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1D(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // setup() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  var pid: int; // privatized object id

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    coforall locDom in locDoms do
      on locDom {
	const locBlock = locDom.myBlock - whole.low,
              numTasks = dist.tasksPerLoc;
	if (numTasks == 1) {
	  yield locBlock;
	} else {
          coforall taskid in 0..#numTasks {
            const (lo,hi) = chpl_computeBlock(locBlock.low, locBlock.numIndices,
                                              locBlock.low, locBlock.high,
                                              numTasks, taskid);
            yield [lo..hi];
          }
        }
      }
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.

    const followThis = follower + whole.low;

    for i in followThis {
      yield i;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    var arr = new Block1DArr(idxType, elemType, stridable, this);
    arr.setup();
    return arr;
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setup();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setup();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist(): Block1D(idxType) {
    return dist;
  }

  def setup() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, stridable, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }

  def supportsPrivatization() param return true;
  def privatize1() {
    var privateDist = new Block1D(idxType, dist);
    var c = new Block1DDom(idxType=idxType, rank=rank, stridable=stridable, dist=privateDist);
    c.locDoms = locDoms;
    c.whole = whole;
    return c;
  }
  def privatize2(pid: int) {
    this.pid = pid;
  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1, stridable);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    halt("This is bogus");
    yield 2;
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {
  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1, stridable);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType, stridable);

  var pid: int; // privatized object id

  def setup() {
    coforall localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, stridable, dom.locDoms(localeIdx));
  }

  def supportsPrivatization() param return true;
  def privatize1() {
    var c = new Block1DArr(idxType, eltType, stridable, dom);
    c.locArr = locArr;
    return c;
  }
  def privatize2(pid) {
    var dompid = dom.pid;
    var thisdom = dom;
    this.dom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
    this.pid = pid;
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    const myLocArr = locArr(here.id);
    local {
      if myLocArr.locDom.myBlock.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: Rewrite this to reuse more of the global domain iterator
    // logic?  (e.g., can we forward the forall to the global domain
    // somehow?
    //
    const numTasks = dom.dist.tasksPerLoc;
    var id = pid, tc = this; // for reprivatization
    coforall locDom in dom.locDoms do
      on locDom {

        // The next line re-privatize 'this'
        var myThis = if _supportsPrivatization(this) then __primitive("chpl_getPrivatizedClass", tc, id) else this;

        const myNumTasks = numTasks;
        const locBlock = locDom.myBlock - myThis.dom.whole.low;
        if (myNumTasks == 1) {
          yield locBlock;
        } else {
          coforall taskid in 0..#myNumTasks {
            const mytaskid = taskid;
            const (lo,hi) = chpl_computeBlock(locBlock.low, locBlock.numIndices,
                                              locBlock.low, locBlock.high,
                                              myNumTasks, mytaskid);
            yield [lo..hi];
          }
        }
      }
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {

    // The next two lines re-privatize 'this'
    var id = pid, tc = this;
    var myThis = if _supportsPrivatization(this) then __primitive("chpl_getPrivatizedClass", tc, id) else this;

    //
    // TODO: Would like to write this as follower += dom.low;
    //
    const followThis = follower + myThis.dom.low;

    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //

    const myLocArr = myThis.locArr(here.id);
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      def accessHelper(i) var {
        local {
          if myLocArr.locDom.myBlock.member(i) then
            return myLocArr.this(i);
        }
        return this(i);
      }
      for i in followThis {
        yield accessHelper(i);
      }
    }

    // SJD: This was here before I added aligned:
    /*
    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if (myLocArr.owns(followThis)) {
      // we own all the elements we're following
      //

      //
      // TODO: Want this, but local does not currently work within
      // follower iterators:
      //
      //      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
        //      }
      //
    } else {
      writeln("Warning: doing expensive these iteration");
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      for i in followThis {
        yield this(i);
      }
    }
    */
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) var where tag == iterator.follower {
    yield myElems(0);
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }

  // INTERNAL INTERFACE:

  def owns(x) {
    //
    // TODO: When this is multidimensional need to do a reduction or something:
    //
    return locDom.myBlock.dim(1).boundsCheck(x.dim(1));
  }
}

//
// a helper function for blocking index ranges:
//

def chpl_computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def procToData(x, lo)
    return lo + (x: lo.type) + (x:real != x:int:real): lo.type;

  const blo = if (blocknum == 0) then waylo
              else procToData((numelems: real * blocknum) / numblocks, lo);
  const bhi = if (blocknum == numblocks - 1) then wayhi
              else procToData((numelems: real * (blocknum+1)) / numblocks, lo) - 1;

  return (blo, bhi);
}

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBlock1D = false;

//
// The distribution class
//
class Block1D : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  var tasksPerLoc: int;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1D(type idxType = int(64), bbox: domain(1, idxType),
              targetLocales: [] locale = Locales,
	      tasksPerLocale = 0) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    tasksPerLoc = tasksPerLocale;
    if (tasksPerLoc == 0) then tasksPerLoc = min reduce targetLocs.numCores;
    //    writeln("Using ", tasksPerLoc, " tasks per locale");

    //    writeln("Created a Block1D:\n", this);
  }

  //
  // builds up a privatized (replicated copy)
  //
  def Block1D(type idxType = int(64),
              other: Block1D(idxType)) {
    boundingBox = other.boundingBox;
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    locDist = other.locDist;
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDomain(param rank: int, type idxType, param stridable: bool) {
    var dom = new Block1DDom(idxType=idxType, dist=this, stridable=stridable);
    dom.setup();
    return dom;
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("Block1D");
    x.writeln("-------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    //    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1D(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const (blo, bhi) = chpl_computeBlock(min(idxType), numelems, lo,
                                         max(idxType), numlocs, localeIdx);

    myChunk = [blo..bhi];
    if debugBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseArithmeticDomain {
  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1D(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // setup() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  var pid: int; // privatized object id

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    coforall locDom in locDoms do
      on locDom {
	const locBlock = locDom.myBlock - whole.low,
              numTasks = dist.tasksPerLoc;
	if (numTasks == 1) {
	  yield locBlock;
	} else {
          coforall taskid in 0..#numTasks {
            const (lo,hi) = chpl_computeBlock(locBlock.low, locBlock.numIndices,
                                              locBlock.low, locBlock.high,
                                              numTasks, taskid);
            yield [lo..hi];
          }
        }
      }
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.

    const followThis = follower + whole.low;

    for i in followThis {
      yield i;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    var arr = new Block1DArr(idxType, elemType, stridable, this);
    arr.setup();
    return arr;
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setup();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setup();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist(): Block1D(idxType) {
    return dist;
  }

  def setup() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, stridable, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }

  def supportsPrivatization() param return true;
  def privatize1() {
    var privateDist = new Block1D(idxType, dist);
    var c = new Block1DDom(idxType=idxType, rank=rank, stridable=stridable, dist=privateDist);
    c.locDoms = locDoms;
    c.whole = whole;
    return c;
  }
  def privatize2(pid: int) {
    this.pid = pid;
  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1, stridable);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    halt("This is bogus");
    yield 2;
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {
  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1, stridable);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType, stridable);

  var pid: int; // privatized object id

  def setup() {
    coforall localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, stridable, dom.locDoms(localeIdx));
  }

  def supportsPrivatization() param return true;
  def privatize1() {
    var c = new Block1DArr(idxType, eltType, stridable, dom);
    c.locArr = locArr;
    return c;
  }
  def privatize2(pid) {
    var dompid = dom.pid;
    var thisdom = dom;
    this.dom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
    this.pid = pid;
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    const myLocArr = locArr(here.id);
    local {
      if myLocArr.locDom.myBlock.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: Rewrite this to reuse more of the global domain iterator
    // logic?  (e.g., can we forward the forall to the global domain
    // somehow?
    //
    const numTasks = dom.dist.tasksPerLoc;
    const dom_id = dom.pid, dom_tc = dom; // for reprivatization
    coforall locDom in dom.locDoms do
      on locDom {

        // The next line re-privatizes dom; referring to this.dom
        // refers to the privatized domain before executing the
        // on-statement; now we want dom to refer to the domain on
        // whatever locale locDom lives on.  Unfortunately, this uses
        // a primitive.  How else can this be done so that
        // user-defined distributions are easier to write?
        const my_dom_id = dom_id;
        const myDom = if _supportsPrivatization(dom) then __primitive("chpl_getPrivatizedClass", dom_tc, my_dom_id) else dom;

        const myNumTasks = numTasks;
        const locBlock = locDom.myBlock - myDom.whole.low;
        if (myNumTasks == 1) {
          yield locBlock;
        } else {
          coforall taskid in 0..#myNumTasks {
            const mytaskid = taskid;
            const (lo,hi) = chpl_computeBlock(locBlock.low, locBlock.numIndices,
                                              locBlock.low, locBlock.high,
                                              myNumTasks, mytaskid);
            yield [lo..hi];
          }
        }
      }
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {

    // The next two lines re-privatize 'this'
    var id = pid, tc = this;
    var myThis = if _supportsPrivatization(this) then __primitive("chpl_getPrivatizedClass", tc, id) else this;

    //
    // TODO: Would like to write this as follower += dom.low;
    //
    const followThis = follower + myThis.dom.low;

    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //

    const myLocArr = myThis.locArr(here.id);
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      def accessHelper(i) var {
        local {
          if myLocArr.locDom.myBlock.member(i) then
            return myLocArr.this(i);
        }
        return this(i);
      }
      for i in followThis {
        yield accessHelper(i);
      }
    }

    // SJD: This was here before I added aligned:
    /*
    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if (myLocArr.owns(followThis)) {
      // we own all the elements we're following
      //

      //
      // TODO: Want this, but local does not currently work within
      // follower iterators:
      //
      //      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
        //      }
      //
    } else {
      writeln("Warning: doing expensive these iteration");
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      for i in followThis {
        yield this(i);
      }
    }
    */
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) var where tag == iterator.follower {
    yield myElems(0);
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }

  // INTERNAL INTERFACE:

  def owns(x) {
    //
    // TODO: When this is multidimensional need to do a reduction or something:
    //
    return locDom.myBlock.dim(1).boundsCheck(x.dim(1));
  }
}

//
// a helper function for blocking index ranges:
//

def chpl_computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def procToData(x, lo)
    return lo + (x: lo.type) + (x:real != x:int:real): lo.type;

  const blo = if (blocknum == 0) then waylo
              else procToData((numelems: real * blocknum) / numblocks, lo);
  const bhi = if (blocknum == numblocks - 1) then wayhi
              else procToData((numelems: real * (blocknum+1)) / numblocks, lo) - 1;

  return (blo, bhi);
}


Fixed a test that should not be working but does.  In a local change, I got
it to work correctly.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@12010 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// Check the record hashing function
record R {
  var x, y, z: int;
}

var r1 = R(0, 0, 0);
writeln( r1);
var r2 = R(1, 0, 0);
writeln( r2);
var r3 = R(0, 1, 0);
writeln( r3);
var r4 = R(0, 0, 1);
writeln( r4);
var r5 = R(1, 1, 1);
writeln( r5);
var r6 = R(1, 0, 1);
writeln( r6);

var h1 = _indefinite_hash( r1);
var h2 = _indefinite_hash( r2);
var h3 = _indefinite_hash( r3);
var h4 = _indefinite_hash( r4);
var h5 = _indefinite_hash( r5);
var h6 = _indefinite_hash( r6);

if (h1==h2 || h1==h3 || h1==h4 || h1==h5 || h1==h6 ||
    h2==h3 || h2==h4 || h2==h5 || h2==h6 ||
    h3==h4 || h3==h5 || h3==h6 ||
    h4==h5 || h4==h6 ||
    h5==h6) {
  writeln( "collision");
  writeln( h1);
  writeln( h2);
  writeln( h3);
  writeln( h4);
  writeln( h5);
  writeln( h6);
} else {
  writeln( "ok, so far");
}

// Check the record hashing function
record R {
  var x, y, z: int;
}

var r1: R( 0, 0, 0);
writeln( r1);
var r2: R(1, 0, 0);
writeln( r2);
var r3: R(0, 1, 0);
writeln( r3);
var r4: R(0, 0, 1);
writeln( r4);
var r5: R(1, 1, 1);
writeln( r5);
var r6: R(1, 0, 1);
writeln( r6);

var h1 = _indefinite_hash( r1);
var h2 = _indefinite_hash( r2);
var h3 = _indefinite_hash( r3);
var h4 = _indefinite_hash( r4);
var h5 = _indefinite_hash( r5);
var h6 = _indefinite_hash( r6);

if (h1==h2 || h1==h3 || h1==h4 || h1==h5 || h1==h6 ||
    h2==h3 || h2==h4 || h2==h5 || h2==h6 ||
    h3==h4 || h3==h5 || h3==h6 ||
    h4==h5 || h4==h6 ||
    h5==h6) {
  writeln( "collision");
  writeln( h1);
  writeln( h2);
  writeln( h3);
  writeln( h4);
  writeln( h5);
  writeln( h6);
} else {
  writeln( "ok, so far");
}


Change the last forall loop in this reduction test to make it deterministic.

Compute the value of a bool independently at each iteration instead of
depending on the value from the previous iteration.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@16103 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// Test logical AND and OR reductions

param M = 10;

var D: domain(1) = [1..M];
var B: [D] bool;
forall i in D do {
  B(i) = true;
}
writeln( "\nB[D] = ", B);
writeln( "&& reduce B[D] = ", && reduce B);
writeln( "|| reduce B[D] = ", || reduce B);

forall i in D do {
  B(i) = false;
}
writeln( "\nB[D] = ", B);
writeln( "&& reduce B[D] = ", && reduce B);
writeln( "|| reduce B[D] = ", || reduce B);

forall i in D do {
  var toggle: bool = i % 2 == 1;
  B(i) = toggle;
}
writeln( "\nB[D] = ", B);
writeln( "&& reduce B[D] = ", && reduce B);
writeln( "|| reduce B[D] = ", || reduce B);

// Test logical AND and OR reductions

param M = 10;

var D: domain(1) = [1..M];
var B: [D] bool;
forall i in D do {
  B(i) = true;
}
writeln( "\nB[D] = ", B);
writeln( "&& reduce B[D] = ", && reduce B);
writeln( "|| reduce B[D] = ", || reduce B);

forall i in D do {
  B(i) = false;
}
writeln( "\nB[D] = ", B);
writeln( "&& reduce B[D] = ", && reduce B);
writeln( "|| reduce B[D] = ", || reduce B);

var toggle: sync bool = false;
forall i in D do {
  var my_toggle = !toggle;
  toggle = my_toggle;
  B(i) = my_toggle;
}
writeln( "\nB[D] = ", B);
writeln( "&& reduce B[D] = ", && reduce B);
writeln( "|| reduce B[D] = ", || reduce B);


Bump ref count on adomain when creating a DefaultOpaqueArr.

The test case classes/bradc/arrayInClass/genericArrayInClass-otharrs was failing because
the implementation of chpl_incRefCountsForDomainsInArrayEltTypes(type eltType) creates a
realio-trulio DefaultOpaqueArray as the value of the variable ev if the element type is
"opaque".  Creating a DefaultOpaqueArray with the same DefaultOpaqueDom as another such
array causes the underlying DefaultSparseDom (==DefaultOpaqueDom.adomain) to be shared.
If we don't bump the reference count at that point, the shared adomain will be reclaimed
prematurely.

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// DefaultOpaque.chpl
//
module DefaultOpaque {

  class DefaultOpaqueDom: BaseOpaqueDom {
    type idxType = _OpaqueIndex;
    param parSafe: bool;
    var dist: DefaultDist;
    var adomain: DefaultAssociativeDom(idxType=_OpaqueIndex, parSafe=parSafe);

    proc linksDistribution() param return false;
    proc dsiLinksDistribution()     return false;

    proc DefaultOpaqueDom(dist: DefaultDist, param parSafe: bool) {
      this.dist = dist;
      adomain = new DefaultAssociativeDom(_OpaqueIndex, dist, parSafe=parSafe);
      // This prevents premature deletion of the contained associated dom, in
      // case it is iterated upon, or other operations that might cause the
      // refCount to take an excursion from zero and return there.
      // When the ref count of a dom goes to zero the dom is reclaimed (freed),
      // and that would be bad.
      adomain.incRefCount();
    }

    proc ~DefaultOpaqueDom() {
      // We assume that adomain is not shared out, so that when the containing
      // opaque dom is deleted, the contained associative domain can be free
      // unconditionally.
      for i in adomain do delete i;
      delete adomain;
    }

    proc dsiCreate() {
      var i = new _OpaqueIndex();
      adomain.dsiAdd(i);
      return i;
    }

    proc dsiGetIndices() return adomain;

    proc dsiSetIndices(b: DefaultAssociativeDom) {
      adomain.dsiSetIndices(b);
    }

    iter these() {
      for i in adomain do
        yield i;
    }

    iter these(param tag: iterKind) where tag == iterKind.leader {
      for block in adomain.these(tag=iterKind.leader) do
        yield block;
    }

    iter these(param tag: iterKind, followThis) where tag == iterKind.follower {
      for i in adomain.these(tag=iterKind.follower, followThis) do
        yield i;
    }

    proc dsiMember(ind: idxType) {
      return adomain.dsiMember(ind);
    }

    proc dsiNumIndices {
      return adomain.dsiNumIndices;
    }

    proc dsiBuildArray(type eltType) {
      var ia = new DefaultOpaqueArr(eltType=eltType, idxType=idxType, parSafe=parSafe, dom=this);
      return ia;
    }
  }

  pragma "auto copy fn"
  proc chpl__autoCopy(x: DefaultOpaqueDom) {
    if ! noRefCount then
      x.incRefCount();
    return x;
  }

  proc chpl__autoDestroy(x: DefaultOpaqueDom) {
    if !noRefCount {
      var cnt = x.destroyDom();
      if cnt == 0 then
        delete x;
    }
  }

  proc DefaultOpaqueDom.dsiRemove(idx: idxType) {
    adomain.dsiRemove(idx);
  }

  proc DefaultOpaqueDom.dsiSerialWrite(f: Writer) {
    adomain.dsiSerialWrite(f);
  }

  proc DefaultOpaqueDom.dsiSerialRead(f: Reader) {
    adomain.dsiSerialRead(f);
  }

  proc DefaultOpaqueArr.dsiSerialWrite(f: Writer) {
    anarray.dsiSerialWrite(f);
  }
  proc DefaultOpaqueArr.dsiSerialRead(f: Reader) {
    anarray.dsiSerialRead(f);
  }

  class DefaultOpaqueArr: BaseArr {
    type eltType;
    type idxType;
    param parSafe: bool;

    var dom: DefaultOpaqueDom(idxType=idxType, parSafe=parSafe);
    var anarray = new DefaultAssociativeArr(eltType=eltType, idxType=idxType,
                                            parSafeDom=parSafe, dom=dom.adomain);

    proc initialize() {
      // In spite of what was promised, the domain owned by the above
      // DefaultOpaqueDom is shared out in the call to new
      // DefaultAssociativeArr() above, so we have to bump its ref count here.
      dom.adomain.incRefCount();
      // We have to bump the reference count to move it away from zero.
      anarray.incRefCount();
    }

    proc ~DefaultOpaqueArr() {
      // Something more may be needed here.
      delete anarray; anarray = nil;

      on dom {
        local dom.remove_arr(this);
        var cnt = dom.destroyDom();
        if cnt == 0 then
          delete dom;
      }
    }

    proc dsiGetBaseDom() return dom;

    proc dsiAccess(ind : idxType) ref : eltType
      return anarray.dsiAccess(ind);

    proc dsiTargetLocales() {
      compilerError("targetLocales is unsupported by opaque domains");
    }

    proc dsiHasSingleLocalSubdomain() param return true;

    proc dsiLocalSubdomain() {
      return _newDomain(dom);
    }

    iter these() ref {
      for e in anarray do
        yield e;
    }

    iter these(param tag: iterKind) where tag == iterKind.leader {
      for block in dom.these(tag=iterKind.leader) do
        yield block;
    }

    iter these(param tag: iterKind, followThis) ref where tag == iterKind.follower {
      for i in dom.these(tag=iterKind.follower, followThis) do
        yield dsiAccess(i);
    }

    iter dsiSorted() {
      for e in anarray.dsiSorted() do
        yield e;
    }
  }

  pragma "auto copy fn"
  proc chpl__autoCopy(x: DefaultOpaqueArr) {
    if !noRefCount then
      x.incRefCount();
    return x;
  }

  proc chpl__autoDestroy(x: DefaultOpaqueArr) {
    if !noRefCount {
      var cnt = x.destroyArr();
      if cnt == 0 then
        delete x;
    }
  }

}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// DefaultOpaque.chpl
//
module DefaultOpaque {

  class DefaultOpaqueDom: BaseOpaqueDom {
    type idxType = _OpaqueIndex;
    param parSafe: bool;
    var dist: DefaultDist;
    var adomain: DefaultAssociativeDom(idxType=_OpaqueIndex, parSafe=parSafe);

    proc linksDistribution() param return false;
    proc dsiLinksDistribution()     return false;

    proc DefaultOpaqueDom(dist: DefaultDist, param parSafe: bool) {
      this.dist = dist;
      adomain = new DefaultAssociativeDom(_OpaqueIndex, dist, parSafe=parSafe);
      // This prevents premature deletion of the contained associated dom, in
      // case it is iterated upon, or other operations that might cause the
      // refCount to take an excursion from zero and return there.
      // When the ref count of a dom goes to zero the dom is reclaimed (freed),
      // and that would be bad.
      adomain.incRefCount();
    }

    proc ~DefaultOpaqueDom() {
      // We assume that adomain is not shared out, so that when the containing
      // opaque dom is deleted, the contained associative domain can be free
      // unconditionally.
      for i in adomain do delete i;
      delete adomain;
    }

    proc dsiCreate() {
      var i = new _OpaqueIndex();
      adomain.dsiAdd(i);
      return i;
    }

    proc dsiGetIndices() return adomain;

    proc dsiSetIndices(b: DefaultAssociativeDom) {
      adomain.dsiSetIndices(b);
    }

    iter these() {
      for i in adomain do
        yield i;
    }

    iter these(param tag: iterKind) where tag == iterKind.leader {
      for block in adomain.these(tag=iterKind.leader) do
        yield block;
    }

    iter these(param tag: iterKind, followThis) where tag == iterKind.follower {
      for i in adomain.these(tag=iterKind.follower, followThis) do
        yield i;
    }

    proc dsiMember(ind: idxType) {
      return adomain.dsiMember(ind);
    }

    proc dsiNumIndices {
      return adomain.dsiNumIndices;
    }

    proc dsiBuildArray(type eltType) {
      var ia = new DefaultOpaqueArr(eltType=eltType, idxType=idxType, parSafe=parSafe, dom=this);
      return ia;
    }
  }

  pragma "auto copy fn"
  proc chpl__autoCopy(x: DefaultOpaqueDom) {
    if ! noRefCount then
      x.incRefCount();
    return x;
  }

  proc chpl__autoDestroy(x: DefaultOpaqueDom) {
    if !noRefCount {
      var cnt = x.destroyDom();
      if cnt == 0 then
        delete x;
    }
  }

  proc DefaultOpaqueDom.dsiRemove(idx: idxType) {
    adomain.dsiRemove(idx);
  }

  proc DefaultOpaqueDom.dsiSerialWrite(f: Writer) {
    adomain.dsiSerialWrite(f);
  }

  proc DefaultOpaqueDom.dsiSerialRead(f: Reader) {
    adomain.dsiSerialRead(f);
  }

  proc DefaultOpaqueArr.dsiSerialWrite(f: Writer) {
    anarray.dsiSerialWrite(f);
  }
  proc DefaultOpaqueArr.dsiSerialRead(f: Reader) {
    anarray.dsiSerialRead(f);
  }

  class DefaultOpaqueArr: BaseArr {
    type eltType;
    type idxType;
    param parSafe: bool;

    var dom: DefaultOpaqueDom(idxType=idxType, parSafe=parSafe);
    var anarray = new DefaultAssociativeArr(eltType=eltType, idxType=idxType,
                                            parSafeDom=parSafe, dom=dom.adomain);

    proc initialize() {
      // We have to bump the reference count to move it away from zero.
      anarray.incRefCount();
    }

    proc ~DefaultOpaqueArr() {
      // Something more may be needed here.
      delete anarray; anarray = nil;

      on dom {
        local dom.remove_arr(this);
        var cnt = dom.destroyDom();
        if cnt == 0 then
          delete dom;
      }
    }

    proc dsiGetBaseDom() return dom;

    proc dsiAccess(ind : idxType) ref : eltType
      return anarray.dsiAccess(ind);

    proc dsiTargetLocales() {
      compilerError("targetLocales is unsupported by opaque domains");
    }

    proc dsiHasSingleLocalSubdomain() param return true;

    proc dsiLocalSubdomain() {
      return _newDomain(dom);
    }

    iter these() ref {
      for e in anarray do
        yield e;
    }

    iter these(param tag: iterKind) where tag == iterKind.leader {
      for block in dom.these(tag=iterKind.leader) do
        yield block;
    }

    iter these(param tag: iterKind, followThis) ref where tag == iterKind.follower {
      for i in dom.these(tag=iterKind.follower, followThis) do
        yield dsiAccess(i);
    }

    iter dsiSorted() {
      for e in anarray.dsiSorted() do
        yield e;
    }
  }

  pragma "auto copy fn"
  proc chpl__autoCopy(x: DefaultOpaqueArr) {
    if !noRefCount then
      x.incRefCount();
    return x;
  }

  proc chpl__autoDestroy(x: DefaultOpaqueArr) {
    if !noRefCount {
      var cnt = x.destroyArr();
      if cnt == 0 then
        delete x;
    }
  }

}