Updated test to meet distribution/domain interface requirements.  This test
fails in codegen with a type mismatch between two runtime types.  They look
identical so I'm going to look into how they are being created.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14599 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    writeln("Created a Block1DDist:\n", this);
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDomain(param rank: int, type idxType, param stridable: bool) {
    return new Block1DDom(idxType=idxType, dist=this);
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseDenseArithmeticDomain {

  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  //
  // TODO: use stridable (sjd: this parameter is required)
  //
  param stridable: bool = true;

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  // CONSTRUCTORS:

  //
  // TODO: Would like to create the following constructor, but this
  //       is problematic because the class' default constructor
  //       sets dist to nil and then tries to derefernce it.  :P
  //
  /*
  def Block1DDom(type idxType, dist: Block1DDist(idxType)) {
    writeln("in my constructor");
    this.dist = dist;
    setupLocalDoms();
  }
  */
  //
  // so doing this instead:
  //
  def initialize() {
    setupLocalDoms();
  }

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    return new Block1DArr(idxType, elemType, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: These are the routines that I hadn't implemented
  // myself in BradsBlock1DPar.chpl:
  //
  def buildEmptyDomain() {
    if this != nil then
      return new Block1DDom(idxType=idxType, dist=dist);
    else {
      //
      // TODO: Add constructors -- now that domains/arrays inherit, it
      // is really annoying to have arguments match in order to all the
      // parent classes first.
      //
      // TODO: This is a complete hack -- the nil distribution is going
      // to get us into trouble, and isn't really what we want to do here.
      // distributions should be creating empty domains
      //
      return new Block1DDom(idxType=idxType,
                            dist= new Block1DDist(idxType, [1..0:idxType],
                                                  Locales));
    }
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setupLocalDoms();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setupLocalDoms();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist() {
    return dist;
  }

  def setupLocalDoms() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType);

  // CONSTRUCTORS:

  def initialize() {
    for localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, dom.locDoms(localeIdx));
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    writeln("Created a Block1DDist:\n", this);
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newDomain(param rank, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newDomain(param rank, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newDomain(param rank, type idxType, param stridable: bool) {
    return new Block1DDom(idxType=idxType, dist=this);
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseDenseArithmeticDomain {

  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  // CONSTRUCTORS:

  //
  // TODO: Would like to create the following constructor, but this
  //       is problematic because the class' default constructor
  //       sets dist to nil and then tries to derefernce it.  :P
  //
  /*
  def Block1DDom(type idxType, dist: Block1DDist(idxType)) {
    writeln("in my constructor");
    this.dist = dist;
    setupLocalDoms();
  }
  */
  //
  // so doing this instead:
  //
  def initialize() {
    setupLocalDoms();
  }

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    return new Block1DArr(idxType, elemType, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: These are the routines that I hadn't implemented
  // myself in BradsBlock1DPar.chpl:
  //
  def buildEmptyDomain() {
    if this != nil then
      return new Block1DDom(idxType=idxType, dist=dist);
    else {
      //
      // TODO: Add constructors -- now that domains/arrays inherit, it
      // is really annoying to have arguments match in order to all the
      // parent classes first.
      //
      // TODO: This is a complete hack -- the nil distribution is going
      // to get us into trouble, and isn't really what we want to do here.
      // distributions should be creating empty domains
      //
      return new Block1DDom(idxType=idxType,
                            dist= new Block1DDist(idxType, [1..0:idxType],
                                                  Locales));
    }
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setupLocalDoms();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setupLocalDoms();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist() {
    return dist;
  }

  def setupLocalDoms() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType);

  // CONSTRUCTORS:

  def initialize() {
    for localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, dom.locDoms(localeIdx));
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}


Fix PARACR-BC-bulk gasnet-fast regression

cf5127fcb987 changed the comm interfaces to take size_t rather than int32_t.
optimizations/bulkcomm/asenjo/redistBlockToCyclic/PARACR-BC-bulk started
seg-faulting with that commit.

This test uses a primitive to call some of the comm interfaces that were
changed, so it just needed a few of the int(32)'s to be updaed to size_t's.

I followed changes made to another test in the original comm interface commit:
optimizations/bulkcomm/asenjo/redistBlockToCyclic/FFT_bulk

proc copyBtoC(A:[], B:[])
{
  A._value.copyBtoC(B);
}

proc copyCtoB(A:[], B:[])
{
  A._value.copyCtoB(B);
}

proc BlockArr.copyBtoC(B)
{
  coforall loc in Locales do on loc
  {
    param stridelevels=1;
    var dststrides:[1..#stridelevels] size_t;
    var srcstrides: [1..#stridelevels] size_t;
    var count: [1..#(stridelevels+1)] size_t;
    var lid=loc.id;

    var numLocales: int(32)=dom.dist.targetLocDom.dim(1).length:int(32);
    var n:int(32)=dom.dist.boundingBox.dim(1).length:int(32);
    var src = locArr[lid].myElems._value.theData;

    dststrides[1]=1;
    srcstrides[1]=numLocales.safeCast(size_t);

    var dststr=dststrides._value.theData;
    var srcstr=srcstrides._value.theData;
    var cnt=count._value.theData;

    //Domain size (n) and first index (arrayini)

    var arrayini:int(32)=dom.dsiLow:int(32);

    //writeln("Domain: ",dom.whole.dims());

    //a,b: first and last global indices in each locale
    var a: int(32)=dom.locDoms[lid].myBlock.low:int(32);
    var b: int(32)=dom.locDoms[lid].myBlock.high:int(32);
    var blksize=b-a+1;
    //writeln("Locale", here.id," : blksize ",blksize," subblock first index  a ",a,

    var t1,t2: real;
    var schunkini: int;
    var chunksize: int(32);
    //Loop of strided chunk sends (chpl_comm_put_strd)
    for k in 0..#numLocales do {

      var dst=(lid+k)%numLocales;

      //Compute the chunk first index in src locale to be send to locale dst.
      if (dst-a%numLocales+arrayini)<0 then schunkini=a+numLocales +(dst-a%numLocales+arrayini);
      else schunkini=a+(dst-a%numLocales+arrayini)%numLocales;

      //Compute chunk size
      if (schunkini+(blksize/numLocales)*numLocales>b) then chunksize=blksize/numLocales;
      else chunksize=blksize/numLocales+1;

      //var destr = privB.locArr[dst].myElems._value.theData;
      var destr = B._value.locArr[dst].myElems._value.theData;
      count[1]=1;
      count[2]=chunksize.safeCast(size_t);

      __primitive("chpl_comm_put_strd",
		  __primitive("array_get",destr,
			      B._value.locArr[dst].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",dststr,dststrides._value.getDataIndex(1)),
		  dst,
		  __primitive("array_get",src,
			      locArr[lid].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",srcstr,srcstrides._value.getDataIndex(1)),
		  __primitive("array_get",cnt, count._value.getDataIndex(1)),
		  stridelevels);

    } // end for dst
  } // end coforall loc
} // end proc

proc  BlockArr.copyCtoB(B)
{

  coforall loc in Locales do on loc
  {
    param stridelevels=1;
    var dststrides:[1..#stridelevels] size_t;
    var srcstrides: [1..#stridelevels] size_t;
    var count: [1..#(stridelevels+1)] size_t;
    var lid=loc.id;
    var numLocales: int=dom.dist.targetLocDom.dim(1).length;
    var n:int(32)=dom.dist.boundingBox.dim(1).length:int(32);

    var dststr=dststrides._value.theData;
    var srcstr=srcstrides._value.theData;
    var cnt=count._value.theData;

    //On each locale (src) we compute the chunk that goes to each dst
    var num: int;
    var schunkini: int;
    var chunksize: int;
    var a: int(32)=dom.locDoms[lid].myBlock.low:int(32);
    var b: int(32)=dom.locDoms[lid].myBlock.high:int(32);
    num=b-a+1;

    var src = locArr[lid].myElems._value.theData;
    var arrayini:int(32)=dom.dsiLow:int(32);

    var t,t1,t2: real;
    t=0.0;
    //Compute the starting index of the src chunk with destination dst

    for dst in 0..#numLocales do {
      if (dst-a%numLocales+arrayini)<0 then schunkini=a+numLocales+(dst-a%numLocales+arrayini);
      else schunkini=a+(dst-a%numLocales+arrayini)%numLocales;

      //Compute the size of the chunk. The number of elements of src subarray is num
      if (schunkini+(num/numLocales)*numLocales>b) then chunksize=num/numLocales;
      else chunksize=num/numLocales+1;

      var destr = B._value.locArr[dst].myElems._value.theData;
      dststrides[1]=numLocales:size_t;
      srcstrides[1]=1;
      count[1]=1;
      count[2]=chunksize:size_t;

      __primitive("chpl_comm_get_strd",
		  __primitive("array_get",src,
			        locArr[lid].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",dststr,dststrides._value.getDataIndex(1)),
		  dst,
		  __primitive("array_get",destr,
			      B._value.locArr[dst].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",srcstr,srcstrides._value.getDataIndex(1)),
		  __primitive("array_get",cnt,count._value.getDataIndex(1)),
		  stridelevels);
    } //end for dst
  }//end for loc
} //end proc copyCtoB

/*********************************************************/
/*		ParaCR algorithm in Chapel               */
/*		Block to cyclic redistribution           */
/*	Loop unrolled to avoid a deep copy of arrays     */
/* Contributed by Juan Lopez, Alberto Sanz & Rafa Asenjo */
/*********************************************************/

//This code solves a dense tridiagonal system following the PARACR algorithm

use Time;
use BlockDist;
use CyclicDist;

//Problem Dimension
config const n:int=1024;

const stages:int = GetNumStages(n);
//writeln("Stages->",stages);

//At the end of stage "RedistStage" it will change from Block to Cyclic Distribution
config const RedistStage:int=log2(numLocales);
//writeln("Stage to redistribute->",RedistStage);

if (RedistStage>stages)
{
  writeln("Not supported: your matrix dimension is smaller than the number of locales");
  exit(0);
}

var error:int=0;

const Space = {1..n};
const Dom = Space dmapped Block(boundingBox=Space);
const DomC = Space dmapped Cyclic(startIdx=(1));

var Dstages: domain(1,int)={1..stages};
var AA,BB,CC,DD, XX:[Dom] real;
var A,B,C,D:[Dom] real;

var P,Q,R,S, X:[DomC] real;
var PP,QQ,RR,SS:[DomC] real;

config const timer : bool =false;

//Timing variables
var t1,t2,t3,t4,t5,t6 :real;
var CalcTime:[Dstages] real=0.0;
var TotCalcTime: real=0.0;

proc main(){

  /* Parallel Cyclic reduction Algorithm */
  SetExampleMatrix();

  if(n==1) {
    X(1)=D(1)/B(1);
    writeln("Done!!. Too easy... bye");
    exit(0);
  }

  if timer then t1=getCurrentTime();

  /*********************/
  /* Elimination Phase */
  /*********************/

  /**********************/
  /* Block Distribution */
  /**********************/
  //Unrolled loop: it does two iterations (stages) in the body
  for j in 1..RedistStage-1 by 2 do {
    //  Odd stage (j)
    ComputeStage(AA,BB,CC,DD,A,B,C,D,j,Dom);
    //  Even stage (j+1)
    ComputeStage(A,B,C,D,AA,BB,CC,DD,j+1,Dom);
  }

  if(RedistStage&1){ //If odd number of stages there is one more odd stage missing
    ComputeStage(AA,BB,CC,DD,A,B,C,D,RedistStage,Dom);
  }

  /********************************************/
  /* Change from Block to Cyclic Distribution */
  /********************************************/
  if timer then t5=getCurrentTime();
  if (RedistStage&1) then {copyBtoC(AA,P);copyBtoC(BB,Q);copyBtoC(CC,R);copyBtoC(DD,S);}
  else {copyBtoC(A,P);copyBtoC(B,Q);copyBtoC(C,R);copyBtoC(D,S);}
  if timer then t6=getCurrentTime();

  /********************************************/
  /*          Cyclic Distribution             */
  /********************************************/

  for j in (RedistStage+1)..stages-1 by 2 do {
    //  Stage (j)
    ComputeStage(PP,QQ,RR,SS,P,Q,R,S,j,DomC);
    //  Stage (j+1)
    ComputeStage(P,Q,R,S,PP,QQ,RR,SS,j+1,DomC);
  }

  if((stages-RedistStage)&1){ //If odd number of remaining stages there is one more stage missing
    ComputeStage(PP,QQ,RR,SS,P,Q,R,S,stages,DomC);
  }
  /**********************/
  /* Substitution Phase */
  /**********************/
  if((stages-RedistStage)&1){ //If odd number of final stages, diagonal is in SS and RHS in QQ
    forall (x,s,q) in zip(X,SS,QQ) do x=s/q;
  }
  else{ //else, diagonal is in S and RHS in Q
    forall (x,s,q) in zip(X,S,Q) do x=s/q;
  }

  if timer then t2=getCurrentTime();

  //writeln("Tridiagonal System Solution:");
  //PrintV(X);
  SetExampleMatrix();
  //PrintV(D);
  Check();

  if(error!=1)
    {
      writeln("Done!");
      if timer then {
	//writeln("Stages breakdown time (s) --> ", CalcTime);
	writeln("Time from Block D. to Cyclic D.: ",t6-t5);
	writeln("Stages computing time --> ", TotCalcTime,"sec.");
	writeln("Time elapsed : ", t2-t1," seconds" );
      }
    }
  else
    writeln("WRONG!!");
} //End Main

proc Check()
{
  var ERROR: real = 0.00001;
  XX=X; // A block distributed version of X (better for matrix vector multiplication)
  forall i in Dom with (ref error)
    {
      if(i==1){
	if (abs(B[i]*XX[i] + C[i]*XX[i+1] - D[i]) > ERROR)
	  { writeln("Case 1 failed"); error = 1;}}
      else if(i==n){
	if (abs(A[i]*XX[i-1] + B[i]*XX[i] - D[i]) > ERROR)
	  { writeln("Case 2 failed"); error = 1;}}
      else{
	if(abs(A[i]*XX[i-1] + B[i]*XX[i] + C[i]*XX[i+1]-D[i])>ERROR)
	  { writeln("Case 3 failed"); error = 1;}}
    }
}

proc ComputeStage(A,B,C,D,AA,BB,CC,DD,j,Dom, msg="")
{
  //  writeln("ComputeStage", msg, " j=", j);
  if timer then t3=getCurrentTime();
  const TtS:int=1<<(j-1); //// TtS stand for "Two to the Stage (minus 1)" which is = 2**(j-1)
  forall i in Dom do{
    //writeln("i ",i, " j ",j);
    const lo=i-TtS;
    const hi=i+TtS;
    if lo<1 then{
      if hi>n then{ // Case 1: out of lower and upper range
	A(i)=0;
	B(i)=BB(i);
	C(i)=0;
	D(i)=DD(i);
      } else { // Case 2: out of lower range
	const c=CC(i),bh=BB(hi);
	A(i)=0;
	B(i)=(BB(i) - (c*AA(hi)/bh));
	C(i)=(-c*CC(hi))/bh;
	D(i)=DD(i) - (DD(hi)*c)/bh;
      }
    }else {
      if hi>n then{ // Case 3: out of upper range
	const a=AA(i),bl=BB(lo);
	A(i)=(-AA(lo)*a)/bl;
	B(i)=(BB(i) - (CC(lo)*a/bl));
	C(i)=0;
	D(i)=DD(i) - (a*DD(lo))/bl;
      } else { // Case 4: no equation out of range
	const a=AA(i),bh=BB(hi),bl=BB(lo),c=CC(i);
	A(i)=(-AA(lo)*a)/bl;
	B(i)=(BB(i) - (CC(lo)*a/bl) - (c*AA(hi)/bh));
	C(i)=(-c*CC(hi))/bh;
	D(i)=DD(i) - (a*DD(lo))/bl - (DD(hi)*c)/bh;
      }//if
    }//if
  }//for all i
  if timer then {
    t4=getCurrentTime();
    CalcTime[j]=t4-t3;
    TotCalcTime+=CalcTime[j];
  }
}

proc PrintV(X)
{
  for i in DomC do{
    writeln("X[",i,"] = ",X(i));
  }
}

proc SetExampleMatrix()
{
  forall (i) in Dom{
    A(i)=1.0;
    B(i)=2.0;
    C(i)=1.0;
  }

  A(1)=0;C(n)=0;
  forall i in 1..(n+1)/2 do {
    D(i)=i;
    D(n-i+1)=i;
  }

  //writeln("Example matrix generated");
  //  writeln("D=",D);

}

proc GetNumStages(n: int)
{
  if n-2**log2(n) != 0 then
    return (log2(n)+1);
  else
    return log2(n);
}

proc copyBtoC(A:[], B:[])
{
  A._value.copyBtoC(B);
}

proc copyCtoB(A:[], B:[])
{
  A._value.copyCtoB(B);
}

proc BlockArr.copyBtoC(B)
{
  coforall loc in Locales do on loc
  {
    param stridelevels=1;
    var dststrides:[1..#stridelevels] int(32);
    var srcstrides: [1..#stridelevels] int(32);
    var count: [1..#(stridelevels+1)] int(32);
    var lid=loc.id;

    var numLocales: int(32)=dom.dist.targetLocDom.dim(1).length:int(32);
    var n:int(32)=dom.dist.boundingBox.dim(1).length:int(32);
    var src = locArr[lid].myElems._value.theData;

    dststrides[1]=1;
    srcstrides[1]=numLocales;

    var dststr=dststrides._value.theData;
    var srcstr=srcstrides._value.theData;
    var cnt=count._value.theData;

    //Domain size (n) and first index (arrayini)

    var arrayini:int(32)=dom.dsiLow:int(32);

    //writeln("Domain: ",dom.whole.dims());

    //a,b: first and last global indices in each locale
    var a: int(32)=dom.locDoms[lid].myBlock.low:int(32);
    var b: int(32)=dom.locDoms[lid].myBlock.high:int(32);
    var blksize=b-a+1;
    //writeln("Locale", here.id," : blksize ",blksize," subblock first index  a ",a,

    var t1,t2: real;
    var schunkini: int;
    var chunksize: int(32);
    //Loop of strided chunk sends (chpl_comm_put_strd)
    for k in 0..#numLocales do {

      var dst=(lid+k)%numLocales;

      //Compute the chunk first index in src locale to be send to locale dst.
      if (dst-a%numLocales+arrayini)<0 then schunkini=a+numLocales +(dst-a%numLocales+arrayini);
      else schunkini=a+(dst-a%numLocales+arrayini)%numLocales;

      //Compute chunk size
      if (schunkini+(blksize/numLocales)*numLocales>b) then chunksize=blksize/numLocales;
      else chunksize=blksize/numLocales+1;

      //var destr = privB.locArr[dst].myElems._value.theData;
      var destr = B._value.locArr[dst].myElems._value.theData;
      count[1]=1;
      count[2]=chunksize;

      __primitive("chpl_comm_put_strd",
		  __primitive("array_get",destr,
			      B._value.locArr[dst].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",dststr,dststrides._value.getDataIndex(1)),
		  dst,
		  __primitive("array_get",src,
			      locArr[lid].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",srcstr,srcstrides._value.getDataIndex(1)),
		  __primitive("array_get",cnt, count._value.getDataIndex(1)),
		  stridelevels);

    } // end for dst
  } // end coforall loc
} // end proc

proc  BlockArr.copyCtoB(B)
{

  coforall loc in Locales do on loc
  {
    param stridelevels=1;
    var dststrides:[1..#stridelevels] int(32);
    var srcstrides: [1..#stridelevels] int(32);
    var count: [1..#(stridelevels+1)] int(32);
    var lid=loc.id;
    var numLocales: int=dom.dist.targetLocDom.dim(1).length;
    var n:int(32)=dom.dist.boundingBox.dim(1).length:int(32);

    var dststr=dststrides._value.theData;
    var srcstr=srcstrides._value.theData;
    var cnt=count._value.theData;

    //On each locale (src) we compute the chunk that goes to each dst
    var num: int;
    var schunkini: int;
    var chunksize: int;
    var a: int(32)=dom.locDoms[lid].myBlock.low:int(32);
    var b: int(32)=dom.locDoms[lid].myBlock.high:int(32);
    num=b-a+1;

    var src = locArr[lid].myElems._value.theData;
    var arrayini:int(32)=dom.dsiLow:int(32);

    var t,t1,t2: real;
    t=0.0;
    //Compute the starting index of the src chunk with destination dst

    for dst in 0..#numLocales do {
      if (dst-a%numLocales+arrayini)<0 then schunkini=a+numLocales+(dst-a%numLocales+arrayini);
      else schunkini=a+(dst-a%numLocales+arrayini)%numLocales;

      //Compute the size of the chunk. The number of elements of src subarray is num
      if (schunkini+(num/numLocales)*numLocales>b) then chunksize=num/numLocales;
      else chunksize=num/numLocales+1;

      var destr = B._value.locArr[dst].myElems._value.theData;
      dststrides[1]=numLocales:int(32);
      srcstrides[1]=1;
      count[1]=1;
      count[2]=chunksize:int(32);

      __primitive("chpl_comm_get_strd",
		  __primitive("array_get",src,
			        locArr[lid].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",dststr,dststrides._value.getDataIndex(1)),
		  dst,
		  __primitive("array_get",destr,
			      B._value.locArr[dst].myElems._value.getDataIndex(schunkini)),
		  __primitive("array_get",srcstr,srcstrides._value.getDataIndex(1)),
		  __primitive("array_get",cnt,count._value.getDataIndex(1)),
		  stridelevels);
    } //end for dst
  }//end for loc
} //end proc copyCtoB

/*********************************************************/
/*		ParaCR algorithm in Chapel               */
/*		Block to cyclic redistribution           */
/*	Loop unrolled to avoid a deep copy of arrays     */
/* Contributed by Juan Lopez, Alberto Sanz & Rafa Asenjo */
/*********************************************************/

//This code solves a dense tridiagonal system following the PARACR algorithm

use Time;
use BlockDist;
use CyclicDist;

//Problem Dimension
config const n:int=1024;

const stages:int = GetNumStages(n);
//writeln("Stages->",stages);

//At the end of stage "RedistStage" it will change from Block to Cyclic Distribution
config const RedistStage:int=log2(numLocales);
//writeln("Stage to redistribute->",RedistStage);

if (RedistStage>stages)
{
  writeln("Not supported: your matrix dimension is smaller than the number of locales");
  exit(0);
}

var error:int=0;

const Space = {1..n};
const Dom = Space dmapped Block(boundingBox=Space);
const DomC = Space dmapped Cyclic(startIdx=(1));

var Dstages: domain(1,int)={1..stages};
var AA,BB,CC,DD, XX:[Dom] real;
var A,B,C,D:[Dom] real;

var P,Q,R,S, X:[DomC] real;
var PP,QQ,RR,SS:[DomC] real;

config const timer : bool =false;

//Timing variables
var t1,t2,t3,t4,t5,t6 :real;
var CalcTime:[Dstages] real=0.0;
var TotCalcTime: real=0.0;

proc main(){

  /* Parallel Cyclic reduction Algorithm */
  SetExampleMatrix();

  if(n==1) {
    X(1)=D(1)/B(1);
    writeln("Done!!. Too easy... bye");
    exit(0);
  }

  if timer then t1=getCurrentTime();

  /*********************/
  /* Elimination Phase */
  /*********************/

  /**********************/
  /* Block Distribution */
  /**********************/
  //Unrolled loop: it does two iterations (stages) in the body
  for j in 1..RedistStage-1 by 2 do {
    //  Odd stage (j)
    ComputeStage(AA,BB,CC,DD,A,B,C,D,j,Dom);
    //  Even stage (j+1)
    ComputeStage(A,B,C,D,AA,BB,CC,DD,j+1,Dom);
  }

  if(RedistStage&1){ //If odd number of stages there is one more odd stage missing
    ComputeStage(AA,BB,CC,DD,A,B,C,D,RedistStage,Dom);
  }

  /********************************************/
  /* Change from Block to Cyclic Distribution */
  /********************************************/
  if timer then t5=getCurrentTime();
  if (RedistStage&1) then {copyBtoC(AA,P);copyBtoC(BB,Q);copyBtoC(CC,R);copyBtoC(DD,S);}
  else {copyBtoC(A,P);copyBtoC(B,Q);copyBtoC(C,R);copyBtoC(D,S);}
  if timer then t6=getCurrentTime();

  /********************************************/
  /*          Cyclic Distribution             */
  /********************************************/

  for j in (RedistStage+1)..stages-1 by 2 do {
    //  Stage (j)
    ComputeStage(PP,QQ,RR,SS,P,Q,R,S,j,DomC);
    //  Stage (j+1)
    ComputeStage(P,Q,R,S,PP,QQ,RR,SS,j+1,DomC);
  }

  if((stages-RedistStage)&1){ //If odd number of remaining stages there is one more stage missing
    ComputeStage(PP,QQ,RR,SS,P,Q,R,S,stages,DomC);
  }
  /**********************/
  /* Substitution Phase */
  /**********************/
  if((stages-RedistStage)&1){ //If odd number of final stages, diagonal is in SS and RHS in QQ
    forall (x,s,q) in zip(X,SS,QQ) do x=s/q;
  }
  else{ //else, diagonal is in S and RHS in Q
    forall (x,s,q) in zip(X,S,Q) do x=s/q;
  }

  if timer then t2=getCurrentTime();

  //writeln("Tridiagonal System Solution:");
  //PrintV(X);
  SetExampleMatrix();
  //PrintV(D);
  Check();

  if(error!=1)
    {
      writeln("Done!");
      if timer then {
	//writeln("Stages breakdown time (s) --> ", CalcTime);
	writeln("Time from Block D. to Cyclic D.: ",t6-t5);
	writeln("Stages computing time --> ", TotCalcTime,"sec.");
	writeln("Time elapsed : ", t2-t1," seconds" );
      }
    }
  else
    writeln("WRONG!!");
} //End Main

proc Check()
{
  var ERROR: real = 0.00001;
  XX=X; // A block distributed version of X (better for matrix vector multiplication)
  forall i in Dom with (ref error)
    {
      if(i==1){
	if (abs(B[i]*XX[i] + C[i]*XX[i+1] - D[i]) > ERROR)
	  { writeln("Case 1 failed"); error = 1;}}
      else if(i==n){
	if (abs(A[i]*XX[i-1] + B[i]*XX[i] - D[i]) > ERROR)
	  { writeln("Case 2 failed"); error = 1;}}
      else{
	if(abs(A[i]*XX[i-1] + B[i]*XX[i] + C[i]*XX[i+1]-D[i])>ERROR)
	  { writeln("Case 3 failed"); error = 1;}}
    }
}

proc ComputeStage(A,B,C,D,AA,BB,CC,DD,j,Dom, msg="")
{
  //  writeln("ComputeStage", msg, " j=", j);
  if timer then t3=getCurrentTime();
  const TtS:int=1<<(j-1); //// TtS stand for "Two to the Stage (minus 1)" which is = 2**(j-1)
  forall i in Dom do{
    //writeln("i ",i, " j ",j);
    const lo=i-TtS;
    const hi=i+TtS;
    if lo<1 then{
      if hi>n then{ // Case 1: out of lower and upper range
	A(i)=0;
	B(i)=BB(i);
	C(i)=0;
	D(i)=DD(i);
      } else { // Case 2: out of lower range
	const c=CC(i),bh=BB(hi);
	A(i)=0;
	B(i)=(BB(i) - (c*AA(hi)/bh));
	C(i)=(-c*CC(hi))/bh;
	D(i)=DD(i) - (DD(hi)*c)/bh;
      }
    }else {
      if hi>n then{ // Case 3: out of upper range
	const a=AA(i),bl=BB(lo);
	A(i)=(-AA(lo)*a)/bl;
	B(i)=(BB(i) - (CC(lo)*a/bl));
	C(i)=0;
	D(i)=DD(i) - (a*DD(lo))/bl;
      } else { // Case 4: no equation out of range
	const a=AA(i),bh=BB(hi),bl=BB(lo),c=CC(i);
	A(i)=(-AA(lo)*a)/bl;
	B(i)=(BB(i) - (CC(lo)*a/bl) - (c*AA(hi)/bh));
	C(i)=(-c*CC(hi))/bh;
	D(i)=DD(i) - (a*DD(lo))/bl - (DD(hi)*c)/bh;
      }//if
    }//if
  }//for all i
  if timer then {
    t4=getCurrentTime();
    CalcTime[j]=t4-t3;
    TotCalcTime+=CalcTime[j];
  }
}

proc PrintV(X)
{
  for i in DomC do{
    writeln("X[",i,"] = ",X(i));
  }
}

proc SetExampleMatrix()
{
  forall (i) in Dom{
    A(i)=1.0;
    B(i)=2.0;
    C(i)=1.0;
  }

  A(1)=0;C(n)=0;
  forall i in 1..(n+1)/2 do {
    D(i)=i;
    D(n-i+1)=i;
  }

  //writeln("Example matrix generated");
  //  writeln("D=",D);

}

proc GetNumStages(n: int)
{
  if n-2**log2(n) != 0 then
    return (log2(n)+1);
  else
    return log2(n);
}