Modified the follower in this benchmark to assume that the follower is a
1-tuple of a range rather than just a range by indexing into the 1-tuple.  This
should get this ra.chpl test up and running.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15997 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// A helper module for the RA benchmark that defines the random stream
// of values
//
module RARandomStream {
  param randWidth = 64;              // the bit-width of the random numbers
  type randType = uint(randWidth);   // the type of the random numbers

  //
  // bitDom is a non-distributed domain whose indices correspond to
  // the bit positions in the random values.  m2 is a table of helper
  // values used to fast-forward through the random stream.
  //
  const bitDom = [0..#randWidth],
        m2: [bitDom] randType = computeM2Vals(randWidth);

  //
  // A serial iterator for the random stream that resets the stream
  // to its 0th element and yields values endlessly.
  //
  def RAStream() {
    var val = getNthRandom(0);
    while (1) {
      getNextRandom(val);
      yield val;
    }
  }

  //
  // A "follower" iterator for the random stream that takes a range of
  // 0-based indices (follower) and yields the pseudo-random values
  // corresponding to those indices.  Follower iterators like these
  // are required for parallel zippered iteration.
  //
  def RAStream(param tag: iterator, follower) where tag == iterator.follower {
    var val = getNthRandom(follower(1).low);
    for follower {
      getNextRandom(val);
      yield val;
    }
  }

  //
  // A helper function for "fast-forwarding" the random stream to
  // position n in O(log2(n)) time
  //
  def getNthRandom(in n: uint(64)) {
    param period = 0x7fffffffffffffff/7;

    n %= period;
    if (n == 0) then return 0x1;
    var ran: randType = 0x2;
    for i in 0..log2(n)-1 by -1 {
      var val: randType = 0;
      for j in bitDom do
        if ((ran >> j) & 1) then val ^= m2(j);
      ran = val;
      if ((n >> i) & 1) then getNextRandom(ran);
    }
    return ran;
  }

  //
  // A helper function for advancing a value from the random stream,
  // x, to the next value
  //
  def getNextRandom(inout x) {
    param POLY = 0x7;
    param hiRandBit = 0x1:randType << (randWidth-1);

    x = (x << 1) ^ (if (x & hiRandBit) then POLY else 0);
  }

  //
  // A helper function for computing the values of the helper array,
  // m2
  //
  def computeM2Vals(numVals) {
    var nextVal = 0x1: randType;
    for i in 1..numVals {
      yield nextVal;
      getNextRandom(nextVal);
      getNextRandom(nextVal);
    }
  }
}

//
// A helper module for the RA benchmark that defines the random stream
// of values
//
module RARandomStream {
  param randWidth = 64;              // the bit-width of the random numbers
  type randType = uint(randWidth);   // the type of the random numbers

  //
  // bitDom is a non-distributed domain whose indices correspond to
  // the bit positions in the random values.  m2 is a table of helper
  // values used to fast-forward through the random stream.
  //
  const bitDom = [0..#randWidth],
        m2: [bitDom] randType = computeM2Vals(randWidth);

  //
  // A serial iterator for the random stream that resets the stream
  // to its 0th element and yields values endlessly.
  //
  def RAStream() {
    var val = getNthRandom(0);
    while (1) {
      getNextRandom(val);
      yield val;
    }
  }

  //
  // A "follower" iterator for the random stream that takes a range of
  // 0-based indices (follower) and yields the pseudo-random values
  // corresponding to those indices.  Follower iterators like these
  // are required for parallel zippered iteration.
  //
  def RAStream(param tag: iterator, follower) where tag == iterator.follower {
    var val = getNthRandom(follower.low);
    for follower {
      getNextRandom(val);
      yield val;
    }
  }

  //
  // A helper function for "fast-forwarding" the random stream to
  // position n in O(log2(n)) time
  //
  def getNthRandom(in n: uint(64)) {
    param period = 0x7fffffffffffffff/7;

    n %= period;
    if (n == 0) then return 0x1;
    var ran: randType = 0x2;
    for i in 0..log2(n)-1 by -1 {
      var val: randType = 0;
      for j in bitDom do
        if ((ran >> j) & 1) then val ^= m2(j);
      ran = val;
      if ((n >> i) & 1) then getNextRandom(ran);
    }
    return ran;
  }

  //
  // A helper function for advancing a value from the random stream,
  // x, to the next value
  //
  def getNextRandom(inout x) {
    param POLY = 0x7;
    param hiRandBit = 0x1:randType << (randWidth-1);

    x = (x << 1) ^ (if (x & hiRandBit) then POLY else 0);
  }

  //
  // A helper function for computing the values of the helper array,
  // m2
  //
  def computeM2Vals(numVals) {
    var nextVal = 0x1: randType;
    for i in 1..numVals {
      yield nextVal;
      getNextRandom(nextVal);
      getNextRandom(nextVal);
    }
  }
}


starting to remove mains

starting to remove mains

module PeelPropagation {
  use Core,
      LinearAlgebra;

  config const epochs: int,      // Dr. Peel calls this "itr"
               alpha: real,      // damping factor
               steps: int,       // how many steps out to propagate
               epsilon: real;    // convergence criteria

  class PeelPropagationModel {
    var ldom: domain(1),          // As long as the number of labels
        dataDom: domain(2),
        D: [dataDom] real,        // Diagonal matrix
        Lnorm: [dataDom] real,    // normalized matrix
        Predictions: LabelMatrix = new LabelMatrix(), // The output of "fit()"
        compiled: bool =false;

    proc fit(X: [] real, L: LabelMatrix ) {
      dataDom = X.domain;
      if !compiled {
        compile(X, L);
      }
      // Now proceed to do explicitly one calculation
      return iterate(L);
    }
    proc compile(X:[] real, L: LabelMatrix) {
      // @TODO make this simpler.
      var rowSums: [X.domain.dim(2)] real;
      var sqs: [X.domain.dim(2)] real;
      [i in X.domain.dim(2)] rowSums[i] = + reduce X[i,..];
      [i in X.domain.dim(2)] sqs[i] = (rowSums[i]+1e-05)** -.5;
      D = diag(sqs);
      Lnorm = dot(D, dot(X,D));
      compiled = true;
    }

    /*
     Runs the iterations and returns the predictions
     */
    proc iterate(L: LabelMatrix) {
      var F: [L.data.domain] real = L.data,      // The current prediction, called F by Dr. Peel
          Fold: [L.data.domain] real = -1.0;
      var err: real = 1.0;
      var e = 0;
      do {
        for s in 1..steps {
          F = dot(Lnorm.T, F);
        }
        F = (1-alpha) * L.data + alpha * F;
        err = + reduce abs(matMinus(F, Fold));
        e += 1;
        Fold = F;
        //writeln("  epoch %n  error %n".format(e, err));
      } while (e < epochs && err > epsilon);
      Predictions.dataDom = L.dataDom;
      Predictions.trainingLabelDom = L.trainingLabelDom;
      [i in L.ldom] Predictions.data[i,argmax(F[i,..])] = 1;
      return Predictions;
    }
  }

  /*
   Calculates error on unlabled data
   */
  proc calculateError(Ygold: LabelMatrix, Ypred: LabelMatrix) {
    var mm: [Ypred.dataDom] real = abs(matMinus(Ygold.data, Ypred.data));
    var e = 0.0;
    for s in Ypred.trainingLabelDom {
      e += + reduce mm[s,..];
    }
    e = 1 - e / Ypred.trainingLabelDom.size;
    return e;
  }
}


Let domain keep track of the array

Let domain keep track of the array

// Wrapper around the pych_array c-struct
// to access NumPy arrays provided by the PyChapel
// Python module
use Random;

param PYCH_MAXDIM: int(32)= 16;

extern record pych_array {
    var two:int(32);
    var nd:int(32);
    var typekind:int(8);
    var itemsize:int(32);
    var flags:int(32);
    var shape:PYCH_MAXDIM*int(32);
    var strides:PYCH_MAXDIM*int(32);
    var ptr_d:_ddata(real);
}

proc pprint_pych_array(arr: pych_array) {
    writeln("[");
    writeln("  two=", arr.two, ",");
    writeln("  nd=", arr.nd, ",");
    writeln("  typekind=", arr.typekind, ",");
    writeln("  itemsize=", arr.itemsize, ",");

    write("  shape=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.shape(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    write("  stride=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.strides(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    writeln("  ptr_d=", arr.ptr_d);
    writeln("]");
}

proc rangify(shape) where isTuple(shape) {
    proc tail(first, remainder...) {
        return remainder;
    }
    if (shape.size == 1) {
        return (0..shape(1):int(64),);
    } else {
        return ((0..shape(1):int(64)), (...rangify(tail((...shape)))));
    }
}

pragma "modifies array blk"
proc pych_to_chpl1D(arr: pych_array) {
    var dom = {0..(arr.shape(1):int(64)-1)};

    var stride = (1,);
    var block = (1,);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    dom._value.add_arr(ret);

    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        //ret.str(i) = (arr.strides(i):int(64) / (arr.itemsize));
        ret.str(i) = 1;
    }
    //writeln("Strides");
    //for str in ret.str {
    //    writeln(str);
    //}
    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        // The block seems to be what NumPy call strides.
        ret.blk(i) = (arr.strides(i):int(64) / arr.itemsize);
    }
    //ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();
    ret.initShiftedData();

    return ret;
}

pragma "modifies array blk"
proc pych_to_chpl2D(arr: pych_array) {

    var dom = {(...rangify(arr.shape))};

    //writeln(pych_to_chplT(arr));
    var stride = (1, 1);
    var block = (3, 1);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    dom._value.add_arr(ret);

    // rank*int tuples

    // stride for each dimension
    // you might be able to get this out of the domain instead.
    //ret.str = stride;

    for i in 1..arr.nd {
        ret.str(i) = arr.strides(i):int(64);
    }

    // I'm not totally sure what this one is. I think it's the size of a row of data.
    // You can test this by writing code like:
    //var x = [1..10, 10..20]: int;
    //writeln("What is this?");

    ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();

    // if you're having segfaults, look into this method. You can also try
    // compiling with '-searlyShiftData=false' to disable this method.
    ret.initShiftedData();

    // only returns the underlying class, still need to turn it into a
    // first-class chapel array. I'm not sure why exactly, but something
    // about the way we return arrays here performs a copy and changes
    // made to the array aren't reflected in the c pointer.
    return ret;
}

// Wrapper around the pych_array c-struct
// to access NumPy arrays provided by the PyChapel
// Python module
use Random;

param PYCH_MAXDIM: int(32)= 16;

extern record pych_array {
    var two:int(32);
    var nd:int(32);
    var typekind:int(8);
    var itemsize:int(32);
    var flags:int(32);
    var shape:PYCH_MAXDIM*int(32);
    var strides:PYCH_MAXDIM*int(32);
    var ptr_d:_ddata(real);
}

proc pprint_pych_array(arr: pych_array) {
    writeln("[");
    writeln("  two=", arr.two, ",");
    writeln("  nd=", arr.nd, ",");
    writeln("  typekind=", arr.typekind, ",");
    writeln("  itemsize=", arr.itemsize, ",");

    write("  shape=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.shape(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    write("  stride=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.strides(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    writeln("  ptr_d=", arr.ptr_d);
    writeln("]");
}

proc rangify(shape) where isTuple(shape) {
    proc tail(first, remainder...) {
        return remainder;
    }
    if (shape.size == 1) {
        return (0..shape(1):int(64),);
    } else {
        return ((0..shape(1):int(64)), (...rangify(tail((...shape)))));
    }
}

pragma "modifies array blk"
proc pych_to_chpl1D(arr: pych_array) {
    var dom = {0..(arr.shape(1):int(64)-1)};

    var stride = (1,);
    var block = (1,);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        //ret.str(i) = (arr.strides(i):int(64) / (arr.itemsize));
        ret.str(i) = 1;
    }
    //writeln("Strides");
    //for str in ret.str {
    //    writeln(str);
    //}
    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        // The block seems to be what NumPy call strides.
        ret.blk(i) = (arr.strides(i):int(64) / arr.itemsize);
    }
    //ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();
    ret.initShiftedData();

    return ret;
}

pragma "modifies array blk"
proc pych_to_chpl2D(arr: pych_array) {

    var dom = {(...rangify(arr.shape))};

    //writeln(pych_to_chplT(arr));
    var stride = (1, 1);
    var block = (3, 1);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    // rank*int tuples

    // stride for each dimension
    // you might be able to get this out of the domain instead.
    //ret.str = stride;

    for i in 1..arr.nd {
        ret.str(i) = arr.strides(i):int(64);
    }

    // I'm not totally sure what this one is. I think it's the size of a row of data.
    // You can test this by writing code like:
    //var x = [1..10, 10..20]: int;
    //writeln("What is this?");

    ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();

    // if you're having segfaults, look into this method. You can also try
    // compiling with '-searlyShiftData=false' to disable this method.
    ret.initShiftedData();

    // only returns the underlying class, still need to turn it into a
    // first-class chapel array. I'm not sure why exactly, but something
    // about the way we return arrays here performs a copy and changes
    // made to the array aren't reflected in the c pointer.
    return ret;
}


Really fix the Cyclic distribution this time.

This commit changes another couple of uses of low/high to first/last.
Again, these are in "intermediate" ranges that are used to define the
iteration space.

This should fix the Cyclic hpl regression.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@18650 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use DSIUtil;

proc _determineRankFromStartIdx(startIdx) param {
  return if isTuple(startIdx) then startIdx.size else 1;
}

proc _determineIdxTypeFromStartIdx(startIdx) type {
  return if isTuple(startIdx) then startIdx(1).type else startIdx.type;
}

config param debugCyclicDist = false;
config param verboseCyclicDistWriters = false;

//
// If the testFastFollowerOptimization flag is set to true, the
// follower will write output to indicate whether the fast follower is
// used or not.  This is used in regression testing to ensure that the
// 'fast follower' optimization is working.
//
config param testFastFollowerOptimization = false;

class Cyclic: BaseDist {
  param rank: int;
  type idxType = int;

  var startIdx: rank*idxType;
  var targetLocDom: domain(rank);
  var targetLocs: [targetLocDom] locale;

  var locDist: [targetLocDom] LocCyclic(rank, idxType);

  var dataParTasksPerLocale: int;
  var dataParIgnoreRunningTasks: bool;
  var dataParMinGranularity: int;

  var pid: int = -1;

  proc Cyclic(startIdx,
             targetLocales: [] locale = thisRealm.Locales,
             dataParTasksPerLocale=getDataParTasksPerLocale(),
             dataParIgnoreRunningTasks=getDataParIgnoreRunningTasks(),
             dataParMinGranularity=getDataParMinGranularity(),
             param rank: int = _determineRankFromStartIdx(startIdx),
             type idxType = _determineIdxTypeFromStartIdx(startIdx))
    where isTuple(startIdx) || _isIntegralType(startIdx.type) {
    var tupleStartIdx: rank*idxType;
    if isTuple(startIdx) then tupleStartIdx = startIdx;
                         else tupleStartIdx(1) = startIdx;
    if rank == 1  {
      targetLocDom = [0..#targetLocales.numElements];
      targetLocs = targetLocales;
    } else if targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank {
        ranges(i) = 0..#factors(i);
      }
      targetLocDom = [(...ranges)];
      for (loc1, loc2) in (targetLocs, targetLocales) {
        loc1 = loc2;
      }
    } else {
      if targetLocales.rank != rank then
        compilerError("locales array rank must be one or match distribution rank");
      var ranges: rank*range;
      for param i in 1..rank do {
        var thisRange = targetLocales.domain.dim(i);
        ranges(i) = 0..#thisRange.length;
      }
      targetLocDom = [(...ranges)];
      targetLocs = reshape(targetLocales, targetLocDom);
    }

    for param i in 1..rank do
      this.startIdx(i) = chpl__mod(tupleStartIdx(i), targetLocDom.dim(i).length);

    // NOTE: When these knobs stop using the global defaults, we will need
    // to add checks to make sure dataParTasksPerLocale<0 and
    // dataParMinGranularity<0
    this.dataParTasksPerLocale = if dataParTasksPerLocale==0 then here.numCores
                                 else dataParTasksPerLocale;
    this.dataParIgnoreRunningTasks = dataParIgnoreRunningTasks;
    this.dataParMinGranularity = dataParMinGranularity;

    coforall locid in targetLocDom {
      on targetLocs(locid) {
        locDist(locid) = new LocCyclic(rank, idxType, locid, this);
      }
    }
    if debugCyclicDist then
      for loc in locDist do writeln(loc);
  }

  proc Cyclic(param rank, type idxType, other: Cyclic(rank, idxType)) {
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    startIdx = other.startIdx;
    locDist = other.locDist;
    dataParTasksPerLocale = other.dataParTasksPerLocale;
    dataParIgnoreRunningTasks = other.dataParIgnoreRunningTasks;
    dataParMinGranularity = other.dataParMinGranularity;
  }

  proc dsiAssign(other: this.type) {
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        delete locDist(locid);
    startIdx = other.startIdx;
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    dataParTasksPerLocale = other.dataParTasksPerLocale;
    dataParIgnoreRunningTasks = other.dataParIgnoreRunningTasks;
    dataParMinGranularity = other.dataParMinGranularity;
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocCyclic(rank, idxType, locid, this);
  }

  proc dsiClone() return new Cyclic(rank=rank, idxType=idxType, other=this);
}

proc Cyclic.getChunk(inds, locid) {
  var sliceBy: rank*range(idxType=idxType, stridable=true);
  var locidtup: rank*int;
  if rank == 1 then
    locidtup(1) = locid;
  else
    locidtup = locid;
  for param i in 1..rank {
    var distStride = targetLocDom.dim(i).length;
    var loclowidx = chpl__mod(startIdx(i) + locidtup(i), distStride);
    var lowmod = chpl__mod(inds.dim(i).low, distStride);
    var offset = loclowidx - lowmod;
    if offset < 0 then
      sliceBy(i) = inds.dim(i).low + (distStride + offset)..inds.dim(i).high by distStride;
    else
      sliceBy(i) = inds.dim(i).low + offset..inds.dim(i).high by distStride;
    // remove alignment
    sliceBy(i) = sliceBy(i).first..sliceBy(i).last by distStride;
  }
  return inds((...sliceBy));
  //
  // WANT:
  //var distWhole = locDist(locid).myChunk.getIndices();
  //return inds((...distWhole));
}

proc Cyclic.dsiDisplayRepresentation() {
  writeln("startIdx = ", startIdx);
  writeln("targetLocDom = ", targetLocDom);
  writeln("targetLocs = ", for tl in targetLocs do tl.id);
  writeln("dataParTasksPerLocale = ", dataParTasksPerLocale);
  writeln("dataParIgnoreRunningTasks = ", dataParIgnoreRunningTasks);
  writeln("dataParMinGranularity = ", dataParMinGranularity);
  for tli in targetLocDom do
    writeln("locDist[", tli, "].myChunk = ", locDist[tli].myChunk);
}

proc Cyclic.dsiSupportsPrivatization() param return true;

proc Cyclic.dsiGetPrivatizeData() return 0;

proc Cyclic.dsiPrivatize(privatizeData) {
  return new Cyclic(rank=rank, idxType=idxType, other=this);
}

proc Cyclic.dsiGetReprivatizeData() return 0;

proc Cyclic.dsiReprivatize(other, reprivatizeData) {
  targetLocDom = other.targetLocDom;
  targetLocs = other.targetLocs;
  locDist = other.locDist;
  startIdx = other.startIdx;
  dataParTasksPerLocale = other.dataParTasksPerLocale;
  dataParIgnoreRunningTasks = other.dataParIgnoreRunningTasks;
  dataParMinGranularity = other.dataParMinGranularity;
}

proc Cyclic.dsiNewRectangularDom(param rank: int, type idxType, param stridable: bool) {
  if idxType != this.idxType then
    compilerError("Cyclic domain index type does not match distribution's");
  if rank != this.rank then
    compilerError("Cyclic domain rank does not match distribution's");
  var dom = new CyclicDom(rank=rank, idxType=idxType, dist = this, stridable=stridable);
  dom.setup();
  return dom;
}

proc Cyclic.dsiCreateReindexDist(newSpace, oldSpace) {
  proc anyStridable(space, param i=1) param
    return if i == space.size
      then space(i).stridable
      else space(i).stridable || anyStridable(space, i+1);

  if anyStridable(newSpace) || anyStridable(oldSpace) then
    compilerWarning("reindexing stridable Cyclic arrays is not yet fully supported");

  var newLow: rank*idxType;
  for param i in 1..rank {
    newLow(i) = newSpace(i).low - oldSpace(i).low + startIdx(i);
  }
  var newDist = new Cyclic(rank=rank, idxType=idxType, startIdx=newLow,
                           targetLocales=targetLocs,
                           dataParTasksPerLocale=dataParTasksPerLocale,
                           dataParIgnoreRunningTasks=dataParIgnoreRunningTasks,
                           dataParMinGranularity=dataParMinGranularity);
  return newDist;
}

//
// Given a tuple of scalars of type t or range(t) match the shape but
// using types rangeType and scalarType e.g. the call:
// _matchArgsShape(range(int), int, (1:int(64), 1:int(64)..5, 1:int(64)..5))
// returns the type: (int, range(int), range(int))
//
proc _cyclic_matchArgsShape(type rangeType, type scalarType, args) type {
  proc tuple(type t ...) type return t;
  proc helper(param i: int) type {
    if i == args.size {
      if isCollapsedDimension(args(i)) then
        return tuple(scalarType);
      else
        return tuple(rangeType);
    } else {
      if isCollapsedDimension(args(i)) then
        return (scalarType, (... helper(i+1)));
      else
        return (rangeType, (... helper(i+1)));
    }
  }
  return helper(1);
}

proc Cyclic.dsiCreateRankChangeDist(param newRank: int, args) {
  var collapsedDimInds: rank*idxType;
  var collapsedLocs: _cyclic_matchArgsShape(range(int), int, args);
  var newLow: newRank*idxType;

  var j: int = 1;
  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) then
      collapsedDimInds(i) = args(i);
    else {
      newLow(j) = startIdx(i);
      j += 1;
    }
  }
  const partialLocIdx = idxToLocaleInd(collapsedDimInds);

  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) {
      collapsedLocs(i) = partialLocIdx(i);
} else {
      collapsedLocs(i) = targetLocDom.dim(i);
    }
  }
  var newTargetLocales = targetLocs[(...collapsedLocs)];
  return new Cyclic(rank=newRank, idxType=idxType, startIdx=newLow, targetLocales=newTargetLocales);
}

proc Cyclic.writeThis(x: Writer) {
  x.writeln(typeToString(this.type));
  x.writeln("------");
  for locid in targetLocDom do
    x.writeln(" [", locid, "=", targetLocs(locid), "] owns chunk: ", locDist(locid).myChunk);
}

proc Cyclic.idxToLocaleInd(i: idxType) {
  const numLocs:idxType = targetLocDom.numIndices:idxType;
  // this is wrong if i is less than startIdx
  //return ((i - startIdx(1)) % numLocs):int;
  // this works even if i is less than startIdx
  return chpl__diffMod(i, startIdx(1), numLocs):int;
}

proc Cyclic.idxToLocaleInd(ind: rank*idxType) {
  var x: rank*int;
  for param i in 1..rank {
    var dimLen = targetLocDom.dim(i).length;
    //x(i) = ((ind(i) - startIdx(i)) % dimLen):int;
    x(i) = chpl__diffMod(ind(i), startIdx(i), dimLen):int;
  }
  if rank == 1 then
    return x(1);
  else
    return x;
}

proc Cyclic.dsiIndexLocale(i: idxType) where rank == 1 {
  return targetLocs(idxToLocaleInd(i));
}

proc Cyclic.dsiIndexLocale(i: rank*idxType) {
  return targetLocs(idxToLocaleInd(i));
}

class LocCyclic {
  param rank: int;
  type idxType;

  const myChunk: domain(rank, idxType, true);

  proc LocCyclic(param rank, type idxType, locid, dist: Cyclic(rank, idxType)) {
    var locidx: rank*int;
    var startIdx = dist.startIdx;

    if rank == 1 then
      locidx(1) = locid;
    else
      locidx = locid;

    var tuple: rank*range(idxType, stridable=true);

    for param i in 1..rank {
      const lower = min(idxType)..(startIdx(i)+locidx(i)) by -dist.targetLocDom.dim(i).length;
      const upper = (startIdx(i) + locidx(i))..max(idxType) by dist.targetLocDom.dim(i).length;
      const lo = lower.last, hi = upper.last;
      tuple(i) = lo..hi by dist.targetLocDom.dim(i).length;
    }
    myChunk = [(...tuple)];
  }
}

class CyclicDom : BaseRectangularDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  const dist: Cyclic(rank, idxType);

  var locDoms: [dist.targetLocDom] LocCyclicDom(rank, idxType, stridable);

  const whole: domain(rank, idxType, stridable);

  var pid: int = -1;
}

proc CyclicDom.setup() {
  if locDoms(dist.targetLocDom.low) == nil {
    coforall localeIdx in dist.targetLocDom {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocCyclicDom(rank, idxType, stridable, dist.getChunk(whole, localeIdx));
    }
  } else {
    coforall localeIdx in dist.targetLocDom {
      on dist.targetLocs(localeIdx) {
        var chunk = dist.getChunk(whole, localeIdx);
        locDoms(localeIdx).myBlock = chunk;
      }
    }
  }
}

proc CyclicDom.dsiBuildArray(type eltType) {
  var arr = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this);
  arr.setup();
  return arr;
}

proc CyclicDom.dsiDisplayRepresentation() {
  writeln("whole = ", whole);
  for tli in dist.targetLocDom do
    writeln("locDoms[", tli, "].myBlock = ", locDoms[tli].myBlock);
  dist.dsiDisplayRepresentation();
}

proc CyclicDom.dsiLow return whole.low;

proc CyclicDom.dsiHigh return whole.high;

proc CyclicDom.dsiStride return whole.stride;

proc CyclicDom.dsiMember(i) return whole.member(i);

proc CyclicDom.dsiIndexOrder(i) return whole.indexOrder(i);

proc CyclicDom.dsiDims() return whole.dims();

proc CyclicDom.dsiDim(d: int) return whole.dim(d);

proc CyclicDom.getLocDom(localeIdx) return locDoms(localeIdx);

proc CyclicDom.dsiMyDist() return dist;

proc CyclicDom.dsiGetIndices() {
  return whole.getIndices();
}

proc CyclicDom.dsiSetIndices(x: domain) {
  whole = x;
  setup();
}

proc CyclicDom.dsiSetIndices(x) {
  whole.setIndices(x);
  setup();
}

proc CyclicDom.dsiSerialWrite(x: Writer) {
  if verboseCyclicDistWriters {
    x.writeln(typeToString(this.type));
    x.writeln("------");
    for loc in dist.targetLocDom {
      x.writeln("[", loc, "=", dist.targetLocs(loc), "] owns ", locDoms(loc).myBlock);
    }
  } else {
    x.write(whole);
  }
}

proc CyclicDom.dsiNumIndices return whole.numIndices;

iter CyclicDom.these() {
  for i in whole do
    yield i;
}

iter CyclicDom.these(param tag: iterator) where tag == iterator.leader {
  const maxTasks = dist.dataParTasksPerLocale;
  const ignoreRunning = dist.dataParIgnoreRunningTasks;
  const minSize = dist.dataParMinGranularity;
  const wholeLow = whole.low;
  coforall locDom in locDoms do on locDom {
    const (numTasks, parDim) = _computeChunkStuff(maxTasks, ignoreRunning,
                                                  minSize,
                                                  locDom.myBlock.dims());

    var result: rank*range(idxType=idxType, stridable=true);
    // Use the internal function for untranslate to avoid having to do
    // extra work to negate the offset
    var zeroedLocalPart = whole((...locDom.myBlock.getIndices())).chpl__unTranslate(wholeLow);
    for param i in 1..rank {
      var dim = zeroedLocalPart.dim(i);
      var wholestride = whole.dim(i).stride;
      if dim.last >= dim.first then
        result(i) = (dim.first / wholestride)..(dim.last / wholestride) by (dim.stride / wholestride);
      else
        result(i) = 1..0 by 1;
    }
    if numTasks == 1 {
      if debugCyclicDist then
        writeln(here.id, ": leader whole: ", whole,
                         " result: ", result,
                         " myblock: ", locDom.myBlock);
      yield result;
    } else {

      coforall taskid in 0:uint(64)..#numTasks {
        var splitRanges: rank*range(idxType=idxType, stridable=true) = result;
        const low = result(parDim).first, high = result(parDim).high;
        const (lo,hi) = _computeBlock(high - low + 1, numTasks, taskid,
                                      high, low, low);
        splitRanges(parDim) = result(parDim)(lo..hi);
        if debugCyclicDist then
          writeln(here.id, ": leader whole: ", whole,
                           " result: ", result,
                           " splitRanges: ", splitRanges);
        // remove alignment
        splitRanges(parDim) = splitRanges(parDim).first..splitRanges(parDim).last by splitRanges(parDim).stride;
        yield splitRanges;
      }
    }
  }
}

iter CyclicDom.these(param tag: iterator, follower) where tag == iterator.follower {
  var t: rank*range(idxType, stridable=true);
  if debugCyclicDist then
    writeln(here.id, ": follower whole is: ", whole,
                     " follower is: ", follower);
  for param i in 1..rank {
    const wholestride = whole.dim(i).stride;
    t(i) = ((follower(i).low*wholestride)..(follower(i).high*wholestride) by (follower(i).stride*wholestride)) + whole.dim(i).low;
  }
  if debugCyclicDist then
    writeln(here.id, ": follower maps to: ", t);
  for i in [(...t)] do
    yield i;
}

proc CyclicDom.dsiSupportsPrivatization() param return true;

proc CyclicDom.dsiGetPrivatizeData() return 0;

proc CyclicDom.dsiPrivatize(privatizeData) {
  var privdist = chpl_getPrivatizedCopy(dist.type, dist.pid);
  var c = new CyclicDom(rank=rank, idxType=idxType, stridable=stridable, dist=privdist);
  c.locDoms = locDoms;
  c.whole = whole;
  return c;
}

proc CyclicDom.dsiGetReprivatizeData() return 0;

proc CyclicDom.dsiReprivatize(other, reprivatizeData) {
  locDoms = other.locDoms;
  whole = other.whole;
}

proc CyclicDom.dsiBuildRectangularDom(param rank, type idxType,
                                    param stridable: bool,
                                    ranges: rank*range(idxType,
                                                       BoundedRangeType.bounded,
                                                       stridable)) {
  if idxType != dist.idxType then
    compilerError("Cyclic domain index type does not match distribution's");
  if rank != dist.rank then
    compilerError("Cyclic domain rank does not match distribution's");

  var dom = new CyclicDom(rank=rank, idxType=idxType,
                         dist=dist, stridable=stridable);
  dom.dsiSetIndices(ranges);
  return dom;

}

proc CyclicDom.dsiLocalSlice(param stridable: bool, ranges) {
  return whole((...ranges));
}

class LocCyclicDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  var myBlock: domain(rank, idxType, true);
}

//
// Added as a performance stopgap to avoid returning a domain
//
proc LocCyclicDom.member(i) return myBlock.member(i);

class CyclicArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;
  var dom: CyclicDom(rank, idxType, stridable);

  var locArr: [dom.dist.targetLocDom] LocCyclicArr(eltType, rank, idxType, stridable);
  var myLocArr: LocCyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable);
  var pid: int = -1;
}

proc CyclicArr.dsiSlice(d: CyclicDom) {
  var alias = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType,
                            stridable=d.stridable, dom=d, pid=pid);
  var thisid = this.locale.uid;
  for i in dom.dist.targetLocDom {
    on dom.dist.targetLocs(i) {
      alias.locArr[i] =
        new LocCyclicArr(eltType=eltType, rank=rank, idxType=idxType,
                         stridable=d.stridable, locDom=d.locDoms[i],
                         myElems=>locArr[i].myElems[d.locDoms[i].myBlock]);
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }
  return alias;
}

proc CyclicArr.dsiRankChange(d, param newRank: int, param stridable: bool, args) {
  var alias = new CyclicArr(eltType=eltType, rank=newRank, idxType=idxType, stridable=stridable, dom = d);
  var thisid = this.locale.uid;
  coforall ind in d.dist.targetLocDom {
    on d.dist.targetLocs(ind) {
      const locDom = d.getLocDom(ind);
      var collapsedDims: rank*idxType;
      var locSlice: _cyclic_matchArgsShape(range(idxType=idxType, stridable=true), idxType, args);
      var locArrInd: rank*int;
      var j = 1;
      for param i in 1..args.size {
        if isCollapsedDimension(args(i)) {
          locSlice(i) = args(i);
          collapsedDims(i) = args(i);
        } else {
          locSlice(i) = locDom.myBlock.dim(j)(args(i));
          j += 1;
        }
      }
      locArrInd = dom.dist.idxToLocaleInd(collapsedDims);
      j = 1;
      // Now that the locArrInd values are known for the collapsed dimensions
      // Pull the rest of the dimensions values from ind
      for param i in 1..args.size {
        if !isCollapsedDimension(args(i)) {
          if newRank > 1 then
            locArrInd(i) = ind(j);
          else
            locArrInd(i) = ind;
          j += 1;
        }
      }
      alias.locArr[ind] =
        new LocCyclicArr(eltType=eltType, rank=newRank, idxType=d.idxType,
                         stridable=d.stridable, locDom=locDom,
                         myElems=>locArr[(...locArrInd)].myElems[(...locSlice)]);
      if here.uid == thisid then
        alias.myLocArr = alias.locArr[ind];
    }
  }
  return alias;
}

proc CyclicArr.dsiReindex(d: CyclicDom) {
  var alias = new CyclicArr(eltType=eltType, rank=rank, idxType=d.idxType,
                            stridable=d.stridable, dom=d);
  var thisid = this.locale.uid;
  coforall i in dom.dist.targetLocDom {
    on dom.dist.targetLocs(i) {
      const locDom = d.getLocDom(i);
      const locAlias: [locDom.myBlock] => locArr[i].myElems;
      alias.locArr[i] = new LocCyclicArr(eltType=eltType, idxType=idxType,
                                         locDom=locDom, stridable=d.stridable,
                                         rank=rank, myElems=>locAlias);
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }
  return alias;
}

proc CyclicArr.dsiLocalSlice(ranges) {
  var low: rank*idxType;
  for param i in 1..rank {
    low(i) = ranges(i).low;
  }

  var A => locArr(dom.dist.idxToLocaleInd(low)).myElems((...ranges));
  return A;
}

proc CyclicArr.dsiDisplayRepresentation() {
  for tli in dom.dist.targetLocDom do
    writeln("locArr[", tli, "].myElems = ", for e in locArr[tli].myElems do e);
  dom.dsiDisplayRepresentation();
}

proc CyclicArr.dsiGetBaseDom() return dom;

proc CyclicArr.setup() {
  coforall localeIdx in dom.dist.targetLocDom {
    on dom.dist.targetLocs(localeIdx) {
      locArr(localeIdx) = new LocCyclicArr(eltType, rank, idxType, stridable, dom.locDoms(localeIdx));
      if this.locale == here then
        myLocArr = locArr(localeIdx);
    }
  }
}

proc CyclicArr.dsiSupportsPrivatization() param return true;

proc CyclicArr.dsiGetPrivatizeData() return 0;

proc CyclicArr.dsiPrivatize(privatizeData) {
  var privdom = chpl_getPrivatizedCopy(dom.type, dom.pid);
  var c = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
  c.locArr = locArr;
  for localeIdx in dom.dist.targetLocDom do
    if c.locArr(localeIdx).locale == here then
      c.myLocArr = c.locArr(localeIdx);
  return c;
}

proc CyclicArr.dsiAccess(i: idxType) var where rank == 1 {
  local {
    if myLocArr != nil && myLocArr.locDom.myBlock.member(i) then
      return myLocArr.this(i);
  }
  return locArr(dom.dist.idxToLocaleInd(i))(i);
}

proc CyclicArr.dsiAccess(i:rank*idxType) var {
  return locArr(dom.dist.idxToLocaleInd(i))(i);
}

iter CyclicArr.these() var {
  for i in dom do
    yield dsiAccess(i);
}

iter CyclicArr.these(param tag: iterator) where tag == iterator.leader {
  for follower in dom.these(tag) do
    yield follower;
}

proc CyclicArr.dsiStaticFastFollowCheck(type leadType) param
  return leadType == this.type || leadType == this.dom.type;

proc CyclicArr.dsiDynamicFastFollowCheck(lead: [])
  return lead.domain._value == this.dom;

proc CyclicArr.dsiDynamicFastFollowCheck(lead: domain)
  return lead._value == this.dom;

iter CyclicArr.these(param tag: iterator, follower, param fast: bool = false) var where tag == iterator.follower {
  if testFastFollowerOptimization then
    writeln((if fast then "fast" else "regular") + " follower invoked for Cyclic array");

  var t: rank*range(idxType=idxType, stridable=true);
  for param i in 1..rank {
    const wholestride = dom.whole.dim(i).stride;
    t(i) = ((follower(i).low*wholestride)..(follower(i).high*wholestride) by (follower(i).stride*wholestride)) + dom.whole.dim(i).low;
  }
  const followThis = [(...t)];
  if fast {
    const arrSection = locArr(dom.dist.idxToLocaleInd(followThis.low));
    if arrSection.locale.uid == here.uid then local {
      for e in arrSection.myElems(followThis) do
        yield e;
    } else {
      for e in arrSection.myElems(followThis) do
        yield e;
    }
  } else {
    proc accessHelper(i) var {
      if myLocArr then local {
        if myLocArr.locDom.member(i) then
          return myLocArr.this(i);
      }
      return dsiAccess(i);
    }

    for i in followThis {
      yield accessHelper(i);
    }
  }
}

proc CyclicArr.dsiSerialWrite(f: Writer) {
  if verboseCyclicDistWriters {
    writeln(typeToString(this.type));
    writeln("------");
  }
  if dom.dsiNumIndices == 0 then return;
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dsiDim(dim).low;
  label next while true {
    f.write(dsiAccess(i));
    if i(rank) <= (dom.dsiDim(rank).high - dom.dsiDim(rank).stride:idxType) {
      f.write(" ");
      i(rank) += dom.dsiDim(rank).stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dsiDim(dim).high - dom.dsiDim(dim).stride:idxType) {
          i(dim) += dom.dsiDim(dim).stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dsiDim(dim2).low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

proc CyclicArr.dsiReallocate(d: domain) {
  // The reallocation happens when the LocCyclicDom.myBlock field is changed
  // in CyclicDom.setup(). Nothing more needs to happen here.
}

class LocCyclicArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  const locDom: LocCyclicDom(rank, idxType, stridable);

  var myElems: [locDom.myBlock] eltType;

}

proc LocCyclicArr.this(i:idxType) var {
  return myElems(i);
}

proc LocCyclicArr.this(i:rank*idxType) var {
  return myElems((...i));
}

iter LocCyclicArr.these() var {
  for elem in myElems {
    yield elem;
  }
}

use DSIUtil;

proc _determineRankFromStartIdx(startIdx) param {
  return if isTuple(startIdx) then startIdx.size else 1;
}

proc _determineIdxTypeFromStartIdx(startIdx) type {
  return if isTuple(startIdx) then startIdx(1).type else startIdx.type;
}

config param debugCyclicDist = false;
config param verboseCyclicDistWriters = false;

//
// If the testFastFollowerOptimization flag is set to true, the
// follower will write output to indicate whether the fast follower is
// used or not.  This is used in regression testing to ensure that the
// 'fast follower' optimization is working.
//
config param testFastFollowerOptimization = false;

class Cyclic: BaseDist {
  param rank: int;
  type idxType = int;

  var startIdx: rank*idxType;
  var targetLocDom: domain(rank);
  var targetLocs: [targetLocDom] locale;

  var locDist: [targetLocDom] LocCyclic(rank, idxType);

  var dataParTasksPerLocale: int;
  var dataParIgnoreRunningTasks: bool;
  var dataParMinGranularity: int;

  var pid: int = -1;

  proc Cyclic(startIdx,
             targetLocales: [] locale = thisRealm.Locales,
             dataParTasksPerLocale=getDataParTasksPerLocale(),
             dataParIgnoreRunningTasks=getDataParIgnoreRunningTasks(),
             dataParMinGranularity=getDataParMinGranularity(),
             param rank: int = _determineRankFromStartIdx(startIdx),
             type idxType = _determineIdxTypeFromStartIdx(startIdx))
    where isTuple(startIdx) || _isIntegralType(startIdx.type) {
    var tupleStartIdx: rank*idxType;
    if isTuple(startIdx) then tupleStartIdx = startIdx;
                         else tupleStartIdx(1) = startIdx;
    if rank == 1  {
      targetLocDom = [0..#targetLocales.numElements];
      targetLocs = targetLocales;
    } else if targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank {
        ranges(i) = 0..#factors(i);
      }
      targetLocDom = [(...ranges)];
      for (loc1, loc2) in (targetLocs, targetLocales) {
        loc1 = loc2;
      }
    } else {
      if targetLocales.rank != rank then
        compilerError("locales array rank must be one or match distribution rank");
      var ranges: rank*range;
      for param i in 1..rank do {
        var thisRange = targetLocales.domain.dim(i);
        ranges(i) = 0..#thisRange.length;
      }
      targetLocDom = [(...ranges)];
      targetLocs = reshape(targetLocales, targetLocDom);
    }

    for param i in 1..rank do
      this.startIdx(i) = chpl__mod(tupleStartIdx(i), targetLocDom.dim(i).length);

    // NOTE: When these knobs stop using the global defaults, we will need
    // to add checks to make sure dataParTasksPerLocale<0 and
    // dataParMinGranularity<0
    this.dataParTasksPerLocale = if dataParTasksPerLocale==0 then here.numCores
                                 else dataParTasksPerLocale;
    this.dataParIgnoreRunningTasks = dataParIgnoreRunningTasks;
    this.dataParMinGranularity = dataParMinGranularity;

    coforall locid in targetLocDom {
      on targetLocs(locid) {
        locDist(locid) = new LocCyclic(rank, idxType, locid, this);
      }
    }
    if debugCyclicDist then
      for loc in locDist do writeln(loc);
  }

  proc Cyclic(param rank, type idxType, other: Cyclic(rank, idxType)) {
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    startIdx = other.startIdx;
    locDist = other.locDist;
    dataParTasksPerLocale = other.dataParTasksPerLocale;
    dataParIgnoreRunningTasks = other.dataParIgnoreRunningTasks;
    dataParMinGranularity = other.dataParMinGranularity;
  }

  proc dsiAssign(other: this.type) {
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        delete locDist(locid);
    startIdx = other.startIdx;
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    dataParTasksPerLocale = other.dataParTasksPerLocale;
    dataParIgnoreRunningTasks = other.dataParIgnoreRunningTasks;
    dataParMinGranularity = other.dataParMinGranularity;
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocCyclic(rank, idxType, locid, this);
  }

  proc dsiClone() return new Cyclic(rank=rank, idxType=idxType, other=this);
}

proc Cyclic.getChunk(inds, locid) {
  var sliceBy: rank*range(idxType=idxType, stridable=true);
  var locidtup: rank*int;
  if rank == 1 then
    locidtup(1) = locid;
  else
    locidtup = locid;
  for param i in 1..rank {
    var distStride = targetLocDom.dim(i).length;
    var loclowidx = chpl__mod(startIdx(i) + locidtup(i), distStride);
    var lowmod = chpl__mod(inds.dim(i).low, distStride);
    var offset = loclowidx - lowmod;
    if offset < 0 then
      sliceBy(i) = inds.dim(i).low + (distStride + offset)..inds.dim(i).high by distStride;
    else
      sliceBy(i) = inds.dim(i).low + offset..inds.dim(i).high by distStride;
    // remove alignment
    sliceBy(i) = sliceBy(i).first..sliceBy(i).last by distStride;
  }
  return inds((...sliceBy));
  //
  // WANT:
  //var distWhole = locDist(locid).myChunk.getIndices();
  //return inds((...distWhole));
}

proc Cyclic.dsiDisplayRepresentation() {
  writeln("startIdx = ", startIdx);
  writeln("targetLocDom = ", targetLocDom);
  writeln("targetLocs = ", for tl in targetLocs do tl.id);
  writeln("dataParTasksPerLocale = ", dataParTasksPerLocale);
  writeln("dataParIgnoreRunningTasks = ", dataParIgnoreRunningTasks);
  writeln("dataParMinGranularity = ", dataParMinGranularity);
  for tli in targetLocDom do
    writeln("locDist[", tli, "].myChunk = ", locDist[tli].myChunk);
}

proc Cyclic.dsiSupportsPrivatization() param return true;

proc Cyclic.dsiGetPrivatizeData() return 0;

proc Cyclic.dsiPrivatize(privatizeData) {
  return new Cyclic(rank=rank, idxType=idxType, other=this);
}

proc Cyclic.dsiGetReprivatizeData() return 0;

proc Cyclic.dsiReprivatize(other, reprivatizeData) {
  targetLocDom = other.targetLocDom;
  targetLocs = other.targetLocs;
  locDist = other.locDist;
  startIdx = other.startIdx;
  dataParTasksPerLocale = other.dataParTasksPerLocale;
  dataParIgnoreRunningTasks = other.dataParIgnoreRunningTasks;
  dataParMinGranularity = other.dataParMinGranularity;
}

proc Cyclic.dsiNewRectangularDom(param rank: int, type idxType, param stridable: bool) {
  if idxType != this.idxType then
    compilerError("Cyclic domain index type does not match distribution's");
  if rank != this.rank then
    compilerError("Cyclic domain rank does not match distribution's");
  var dom = new CyclicDom(rank=rank, idxType=idxType, dist = this, stridable=stridable);
  dom.setup();
  return dom;
}

proc Cyclic.dsiCreateReindexDist(newSpace, oldSpace) {
  proc anyStridable(space, param i=1) param
    return if i == space.size
      then space(i).stridable
      else space(i).stridable || anyStridable(space, i+1);

  if anyStridable(newSpace) || anyStridable(oldSpace) then
    compilerWarning("reindexing stridable Cyclic arrays is not yet fully supported");

  var newLow: rank*idxType;
  for param i in 1..rank {
    newLow(i) = newSpace(i).low - oldSpace(i).low + startIdx(i);
  }
  var newDist = new Cyclic(rank=rank, idxType=idxType, startIdx=newLow,
                           targetLocales=targetLocs,
                           dataParTasksPerLocale=dataParTasksPerLocale,
                           dataParIgnoreRunningTasks=dataParIgnoreRunningTasks,
                           dataParMinGranularity=dataParMinGranularity);
  return newDist;
}

//
// Given a tuple of scalars of type t or range(t) match the shape but
// using types rangeType and scalarType e.g. the call:
// _matchArgsShape(range(int), int, (1:int(64), 1:int(64)..5, 1:int(64)..5))
// returns the type: (int, range(int), range(int))
//
proc _cyclic_matchArgsShape(type rangeType, type scalarType, args) type {
  proc tuple(type t ...) type return t;
  proc helper(param i: int) type {
    if i == args.size {
      if isCollapsedDimension(args(i)) then
        return tuple(scalarType);
      else
        return tuple(rangeType);
    } else {
      if isCollapsedDimension(args(i)) then
        return (scalarType, (... helper(i+1)));
      else
        return (rangeType, (... helper(i+1)));
    }
  }
  return helper(1);
}

proc Cyclic.dsiCreateRankChangeDist(param newRank: int, args) {
  var collapsedDimInds: rank*idxType;
  var collapsedLocs: _cyclic_matchArgsShape(range(int), int, args);
  var newLow: newRank*idxType;

  var j: int = 1;
  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) then
      collapsedDimInds(i) = args(i);
    else {
      newLow(j) = startIdx(i);
      j += 1;
    }
  }
  const partialLocIdx = idxToLocaleInd(collapsedDimInds);

  for param i in 1..args.size {
    if isCollapsedDimension(args(i)) {
      collapsedLocs(i) = partialLocIdx(i);
} else {
      collapsedLocs(i) = targetLocDom.dim(i);
    }
  }
  var newTargetLocales = targetLocs[(...collapsedLocs)];
  return new Cyclic(rank=newRank, idxType=idxType, startIdx=newLow, targetLocales=newTargetLocales);
}

proc Cyclic.writeThis(x: Writer) {
  x.writeln(typeToString(this.type));
  x.writeln("------");
  for locid in targetLocDom do
    x.writeln(" [", locid, "=", targetLocs(locid), "] owns chunk: ", locDist(locid).myChunk);
}

proc Cyclic.idxToLocaleInd(i: idxType) {
  const numLocs:idxType = targetLocDom.numIndices:idxType;
  // this is wrong if i is less than startIdx
  //return ((i - startIdx(1)) % numLocs):int;
  // this works even if i is less than startIdx
  return chpl__diffMod(i, startIdx(1), numLocs):int;
}

proc Cyclic.idxToLocaleInd(ind: rank*idxType) {
  var x: rank*int;
  for param i in 1..rank {
    var dimLen = targetLocDom.dim(i).length;
    //x(i) = ((ind(i) - startIdx(i)) % dimLen):int;
    x(i) = chpl__diffMod(ind(i), startIdx(i), dimLen):int;
  }
  if rank == 1 then
    return x(1);
  else
    return x;
}

proc Cyclic.dsiIndexLocale(i: idxType) where rank == 1 {
  return targetLocs(idxToLocaleInd(i));
}

proc Cyclic.dsiIndexLocale(i: rank*idxType) {
  return targetLocs(idxToLocaleInd(i));
}

class LocCyclic {
  param rank: int;
  type idxType;

  const myChunk: domain(rank, idxType, true);

  proc LocCyclic(param rank, type idxType, locid, dist: Cyclic(rank, idxType)) {
    var locidx: rank*int;
    var startIdx = dist.startIdx;

    if rank == 1 then
      locidx(1) = locid;
    else
      locidx = locid;

    var tuple: rank*range(idxType, stridable=true);

    for param i in 1..rank {
      const lower = min(idxType)..(startIdx(i)+locidx(i)) by -dist.targetLocDom.dim(i).length;
      const upper = (startIdx(i) + locidx(i))..max(idxType) by dist.targetLocDom.dim(i).length;
      const lo = lower.last, hi = upper.last;
      tuple(i) = lo..hi by dist.targetLocDom.dim(i).length;
    }
    myChunk = [(...tuple)];
  }
}

class CyclicDom : BaseRectangularDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  const dist: Cyclic(rank, idxType);

  var locDoms: [dist.targetLocDom] LocCyclicDom(rank, idxType, stridable);

  const whole: domain(rank, idxType, stridable);

  var pid: int = -1;
}

proc CyclicDom.setup() {
  if locDoms(dist.targetLocDom.low) == nil {
    coforall localeIdx in dist.targetLocDom {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocCyclicDom(rank, idxType, stridable, dist.getChunk(whole, localeIdx));
    }
  } else {
    coforall localeIdx in dist.targetLocDom {
      on dist.targetLocs(localeIdx) {
        var chunk = dist.getChunk(whole, localeIdx);
        locDoms(localeIdx).myBlock = chunk;
      }
    }
  }
}

proc CyclicDom.dsiBuildArray(type eltType) {
  var arr = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this);
  arr.setup();
  return arr;
}

proc CyclicDom.dsiDisplayRepresentation() {
  writeln("whole = ", whole);
  for tli in dist.targetLocDom do
    writeln("locDoms[", tli, "].myBlock = ", locDoms[tli].myBlock);
  dist.dsiDisplayRepresentation();
}

proc CyclicDom.dsiLow return whole.low;

proc CyclicDom.dsiHigh return whole.high;

proc CyclicDom.dsiStride return whole.stride;

proc CyclicDom.dsiMember(i) return whole.member(i);

proc CyclicDom.dsiIndexOrder(i) return whole.indexOrder(i);

proc CyclicDom.dsiDims() return whole.dims();

proc CyclicDom.dsiDim(d: int) return whole.dim(d);

proc CyclicDom.getLocDom(localeIdx) return locDoms(localeIdx);

proc CyclicDom.dsiMyDist() return dist;

proc CyclicDom.dsiGetIndices() {
  return whole.getIndices();
}

proc CyclicDom.dsiSetIndices(x: domain) {
  whole = x;
  setup();
}

proc CyclicDom.dsiSetIndices(x) {
  whole.setIndices(x);
  setup();
}

proc CyclicDom.dsiSerialWrite(x: Writer) {
  if verboseCyclicDistWriters {
    x.writeln(typeToString(this.type));
    x.writeln("------");
    for loc in dist.targetLocDom {
      x.writeln("[", loc, "=", dist.targetLocs(loc), "] owns ", locDoms(loc).myBlock);
    }
  } else {
    x.write(whole);
  }
}

proc CyclicDom.dsiNumIndices return whole.numIndices;

iter CyclicDom.these() {
  for i in whole do
    yield i;
}

iter CyclicDom.these(param tag: iterator) where tag == iterator.leader {
  const maxTasks = dist.dataParTasksPerLocale;
  const ignoreRunning = dist.dataParIgnoreRunningTasks;
  const minSize = dist.dataParMinGranularity;
  const wholeLow = whole.low;
  coforall locDom in locDoms do on locDom {
    const (numTasks, parDim) = _computeChunkStuff(maxTasks, ignoreRunning,
                                                  minSize,
                                                  locDom.myBlock.dims());

    var result: rank*range(idxType=idxType, stridable=true);
    // Use the internal function for untranslate to avoid having to do
    // extra work to negate the offset
    var zeroedLocalPart = whole((...locDom.myBlock.getIndices())).chpl__unTranslate(wholeLow);
    for param i in 1..rank {
      var dim = zeroedLocalPart.dim(i);
      var wholestride = whole.dim(i).stride;
      if dim.high >= dim.low then
        result(i) = (dim.low / wholestride)..(dim.high / wholestride) by (dim.stride / wholestride);
      else
        result(i) = 1..0 by 1;
    }
    if numTasks == 1 {
      if debugCyclicDist then
        writeln(here.id, ": leader whole: ", whole,
                         " result: ", result,
                         " myblock: ", locDom.myBlock);
      yield result;
    } else {

      coforall taskid in 0:uint(64)..#numTasks {
        var splitRanges: rank*range(idxType=idxType, stridable=true) = result;
        const low = result(parDim).first, high = result(parDim).high;
        const (lo,hi) = _computeBlock(high - low + 1, numTasks, taskid,
                                      high, low, low);
        splitRanges(parDim) = result(parDim)(lo..hi);
        if debugCyclicDist then
          writeln(here.id, ": leader whole: ", whole,
                           " result: ", result,
                           " splitRanges: ", splitRanges);
        // remove alignment
        splitRanges(parDim) = splitRanges(parDim).first..splitRanges(parDim).last by splitRanges(parDim).stride;
        yield splitRanges;
      }
    }
  }
}

iter CyclicDom.these(param tag: iterator, follower) where tag == iterator.follower {
  var t: rank*range(idxType, stridable=true);
  if debugCyclicDist then
    writeln(here.id, ": follower whole is: ", whole,
                     " follower is: ", follower);
  for param i in 1..rank {
    const wholestride = whole.dim(i).stride;
    t(i) = ((follower(i).low*wholestride)..(follower(i).high*wholestride) by (follower(i).stride*wholestride)) + whole.dim(i).low;
  }
  if debugCyclicDist then
    writeln(here.id, ": follower maps to: ", t);
  for i in [(...t)] do
    yield i;
}

proc CyclicDom.dsiSupportsPrivatization() param return true;

proc CyclicDom.dsiGetPrivatizeData() return 0;

proc CyclicDom.dsiPrivatize(privatizeData) {
  var privdist = chpl_getPrivatizedCopy(dist.type, dist.pid);
  var c = new CyclicDom(rank=rank, idxType=idxType, stridable=stridable, dist=privdist);
  c.locDoms = locDoms;
  c.whole = whole;
  return c;
}

proc CyclicDom.dsiGetReprivatizeData() return 0;

proc CyclicDom.dsiReprivatize(other, reprivatizeData) {
  locDoms = other.locDoms;
  whole = other.whole;
}

proc CyclicDom.dsiBuildRectangularDom(param rank, type idxType,
                                    param stridable: bool,
                                    ranges: rank*range(idxType,
                                                       BoundedRangeType.bounded,
                                                       stridable)) {
  if idxType != dist.idxType then
    compilerError("Cyclic domain index type does not match distribution's");
  if rank != dist.rank then
    compilerError("Cyclic domain rank does not match distribution's");

  var dom = new CyclicDom(rank=rank, idxType=idxType,
                         dist=dist, stridable=stridable);
  dom.dsiSetIndices(ranges);
  return dom;

}

proc CyclicDom.dsiLocalSlice(param stridable: bool, ranges) {
  return whole((...ranges));
}

class LocCyclicDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  var myBlock: domain(rank, idxType, true);
}

//
// Added as a performance stopgap to avoid returning a domain
//
proc LocCyclicDom.member(i) return myBlock.member(i);

class CyclicArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;
  var dom: CyclicDom(rank, idxType, stridable);

  var locArr: [dom.dist.targetLocDom] LocCyclicArr(eltType, rank, idxType, stridable);
  var myLocArr: LocCyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable);
  var pid: int = -1;
}

proc CyclicArr.dsiSlice(d: CyclicDom) {
  var alias = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType,
                            stridable=d.stridable, dom=d, pid=pid);
  var thisid = this.locale.uid;
  for i in dom.dist.targetLocDom {
    on dom.dist.targetLocs(i) {
      alias.locArr[i] =
        new LocCyclicArr(eltType=eltType, rank=rank, idxType=idxType,
                         stridable=d.stridable, locDom=d.locDoms[i],
                         myElems=>locArr[i].myElems[d.locDoms[i].myBlock]);
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }
  return alias;
}

proc CyclicArr.dsiRankChange(d, param newRank: int, param stridable: bool, args) {
  var alias = new CyclicArr(eltType=eltType, rank=newRank, idxType=idxType, stridable=stridable, dom = d);
  var thisid = this.locale.uid;
  coforall ind in d.dist.targetLocDom {
    on d.dist.targetLocs(ind) {
      const locDom = d.getLocDom(ind);
      var collapsedDims: rank*idxType;
      var locSlice: _cyclic_matchArgsShape(range(idxType=idxType, stridable=true), idxType, args);
      var locArrInd: rank*int;
      var j = 1;
      for param i in 1..args.size {
        if isCollapsedDimension(args(i)) {
          locSlice(i) = args(i);
          collapsedDims(i) = args(i);
        } else {
          locSlice(i) = locDom.myBlock.dim(j)(args(i));
          j += 1;
        }
      }
      locArrInd = dom.dist.idxToLocaleInd(collapsedDims);
      j = 1;
      // Now that the locArrInd values are known for the collapsed dimensions
      // Pull the rest of the dimensions values from ind
      for param i in 1..args.size {
        if !isCollapsedDimension(args(i)) {
          if newRank > 1 then
            locArrInd(i) = ind(j);
          else
            locArrInd(i) = ind;
          j += 1;
        }
      }
      alias.locArr[ind] =
        new LocCyclicArr(eltType=eltType, rank=newRank, idxType=d.idxType,
                         stridable=d.stridable, locDom=locDom,
                         myElems=>locArr[(...locArrInd)].myElems[(...locSlice)]);
      if here.uid == thisid then
        alias.myLocArr = alias.locArr[ind];
    }
  }
  return alias;
}

proc CyclicArr.dsiReindex(d: CyclicDom) {
  var alias = new CyclicArr(eltType=eltType, rank=rank, idxType=d.idxType,
                            stridable=d.stridable, dom=d);
  var thisid = this.locale.uid;
  coforall i in dom.dist.targetLocDom {
    on dom.dist.targetLocs(i) {
      const locDom = d.getLocDom(i);
      const locAlias: [locDom.myBlock] => locArr[i].myElems;
      alias.locArr[i] = new LocCyclicArr(eltType=eltType, idxType=idxType,
                                         locDom=locDom, stridable=d.stridable,
                                         rank=rank, myElems=>locAlias);
      if thisid == here.uid then
        alias.myLocArr = alias.locArr[i];
    }
  }
  return alias;
}

proc CyclicArr.dsiLocalSlice(ranges) {
  var low: rank*idxType;
  for param i in 1..rank {
    low(i) = ranges(i).low;
  }

  var A => locArr(dom.dist.idxToLocaleInd(low)).myElems((...ranges));
  return A;
}

proc CyclicArr.dsiDisplayRepresentation() {
  for tli in dom.dist.targetLocDom do
    writeln("locArr[", tli, "].myElems = ", for e in locArr[tli].myElems do e);
  dom.dsiDisplayRepresentation();
}

proc CyclicArr.dsiGetBaseDom() return dom;

proc CyclicArr.setup() {
  coforall localeIdx in dom.dist.targetLocDom {
    on dom.dist.targetLocs(localeIdx) {
      locArr(localeIdx) = new LocCyclicArr(eltType, rank, idxType, stridable, dom.locDoms(localeIdx));
      if this.locale == here then
        myLocArr = locArr(localeIdx);
    }
  }
}

proc CyclicArr.dsiSupportsPrivatization() param return true;

proc CyclicArr.dsiGetPrivatizeData() return 0;

proc CyclicArr.dsiPrivatize(privatizeData) {
  var privdom = chpl_getPrivatizedCopy(dom.type, dom.pid);
  var c = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
  c.locArr = locArr;
  for localeIdx in dom.dist.targetLocDom do
    if c.locArr(localeIdx).locale == here then
      c.myLocArr = c.locArr(localeIdx);
  return c;
}

proc CyclicArr.dsiAccess(i: idxType) var where rank == 1 {
  local {
    if myLocArr != nil && myLocArr.locDom.myBlock.member(i) then
      return myLocArr.this(i);
  }
  return locArr(dom.dist.idxToLocaleInd(i))(i);
}

proc CyclicArr.dsiAccess(i:rank*idxType) var {
  return locArr(dom.dist.idxToLocaleInd(i))(i);
}

iter CyclicArr.these() var {
  for i in dom do
    yield dsiAccess(i);
}

iter CyclicArr.these(param tag: iterator) where tag == iterator.leader {
  for follower in dom.these(tag) do
    yield follower;
}

proc CyclicArr.dsiStaticFastFollowCheck(type leadType) param
  return leadType == this.type || leadType == this.dom.type;

proc CyclicArr.dsiDynamicFastFollowCheck(lead: [])
  return lead.domain._value == this.dom;

proc CyclicArr.dsiDynamicFastFollowCheck(lead: domain)
  return lead._value == this.dom;

iter CyclicArr.these(param tag: iterator, follower, param fast: bool = false) var where tag == iterator.follower {
  if testFastFollowerOptimization then
    writeln((if fast then "fast" else "regular") + " follower invoked for Cyclic array");

  var t: rank*range(idxType=idxType, stridable=true);
  for param i in 1..rank {
    const wholestride = dom.whole.dim(i).stride;
    t(i) = ((follower(i).low*wholestride)..(follower(i).high*wholestride) by (follower(i).stride*wholestride)) + dom.whole.dim(i).low;
  }
  const followThis = [(...t)];
  if fast {
    const arrSection = locArr(dom.dist.idxToLocaleInd(followThis.low));
    if arrSection.locale.uid == here.uid then local {
      for e in arrSection.myElems(followThis) do
        yield e;
    } else {
      for e in arrSection.myElems(followThis) do
        yield e;
    }
  } else {
    proc accessHelper(i) var {
      if myLocArr then local {
        if myLocArr.locDom.member(i) then
          return myLocArr.this(i);
      }
      return dsiAccess(i);
    }

    for i in followThis {
      yield accessHelper(i);
    }
  }
}

proc CyclicArr.dsiSerialWrite(f: Writer) {
  if verboseCyclicDistWriters {
    writeln(typeToString(this.type));
    writeln("------");
  }
  if dom.dsiNumIndices == 0 then return;
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dsiDim(dim).low;
  label next while true {
    f.write(dsiAccess(i));
    if i(rank) <= (dom.dsiDim(rank).high - dom.dsiDim(rank).stride:idxType) {
      f.write(" ");
      i(rank) += dom.dsiDim(rank).stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dsiDim(dim).high - dom.dsiDim(dim).stride:idxType) {
          i(dim) += dom.dsiDim(dim).stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dsiDim(dim2).low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

proc CyclicArr.dsiReallocate(d: domain) {
  // The reallocation happens when the LocCyclicDom.myBlock field is changed
  // in CyclicDom.setup(). Nothing more needs to happen here.
}

class LocCyclicArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  const locDom: LocCyclicDom(rank, idxType, stridable);

  var myElems: [locDom.myBlock] eltType;

}

proc LocCyclicArr.this(i:idxType) var {
  return myElems(i);
}

proc LocCyclicArr.this(i:rank*idxType) var {
  return myElems((...i));
}

iter LocCyclicArr.these() var {
  for elem in myElems {
    yield elem;
  }
}


Fixed bug in the Atomics primer

Fixed bug in the Atomics primer

Fixed a bug in the Atomics primer that inadvertently assumed that the
coforall tasks would run in order (who the heck wrote this thing!?!?!)

Tested in a loop for ~30 minutes (each test takes an almost
immeasurable amount time).

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@21578 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Atomics primer
//
// This primer illustrates Chapel's atomic variables.  For more information
// on Chapel's atomics, see $CHPL_HOME/docs/technotes/README.atomics
//
config const n = 31;
const R = 1..n;

// The 'atomic' keyword is a type qualifiers that can be applied to
// the following Chapel primitive types to declare atomic variables:
//
// - bool
// - int (all supported sizes)
// - uint (all supported sizes)
// - real (all supported sizes)
//
// Atomic variables support a limited number of operations that are
// currently implemented as methods.
//

var x: atomic int;

// Atomic variables can only be used or assigned via the read() and
// write() methods, respectively.
//

x.write(n);
if x.read() != n then
  halt("Error: x (", x.read(), ") != n (", n, ")");

// All atomic types support atomic exchange(),
// compareExchangeStrong(), and compareExchangeWeak(),
// compareExchange() (same as compareExchangeStrong()).
//
// The exchange() method atomically swaps the old value with the given
// argument and returns the old value.  The compareExchange(),
// compareExchangeStrong() and compareExchangeWeak() methods only
// perform the swap if the current value is equal to the first argument,
// returning true if the exchange was performed  See README.atomics for
// the difference between the weak and strong variety.
//
// In the following example, n parallel tasks are created via a
// coforall statement.  Each task tries to set the current value of x
// id-1, but only will succeed.
//
var numFound: atomic int;
coforall id in R {
  numFound.add(x.compareExchangeWeak(n, id-1));
}
if numFound.read() != 1 then
  halt("Error: numFound != 1 (", numFound.read(), ")");
if x.read() == n then
  halt("Error: x == n");

// As a convenience, atomic bools also support testAndSet() and clear().
// The testAndSet() method atomically reads the value of the variable, sets
// it to true, then returns the original value.  The clear() method
// atomically sets the value to false.
//
// In the following example, we create n tasks and each task calls
// testAndSet() on the atomic variable named flag and saves the
// result in a unique element of the result array.
//
var flag: atomic bool;
var result: [R] bool;
coforall r in result do
  r = flag.testAndSet();

// When the coforall is complete, only one of the above testAndSet()
// calls should have returned false.
//
var found = false;
for r in result {
  if !r then
    if found then
      halt("Error: found multiple times!");
    else
      found = true;
}
if !found then
  halt("Error: not found!");
flag.clear();

// The integral and real atomic types also support the following atomic
// fetch and non-fetching operations:
//
// - fetchAdd() and add()
// - fetchSub() and sub()
// - fetchOr() and or() (bit-wise) (integral types only)
// - fetchAnd() and and() (bit-wise) (integral types only)
//
// Each of the above atomically reads the variable, stores the result
// of the operation (+, -, |, or &) using the value and the method
// argument, then, for the fetchOps functions, returns the original
// value.
//
// In the following example, we create n tasks to atomically increment
// the atomic variable a with the square of the task's given id.
//
var a: atomic int;
coforall id in R do a.add(id*id);

// The sum of this finite series should be n(n+1)*(2n+1)/6
//
var expected = n*(n+1)*(2*n+1)/6;
if a.read() != expected then
  halt("Error: a=", a, " (should be ", expected, ")");

// In the following example, we create n tasks to atomically increment
// the atomic variable a with the square of the task's given id.  The
// sum of this finite series is n(n+1)*(2n+1)/6. If the returned value
// is the expected value less id*id (last task to arrive), check the
// results.
//
a.write(0);
const sumOfSq = n*(n+1)*(2*n+1)/6;
coforall id in R {
  const mySq = id*id;
  const last = a.fetchAdd(mySq);
  if sumOfSq-mySq == last {
    //
    const t = a.read();
    if t != n*(n+1)*(2*n+1)/6 then
      halt("Error: a=", t, " (should be ", sumOfSq, ") id=", id);
  }
}

//
// Atomics primer
//
// This primer illustrates Chapel's atomic variables.  For more information
// on Chapel's atomics, see $CHPL_HOME/docs/technotes/README.atomics
//
config const n = 31;
const R = 1..n;

// The 'atomic' keyword is a type qualifiers that can be applied to
// the following Chapel primitive types to declare atomic variables:
//
// - bool
// - int (all supported sizes)
// - uint (all supported sizes)
// - real (all supported sizes)
//
// Atomic variables support a limited number of operations that are
// currently implemented as methods.
//

var x: atomic int;

// Atomic variables can only be used or assigned via the read() and
// write() methods, respectively.
//

x.write(n);
if x.read() != n then
  halt("Error: x (", x.read(), ") != n (", n, ")");

// All atomic types support atomic exchange(),
// compareExchangeStrong(), and compareExchangeWeak(),
// compareExchange() (same as compareExchangeStrong()).
//
// The exchange() method atomically swaps the old value with the given
// argument and returns the old value.  The compareExchange(),
// compareExchangeStrong() and compareExchangeWeak() methods only
// perform the swap if the current value is equal to the first argument,
// returning true if the exchange was performed  See README.atomics for
// the difference between the weak and strong variety.
//
// In the following example, n parallel tasks are created via a
// coforall statement.  Each task tries to set the current value of x
// id-1, but only will succeed.
//
var numFound: atomic int;
coforall id in R {
  numFound.add(x.compareExchangeWeak(n, id-1));
}
if numFound.read() != 1 then
  halt("Error: numFound != 1 (", numFound.read(), ")");
if x.read() == n then
  halt("Error: x == n");

// As a convenience, atomic bools also support testAndSet() and clear().
// The testAndSet() method atomically reads the value of the variable, sets
// it to true, then returns the original value.  The clear() method
// atomically sets the value to false.
//
// In the following example, we create n tasks and each task calls
// testAndSet() on the atomic variable named flag and saves the
// result in a unique element of the result array.
//
var flag: atomic bool;
var result: [R] bool;
coforall r in result do
  r = flag.testAndSet();

// When the coforall is complete, only one of the above testAndSet()
// calls should have returned false.
//
var found = false;
for r in result {
  if !r then
    if found then
      halt("Error: found multiple times!");
    else
      found = true;
}
if !found then
  halt("Error: not found!");
flag.clear();

// The integral and real atomic types also support the following atomic
// fetch and non-fetching operations:
//
// - fetchAdd() and add()
// - fetchSub() and sub()
// - fetchOr() and or() (bit-wise) (integral types only)
// - fetchAnd() and and() (bit-wise) (integral types only)
//
// Each of the above atomically reads the variable, stores the result
// of the operation (+, -, |, or &) using the value and the method
// argument, then, for the fetchOps functions, returns the original
// value.
//
// In the following example, we create n tasks to atomically increment
// the atomic variable a with the square of the task's given id.
//
var a: atomic int;
coforall id in R do a.add(id*id);

// The sum of this finite series should be n(n+1)*(2n+1)/6
//
var expected = n*(n+1)*(2*n+1)/6;
if a.read() != expected then
  halt("Error: a=", a, " (should be", expected, ")");

// In the following example, we create n tasks to atomically increment
// the atomic variable a with the square of the task's given id.  If
// the returned value is (n-1)*(n)*(2*(n-1)+1)/6 (last task to
// arrive), check the results.
//
a.write(0);
coforall id in R {
  if a.fetchAdd(id*id)==(n-1)*(n)*(2*(n-1)+1)/6 {
    // The sum of this finite series should be n(n+1)*(2n+1)/6
    //
    if a.read() != n*(n+1)*(2*n+1)/6 then
      halt("Error: a=", a, " (should be", n*(n+1)*(2*n+1)/6, ")");
  }
}