Added getBaseDom() method to CyclicDist distribution test.  See r15724
for more info on this method.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15868 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module CyclicDistrib {
//
// naive routine for dividing numLocales into rank factors
//
def _factor(param rank: int, value) {
  var factors: rank*int;
  for param i in 1..rank do
    factors(i) = 1;
  if value >= 1 {
    var iv = value;
    var factor = 1;
    while iv > 1 {
      for i in 2..iv {
        if iv % i == 0 {
          var j = 1;
          for i in 2..rank {
            if factors(i) < factors(j) then
              j = i;
          }
          factors(j) *= i;
          iv = iv / i;
          break;
        }
      }
    }
  }
  for i in 1..rank do
    for j in i+1..rank do
      if factors(i) < factors(j) then
        factors(i) <=> factors(j);
  return factors;
}

class CyclicDist : BaseDist {
  param rank: int = 1;
  type idxType = int(64);

  const targetLocDom: domain(rank);
  const targetLocs: [targetLocDom] locale;

  const locDist: [targetLocDom] LocCyclicDist(rank, idxType);

  const tasksPerLoc = 1;

  def CyclicDist(param rank: int,
                 type idxType = int(64),
                 targetLocales: [] locale = Locales, tasksPerLocale=1) {
    if rank == 1  {
      targetLocDom = [0..#targetLocales.numElements];
      targetLocs = targetLocales;
    } else if targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank {
        ranges(i) = 0..#factors(i);
      }
      targetLocDom = [(...ranges)];
      for (loc1, loc2) in (targetLocs, targetLocales) {
        loc1 = loc2;
      }
    } else {
      if targetLocales.rank != rank then
        compilerError("locales array rank must be one or match distribution rank");
      halt("This case not handled yet.");
    }
    tasksPerLoc = tasksPerLocale;
    coforall locid in targetLocDom {
      on targetLocs(locid) {
        locDist(locid) = new LocCyclicDist(rank, idxType, locid, this);
      }
    }
  }

  def CyclicDist(param rank: int, type idxType, other: CyclicDist(rank, idxType)) {
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    locDist = other.locDist;
    tasksPerLoc = other.tasksPerLoc;
  }

  def getChunk(inds, locid) {
    return inds((...locDist(locid).myChunk.getIndices()));
  }

  def newArithmeticDom(param rank: int, type idxType, param stridable: bool, param alias: bool = false) {
    if idxType != this.idxType then
      compilerError("Block domain index type does not match distribution's");
    if rank != this.rank then
      compilerError("Block domain rank does not match distribution's");
    var dom = new CyclicDom(rank=rank, idxType=idxType, dist = this, stridable=stridable, alias=false);
    dom.setup();
    return dom;
  }

  def writeThis(x: Writer) {
    x.writeln("Cyclic");
    x.writeln("------");
    for locid in targetLocDom do
      x.writeln(" [", locid, "=", targetLocs(locid), "] owns chunk: ", locDist(locid).myChunk);
  }

  def ind2locInd(i: idxType) {
    pragma "inline" def mod(a:integral, b:integral) {
      if (b <= 0) then
        halt("modulus divisor must be positive");
      var tmp = a % b:a.type;
      return if tmp < 0 then b:a.type + tmp else tmp;
    }
    const numLocs:idxType = targetLocDom.numIndices:idxType;
    return mod(mod(i, numLocs) - mod(min(idxType), numLocs), numLocs):int;
  }

  def ind2locInd(ind: rank*idxType) {
    var x: rank*idxType;
    var minIdx = min(idxType);
    for param i in 1..rank {
      var dimLen = targetLocDom.dim(i).length;
      x(i) = mod(mod(ind(i), dimLen) - mod(minIdx, dimLen), dimLen);
    }
    return x;
  }
  def ind2loc(i: idxType) where rank == 1 {
    return targetLocs(ind2locInd(i));
  }
  def ind2loc(i: rank*idxType) {
    return targetLocs(ind2locInd(i));
  }
}

class LocCyclicDist {
  param rank: int;
  type idxType;

  const myChunk: domain(rank, idxType, true);

  def LocCyclicDist(param rank, type idxType, locid, dist: CyclicDist(rank, idxType)) {
    if rank == 1 {
      const lo = min(idxType) + locid:idxType;
      const hi = max(idxType);
      myChunk = [lo..hi by dist.targetLocDom.numIndices];
    } else {
      var tuple: rank*range(idxType, stridable=true);
      for param i in 1..rank {
        tuple(i) = (min(idxType) + locid(i):idxType)..max(idxType) by dist.targetLocDom.dim(i).length;
      }
      myChunk = [(...tuple)];
    }
  }
}

class CyclicDom : BaseArithmeticDom {
  param rank: int;
  type idxType;
  param stridable: bool;
  param alias: bool = false;

  const dist: CyclicDist(rank, idxType);

  var locDoms: [dist.targetLocDom] LocCyclicDom(rank, idxType, stridable);

  const whole: domain(rank, idxType, stridable);

  var pid: int = -1;

  def setup() {
    coforall localeIdx in dist.targetLocDom {
      on dist.targetLocs(localeIdx) {
        locDoms(localeIdx) = new LocCyclicDom(rank, idxType, stridable, this,
                                              dist.getChunk(whole, localeIdx));
      }
    }
  }

  def buildArray(type eltType) {
    var arr = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this, alias=false);
    arr.setup();
    return arr;
  }

  def setIndices(x) {
    whole.setIndices(x);
    setup();
  }

  def writeThis(x: Writer) {
    for loc in dist.targetLocDom {
      x.writeln("[", loc, "=", dist.targetLocs(loc), "] owns ", locDoms(loc).myBlock);
    }
  }
  def numIndices return whole.numIndices;

  def these() {
    for i in whole do
      yield i;
  }

  def these(param tag: iterator) where tag == iterator.leader {
    const wholeLow = whole.low,
          precomputedNumTasks = dist.tasksPerLoc;
    coforall locDom in locDoms do on locDom {
      const locBlock = locDom.myBlock - wholeLow;
      const numTasks = precomputedNumTasks;
      if numTasks == 1 {
        yield locBlock;
      } else {
        if rank == 1 {
          coforall taskid in 0..#numTasks {
            const low = locBlock.low, high = locBlock.high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                         low, high,
                                         numTasks, taskid);
            yield locBlock[lo..hi];
          }
        } else {
          var ranges: rank*range(stridable=true);
          for param i in 2..rank do
            ranges(i) = locBlock.dim(i);
          coforall taskid in 0..#numTasks {
            const low = locBlock.dim(1).low, high = locBlock.dim(1).high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                          low, high,
                                          numTasks, taskid);
            ranges(1) = locBlock.dim(1)(lo..hi);
            yield locBlock[(...ranges)];
          }
        }
      }
    }
  }

  def these(param tag: iterator, follower, param aligned: bool = false) where tag == iterator.follower {
    const followThis = follower + whole.low;
    for i in followThis do
      yield i;
  }

  def supportsPrivatization() param return true;
  def privatize() {
    var privateDist = new CyclicDist(rank, idxType, dist);
    var c = new CyclicDom(idxType=idxType, rank=rank, stridable=stridable, dist=privateDist);
    c.locDoms = locDoms;
    c.whole = whole;
    return c;
  }
  def reprivatize(other) {
    locDoms = other.locDoms;
    whole = other.whole;
  }
  def dim(d: int) return whole.dim(d);
}

class LocCyclicDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  const wholeDom: CyclicDom(rank, idxType, stridable);
  var myBlock: domain(rank, idxType, true);
}

def _computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def procToData(x, lo)
    return lo + (x:lo.type) + (x:real != x:int:real):lo.type;

  const blo =
    if blocknum == 0 then waylo
      else procToData((numelems:real * blocknum) / numblocks, lo);
  const bhi =
    if blocknum == numblocks - 1 then wayhi
      else procToData((numelems:real * (blocknum+1)) / numblocks, lo) - 1;

  return (blo, bhi);
}

class CyclicArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;
  param alias: bool = false;
  var dom: CyclicDom(rank, idxType, stridable);

  var locArr: [dom.dist.targetLocDom] LocCyclicArr(eltType, rank, idxType, stridable);
  var pid: int = -1;

  def getBaseDom() return dom;

  def setup() {
    coforall localeIdx in dom.dist.targetLocDom {
      on dom.dist.targetLocs(localeIdx) {
        locArr(localeIdx) = new LocCyclicArr(eltType, rank, idxType, stridable, dom.locDoms(localeIdx));
      }
    }
  }

  def supportsPrivatization() param return true;
  def privatize() {
    var dompid = dom.pid;
    var thisdom = dom;
    var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
    var c = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
    c.locArr = locArr;
    return c;
  }

  def this(i: idxType) var where rank == 1 {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  def this(i:rank*idxType) var {
    var x = locArr(dom.dist.ind2locInd(i));
    return x(i);
  }

  def these() var {
    for i in dom do
      yield this(i);
  }

  def these(param tag: iterator) where tag == iterator.leader {
    const wholeLow = dom.whole.low,
          precomputedNumTasks = dom.dist.tasksPerLoc;
    coforall locDom in dom.locDoms do on locDom {
      const locBlock = locDom.myBlock - wholeLow;
      const numTasks = precomputedNumTasks;
      if numTasks == 1 {
        yield locBlock;
      } else {
        if rank == 1 {
          coforall taskid in 0..#numTasks {
            const low = locBlock.low, high = locBlock.high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                         low, high,
                                         numTasks, taskid);
            yield locBlock[lo..hi];
          }
        } else {
          var ranges: rank*range(stridable=true);
          for param i in 2..rank do
            ranges(i) = locBlock.dim(i);
          coforall taskid in 0..#numTasks {
            const low = locBlock.dim(1).low, high = locBlock.dim(1).high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                          low, high,
                                          numTasks, taskid);
            ranges(1) = locBlock.dim(1)(lo..hi);
            yield locBlock[(...ranges)];
          }
        }
      }
    }
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
    const followThis = follower + dom.whole.low;
    const myLocArr = locArr(dom.dist.ind2locInd(followThis.low));
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      def accessHelper(i) var {
        local {
          if myLocArr.locDom.myBlock.member(i) then
            return myLocArr.this(i);
        }
        return this(i);
      }

      for i in followThis {
        yield accessHelper(i);
      }
    }
  }

  def writeThis(f: Writer) {
    for i in dom {
      writeln(this(i));
    }
  }
}

class LocCyclicArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  const locDom: LocCyclicDom(rank, idxType, stridable);

  var myElems: [locDom.myBlock] eltType;

  def this(i:idxType) var {
    return myElems(i);
  }

  def this(i:rank*idxType) var {
    return myElems((...i));
  }

  def these() var {
    for elem in myElems {
      yield elem;
    }
  }
}

config const tpl = 2;

def smain {
  const r = 3.0;
  const Dist = new CyclicDist(1, int(64), tasksPerLocale = 2);
  const Dom: domain(1, int(64)) distributed Dist = [1..10:int(64)];
  var A, B: [Dom] real;

  forall i in Dom {
    B(i) = i;
  }

  forall (a, b) in (A, B) {
    a = b * r;
  }

  writeln(A);
  writeln(B);
}

def tmain {
  var Locs = Locales;
  var Dist = new CyclicDist(2, int, Locs, tasksPerLocale=tpl);
  var Dom: domain(2, int) distributed Dist = [1..10, 1..10];
  var Arr: [Dom] real;

  writeln(Dist);
  writeln(Dom);
  writeln(Arr);
  for (i,j) in Dom {
    writeln(Arr(i,j).locale, ": Arr(", i, ",", j, ") = ", Arr(i,j));
  }

  forall (i,j) in Dom { Arr(i,j) = i*100 + j; }
  forall a in Arr { a += 1; }
  writeln(Arr);
  /*
  var Dom: domain(1, int) distributed Dist = [-10..#20];
  var Arr: [Dom] int;
  Arr(5) = 1;
  writeln(Dist);
  writeln(Dom);
  writeln(Arr);
  for i in Dom {
    writeln("Arr(", i, ").locale = ", Arr(i).locale);
  }
  for a in Arr {
    writeln("a.locale = ", a.locale);
  }
  forall a in Arr {
    writeln(a.locale);
  }
  forall i in Dom {
    writeln(here, ": Arr(", i, ") = ", Arr(i), " Arr(", i, ").locale = ", Arr(i).locale);
  }
  */
}
}

module CyclicDistrib {
//
// naive routine for dividing numLocales into rank factors
//
def _factor(param rank: int, value) {
  var factors: rank*int;
  for param i in 1..rank do
    factors(i) = 1;
  if value >= 1 {
    var iv = value;
    var factor = 1;
    while iv > 1 {
      for i in 2..iv {
        if iv % i == 0 {
          var j = 1;
          for i in 2..rank {
            if factors(i) < factors(j) then
              j = i;
          }
          factors(j) *= i;
          iv = iv / i;
          break;
        }
      }
    }
  }
  for i in 1..rank do
    for j in i+1..rank do
      if factors(i) < factors(j) then
        factors(i) <=> factors(j);
  return factors;
}

class CyclicDist : BaseDist {
  param rank: int = 1;
  type idxType = int(64);

  const targetLocDom: domain(rank);
  const targetLocs: [targetLocDom] locale;

  const locDist: [targetLocDom] LocCyclicDist(rank, idxType);

  const tasksPerLoc = 1;

  def CyclicDist(param rank: int,
                 type idxType = int(64),
                 targetLocales: [] locale = Locales, tasksPerLocale=1) {
    if rank == 1  {
      targetLocDom = [0..#targetLocales.numElements];
      targetLocs = targetLocales;
    } else if targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank {
        ranges(i) = 0..#factors(i);
      }
      targetLocDom = [(...ranges)];
      for (loc1, loc2) in (targetLocs, targetLocales) {
        loc1 = loc2;
      }
    } else {
      if targetLocales.rank != rank then
        compilerError("locales array rank must be one or match distribution rank");
      halt("This case not handled yet.");
    }
    tasksPerLoc = tasksPerLocale;
    coforall locid in targetLocDom {
      on targetLocs(locid) {
        locDist(locid) = new LocCyclicDist(rank, idxType, locid, this);
      }
    }
  }

  def CyclicDist(param rank: int, type idxType, other: CyclicDist(rank, idxType)) {
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    locDist = other.locDist;
    tasksPerLoc = other.tasksPerLoc;
  }

  def getChunk(inds, locid) {
    return inds((...locDist(locid).myChunk.getIndices()));
  }

  def newArithmeticDom(param rank: int, type idxType, param stridable: bool, param alias: bool = false) {
    if idxType != this.idxType then
      compilerError("Block domain index type does not match distribution's");
    if rank != this.rank then
      compilerError("Block domain rank does not match distribution's");
    var dom = new CyclicDom(rank=rank, idxType=idxType, dist = this, stridable=stridable, alias=false);
    dom.setup();
    return dom;
  }

  def writeThis(x: Writer) {
    x.writeln("Cyclic");
    x.writeln("------");
    for locid in targetLocDom do
      x.writeln(" [", locid, "=", targetLocs(locid), "] owns chunk: ", locDist(locid).myChunk);
  }

  def ind2locInd(i: idxType) {
    pragma "inline" def mod(a:integral, b:integral) {
      if (b <= 0) then
        halt("modulus divisor must be positive");
      var tmp = a % b:a.type;
      return if tmp < 0 then b:a.type + tmp else tmp;
    }
    const numLocs:idxType = targetLocDom.numIndices:idxType;
    return mod(mod(i, numLocs) - mod(min(idxType), numLocs), numLocs):int;
  }

  def ind2locInd(ind: rank*idxType) {
    var x: rank*idxType;
    var minIdx = min(idxType);
    for param i in 1..rank {
      var dimLen = targetLocDom.dim(i).length;
      x(i) = mod(mod(ind(i), dimLen) - mod(minIdx, dimLen), dimLen);
    }
    return x;
  }
  def ind2loc(i: idxType) where rank == 1 {
    return targetLocs(ind2locInd(i));
  }
  def ind2loc(i: rank*idxType) {
    return targetLocs(ind2locInd(i));
  }
}

class LocCyclicDist {
  param rank: int;
  type idxType;

  const myChunk: domain(rank, idxType, true);

  def LocCyclicDist(param rank, type idxType, locid, dist: CyclicDist(rank, idxType)) {
    if rank == 1 {
      const lo = min(idxType) + locid:idxType;
      const hi = max(idxType);
      myChunk = [lo..hi by dist.targetLocDom.numIndices];
    } else {
      var tuple: rank*range(idxType, stridable=true);
      for param i in 1..rank {
        tuple(i) = (min(idxType) + locid(i):idxType)..max(idxType) by dist.targetLocDom.dim(i).length;
      }
      myChunk = [(...tuple)];
    }
  }
}

class CyclicDom : BaseArithmeticDom {
  param rank: int;
  type idxType;
  param stridable: bool;
  param alias: bool = false;

  const dist: CyclicDist(rank, idxType);

  var locDoms: [dist.targetLocDom] LocCyclicDom(rank, idxType, stridable);

  const whole: domain(rank, idxType, stridable);

  var pid: int = -1;

  def setup() {
    coforall localeIdx in dist.targetLocDom {
      on dist.targetLocs(localeIdx) {
        locDoms(localeIdx) = new LocCyclicDom(rank, idxType, stridable, this,
                                              dist.getChunk(whole, localeIdx));
      }
    }
  }

  def buildArray(type eltType) {
    var arr = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this, alias=false);
    arr.setup();
    return arr;
  }

  def setIndices(x) {
    whole.setIndices(x);
    setup();
  }

  def writeThis(x: Writer) {
    for loc in dist.targetLocDom {
      x.writeln("[", loc, "=", dist.targetLocs(loc), "] owns ", locDoms(loc).myBlock);
    }
  }
  def numIndices return whole.numIndices;

  def these() {
    for i in whole do
      yield i;
  }

  def these(param tag: iterator) where tag == iterator.leader {
    const wholeLow = whole.low,
          precomputedNumTasks = dist.tasksPerLoc;
    coforall locDom in locDoms do on locDom {
      const locBlock = locDom.myBlock - wholeLow;
      const numTasks = precomputedNumTasks;
      if numTasks == 1 {
        yield locBlock;
      } else {
        if rank == 1 {
          coforall taskid in 0..#numTasks {
            const low = locBlock.low, high = locBlock.high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                         low, high,
                                         numTasks, taskid);
            yield locBlock[lo..hi];
          }
        } else {
          var ranges: rank*range(stridable=true);
          for param i in 2..rank do
            ranges(i) = locBlock.dim(i);
          coforall taskid in 0..#numTasks {
            const low = locBlock.dim(1).low, high = locBlock.dim(1).high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                          low, high,
                                          numTasks, taskid);
            ranges(1) = locBlock.dim(1)(lo..hi);
            yield locBlock[(...ranges)];
          }
        }
      }
    }
  }

  def these(param tag: iterator, follower, param aligned: bool = false) where tag == iterator.follower {
    const followThis = follower + whole.low;
    for i in followThis do
      yield i;
  }

  def supportsPrivatization() param return true;
  def privatize() {
    var privateDist = new CyclicDist(rank, idxType, dist);
    var c = new CyclicDom(idxType=idxType, rank=rank, stridable=stridable, dist=privateDist);
    c.locDoms = locDoms;
    c.whole = whole;
    return c;
  }
  def reprivatize(other) {
    locDoms = other.locDoms;
    whole = other.whole;
  }
  def dim(d: int) return whole.dim(d);
}

class LocCyclicDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  const wholeDom: CyclicDom(rank, idxType, stridable);
  var myBlock: domain(rank, idxType, true);
}

def _computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def procToData(x, lo)
    return lo + (x:lo.type) + (x:real != x:int:real):lo.type;

  const blo =
    if blocknum == 0 then waylo
      else procToData((numelems:real * blocknum) / numblocks, lo);
  const bhi =
    if blocknum == numblocks - 1 then wayhi
      else procToData((numelems:real * (blocknum+1)) / numblocks, lo) - 1;

  return (blo, bhi);
}

class CyclicArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;
  param alias: bool = false;
  var dom: CyclicDom(rank, idxType, stridable);

  var locArr: [dom.dist.targetLocDom] LocCyclicArr(eltType, rank, idxType, stridable);
  var pid: int = -1;

  def setup() {
    coforall localeIdx in dom.dist.targetLocDom {
      on dom.dist.targetLocs(localeIdx) {
        locArr(localeIdx) = new LocCyclicArr(eltType, rank, idxType, stridable, dom.locDoms(localeIdx));
      }
    }
  }

  def supportsPrivatization() param return true;
  def privatize() {
    var dompid = dom.pid;
    var thisdom = dom;
    var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
    var c = new CyclicArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
    c.locArr = locArr;
    return c;
  }

  def this(i: idxType) var where rank == 1 {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  def this(i:rank*idxType) var {
    var x = locArr(dom.dist.ind2locInd(i));
    return x(i);
  }

  def these() var {
    for i in dom do
      yield this(i);
  }

  def these(param tag: iterator) where tag == iterator.leader {
    const wholeLow = dom.whole.low,
          precomputedNumTasks = dom.dist.tasksPerLoc;
    coforall locDom in dom.locDoms do on locDom {
      const locBlock = locDom.myBlock - wholeLow;
      const numTasks = precomputedNumTasks;
      if numTasks == 1 {
        yield locBlock;
      } else {
        if rank == 1 {
          coforall taskid in 0..#numTasks {
            const low = locBlock.low, high = locBlock.high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                         low, high,
                                         numTasks, taskid);
            yield locBlock[lo..hi];
          }
        } else {
          var ranges: rank*range(stridable=true);
          for param i in 2..rank do
            ranges(i) = locBlock.dim(i);
          coforall taskid in 0..#numTasks {
            const low = locBlock.dim(1).low, high = locBlock.dim(1).high;
            const (lo,hi) = _computeBlock(low, high - low + 1,
                                          low, high,
                                          numTasks, taskid);
            ranges(1) = locBlock.dim(1)(lo..hi);
            yield locBlock[(...ranges)];
          }
        }
      }
    }
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
    const followThis = follower + dom.whole.low;
    const myLocArr = locArr(dom.dist.ind2locInd(followThis.low));
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      def accessHelper(i) var {
        local {
          if myLocArr.locDom.myBlock.member(i) then
            return myLocArr.this(i);
        }
        return this(i);
      }

      for i in followThis {
        yield accessHelper(i);
      }
    }
  }

  def writeThis(f: Writer) {
    for i in dom {
      writeln(this(i));
    }
  }
}

class LocCyclicArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  const locDom: LocCyclicDom(rank, idxType, stridable);

  var myElems: [locDom.myBlock] eltType;

  def this(i:idxType) var {
    return myElems(i);
  }

  def this(i:rank*idxType) var {
    return myElems((...i));
  }

  def these() var {
    for elem in myElems {
      yield elem;
    }
  }
}

config const tpl = 2;

def smain {
  const r = 3.0;
  const Dist = new CyclicDist(1, int(64), tasksPerLocale = 2);
  const Dom: domain(1, int(64)) distributed Dist = [1..10:int(64)];
  var A, B: [Dom] real;

  forall i in Dom {
    B(i) = i;
  }

  forall (a, b) in (A, B) {
    a = b * r;
  }

  writeln(A);
  writeln(B);
}

def tmain {
  var Locs = Locales;
  var Dist = new CyclicDist(2, int, Locs, tasksPerLocale=tpl);
  var Dom: domain(2, int) distributed Dist = [1..10, 1..10];
  var Arr: [Dom] real;

  writeln(Dist);
  writeln(Dom);
  writeln(Arr);
  for (i,j) in Dom {
    writeln(Arr(i,j).locale, ": Arr(", i, ",", j, ") = ", Arr(i,j));
  }

  forall (i,j) in Dom { Arr(i,j) = i*100 + j; }
  forall a in Arr { a += 1; }
  writeln(Arr);
  /*
  var Dom: domain(1, int) distributed Dist = [-10..#20];
  var Arr: [Dom] int;
  Arr(5) = 1;
  writeln(Dist);
  writeln(Dom);
  writeln(Arr);
  for i in Dom {
    writeln("Arr(", i, ").locale = ", Arr(i).locale);
  }
  for a in Arr {
    writeln("a.locale = ", a.locale);
  }
  forall a in Arr {
    writeln(a.locale);
  }
  forall i in Dom {
    writeln(here, ": Arr(", i, ") = ", Arr(i), " Arr(", i, ").locale = ", Arr(i).locale);
  }
  */
}
}


D'oh!  Let's try that new pidigits version again.

Yesterday, I svn cp'd the release version of pidigits to maintain its
history and supporting files, and then intended to copy my new version
over top of it, but then forgot.  So here's the version that I meant
to commit yesterday.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@23367 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/* The Computer Language Benchmarks Game
   http://shootout.alioth.debian.org/

   contributed by Tom Hildebrandt, Brad Chamberlain, Lydia Duncan
   derived from the GNU C version by Ledrug
*/

use GMP;

config const n = 50;                // Compute n digits of Pi, 50 by default

proc main() {
  for (d,i) in zip(gen_digits(n), 1..) {
    write(d);
    if i % 10 == 0 then
      writeln("\t:",i);
  }
}

iter gen_digits(numDigits) {
  var tmp1, tmp2, acc, den, num: mpz_t;

  mpz_init(tmp1);                          // init tmp1
  mpz_init(tmp2);                          // init tmp2
  mpz_init_set_ui(acc, 0);                 // acc = 0
  mpz_init_set_ui(den, 1);                 // den = 1
  mpz_init_set_ui(num, 1);                 // num = 1

  var d, k: c_ulong;

  for i in 1..numDigits {
    do {
      do {
        k += 1;
        const k2 = 2 * k + 1;

        //
        // Compute the next term
        //
        mpz_addmul_ui(acc, num, 2);        // acc += num * 2
        mpz_mul_ui(acc, acc, k2);	   // acc *= (2k+1)
        mpz_mul_ui(den, den, k2);          // den *= (2k+1)
        mpz_mul_ui(num, num, k);           // num *= k

        //
        // Continue looping until the digit is ready
        //
      } while (mpz_cmp(num, acc) > 0);     // while num > acc

      d = extract_digit(3);
    } while (d != extract_digit(4));       // while the 3rd digit, d != the 4th

    yield d;                               // once it differs, yield it

    //
    // eliminate digit d
    //
    mpz_submul_ui(acc, den, d);            // acc = (acc - den) * d
    mpz_mul_ui(acc, acc, 10);              // acc *= 10
    mpz_mul_ui(num, num, 10);              // num *= 10
  }

  //
  // Helper function to extract the nth digit
  //
  proc extract_digit(nth: c_ulong) {
    mpz_mul_ui(tmp1, num, nth);              // tmp1 = num * nth
    mpz_add(tmp2, tmp1, acc);                // tmp2 = tmp1 + acc
    mpz_tdiv_q(tmp1, tmp2, den);             // tmp1 = tmp2 / den

    return mpz_get_ui(tmp1);                 // convert tmp1 to a c_uint
  }
}

/* The Computer Language Benchmarks Game
   http://shootout.alioth.debian.org/

   contributed by Tom Hildebrandt, Brad Chamberlain, Lydia Duncan
   derived from the GNU C version by Bonzini, Bartlett, and Mellor
*/

use GMP;

config const n = 50;                // Compute n digits of Pi, 50 by default

proc main() {
  for (d,i) in zip(gen_digits(n), 1..) {
    write(d);
    if i % 10 == 0 then
      writeln("\t:",i);
  }
}

iter gen_digits(numDigits) {
  var numer, accum, denom, tmp1, tmp2: mpz_t;

  mpz_init_set_ui(numer, 1);               // numer = 1
  mpz_init_set_ui(accum, 0);               // accum = 0
  mpz_init_set_ui(denom, 1);               // denom = 1
  mpz_init(tmp1);                          // init tmp1
  mpz_init(tmp2);                          // init tmp2

  var k: c_ulong;
  for i in 1..numDigits {
    do {
      do {
        k += 1;
        const y2 = 2 * k + 1;

        //
        // Compute the next term
        //
        mpz_mul_2exp(tmp1, numer, 1);      // tmp1  = numer * 2
        mpz_add(accum, accum, tmp1);	   // accum += numer * 2
        mpz_mul_ui(accum, accum, y2);	   // accum *= (2k+1)
        mpz_mul_ui(numer, numer, k);       // numer *= k
        mpz_mul_ui(denom, denom, y2);      // denom *= (2k+1)

        //
        // Continue looping until the digit is ready
        //
      } while (mpz_cmp(numer, accum) > 0); // while numer > accum

      //
      // Compute (numer * 3 + accum)
      //
      mpz_mul_2exp(tmp1, numer, 1);        // tmp1 = numer * 2
      mpz_add(tmp1, tmp1, numer);          // tmp1 += numer
      mpz_add(tmp1, tmp1, accum);          // tmp1 += accum

      //
      // tmp1 = tmp1 / denom; tmp2 = tmp1 % denom
      //
      mpz_fdiv_qr(tmp1, tmp2, tmp1, denom);

      //
      // Now, if (numer * 3 + accum) % denom + numer
      // == (numer * 4 + accum) % denom
      //
      mpz_add(tmp2, tmp2, numer);
    } while (mpz_cmp(tmp2, denom) >= 0);   // while tmp2 >= denom

    //
    // compute and yield the digit, d
    //
    const d = mpz_get_ui(tmp1);            // get least significant digit
    yield d;

    //
    // eliminate digit d
    //
    mpz_submul_ui(accum, denom, d);        // accum = (accum - denom) * d
    mpz_mul_ui(accum, accum, 10);          // accum *= 10
    mpz_mul_ui(numer, numer, 10);          // numer *= 10
  }
}


Fixed race-y test with some additional locking using a single var.

FYI, when running repeatedly in a loop (and checking for failure), I
can infrequently get the following error:

        pthread_mutex_lock.c:87: __pthread_mutex_lock: Assertion `mutex->__data.__owner == 0' failed.

This also died with seg fault once.  Might be a decent test case to
use to debug.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@17213 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config const n = 77;

var t: [1..n-1] sync bool;
var q: [2..n] sync bool;
var c: sync int = n;
var s: single bool;

coforall i in 1..n {
  var myc:int = c-1;
  if myc==0 then s=true; else c=myc;
  // wait for everyone to get here
  s;

  if i!=n then
  t[i];
  writeln(here.runningTasks());
  if i!=1 then
    t[i-1] = true;
}

config const n = 77;

var l: sync bool = false;

coforall i in 1..n {
  l;
  writeln(here.runningTasks());
  l = true;
}


Mandelbrot-fancy stored a variable but only used it after squaring its value.
Since the squared value is what matches the reference implementations, I changed
the code to remove the unnecessary operations and updated the variable.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@22440 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config const size : uint = 200;

// for binary output
extern proc putchar(i : int) : int;

proc main()
{
  const bytesRequired : uint = (size + 7) / 8;
  const byteRange = 0..#bytesRequired:int(64);
  const sizeRange = 0..#size:int(64);

  var bytes : [sizeRange, byteRange] uint(8);

  const limit : real = 4.0;
  const maxIter : int = 50;

  forall (byte,(y,bytex)) in zip(bytes,bytes.domain) do
    {
      var byte_acc : uint(8) = 0;

      for x in (bytex*8)..#8 do
        {
          var Zr : real = 0.0;
          var Zi : real = 0.0;
          var Tr : real = 0.0;
          var Ti : real = 0.0;
          const Cr : real = (2.0*x/size - 1.5);
          const Ci : real = (2.0*y/size - 1.0);

          {
            var iterNo : int = maxIter;
            while (iterNo > 0 && (Tr+Ti <= limit))
              {
                Zi = 2.0*Zr*Zi + Ci;
                Zr = Tr - Ti + Cr;
                Tr = Zr * Zr;
                Ti = Zi * Zi;
                iterNo -= 1;
              }
          }

          byte_acc <<= 1;
          if(Tr+Ti <= limit) then
            byte_acc |= 0x01;
        }

      byte = byte_acc;
    }

  writeln("P4");
  writeln(size," ",size);

  for (x,y) in bytes.domain do
    {
      putchar(bytes(x,y));
    }

}

config const size : uint = 200;

// for binary output
extern proc putchar(i : int) : int;

proc main()
{
  const bytesRequired : uint = (size + 7) / 8;
  const byteRange = 0..#bytesRequired:int(64);
  const sizeRange = 0..#size:int(64);

  var bytes : [sizeRange, byteRange] uint(8);

  const limit : real = 2.0;
  const maxIter : int = 50;

  forall (byte,(y,bytex)) in zip(bytes,bytes.domain) do
    {
      var byte_acc : uint(8) = 0;

      for x in (bytex*8)..#8 do
        {
          var Zr : real = 0.0;
          var Zi : real = 0.0;
          var Tr : real = 0.0;
          var Ti : real = 0.0;
          const Cr : real = (2.0*x/size - 1.5);
          const Ci : real = (2.0*y/size - 1.0);

          {
            var iterNo : int = maxIter;
            while (iterNo > 0 && (Tr+Ti <= limit**2))
              {
                Zi = 2.0*Zr*Zi + Ci;
                Zr = Tr - Ti + Cr;
                Tr = Zr * Zr;
                Ti = Zi * Zi;
                iterNo -= 1;
              }
          }

          byte_acc <<= 1;
          if(Tr+Ti <= limit**2) then
            byte_acc |= 0x01;
        }

      byte = byte_acc;
    }

  writeln("P4");
  writeln(size," ",size);

  for (x,y) in bytes.domain do
    {
      putchar(bytes(x,y));
    }

}


Rewrote test to use sync vars.

Rewrote test to use sync vars.

This test failed two nights ago on Valgrind despite sleep(1) delays.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13540 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use Time;

var s1, s2, s3: sync int;

def one() {
  writeln("one");
}

def two() {
  writeln("two");
}

def three() {
  writeln("three");
}

serial (2==9) {
  cobegin {
    begin { s3; three();}
    begin { s2; two(); s3 = 1; }
    begin { s1; one(); s2 = 1; }
    begin { writeln("zero"); s1 = 1; }
  }
}

use Time;

def one() {
  writeln("one");
}

def two() {
  writeln("two");
}

def three() {
  writeln("three");
}

serial (2==9) {
  cobegin {
    begin {sleep(3); three();}
    begin {sleep(2); two();}
    begin {sleep(1); one();}
    begin writeln("zero");
  }
}


Removing "no docs" pragmas that are no longer necessary

// Wrapper for FFTW version 3.

/*
  Support for key FFTW version 3 routines in Chapel trappings

  This module defines a Chapel wrapper around key routines from FFTW
  version 3.  Over time, the intention is to expand this support to
  additional routines, prioritizing based on user feedback.

  As in general FFTW, the flow is to:

  0) Call init_threads() and plan_with_nthreads() if using the
     multithreaded FFTW interface.

  1) Create plan(s) using the plan_dft*() routines.

  2) Execute the plan(s) using the execute() routine.

  3) Destroy the plan(s) using the destroy_plan() routine.

  4) If using the multithreaded FFTW interface, call
     cleanup_threads().

  5) Call cleanup().

  At present, clients of the FFTW module must specify fftw3.h and -I,
  -L, -l flags sufficient to specify the required libraries and find
  the files on the 'chpl' command line.  We are currently
  investigating techniques to streamline this process.
*/

// TODOs for Brad (in priority order):
//
// - How can we get away from fftw_complex and use Chapel's complex?
//
// - How do we feel about FFTW_ALLCAPS names given that the routines
//   themselves don't have fftw_ prefixes, that they're defined as
//   c_int's, and that they're all-caps?
//
// - Related question about fftw_ types taking type c_int, though
//   maybe we don't care that much since symbolic values are taken
//   in?
//

module FFTW {

  use SysCTypes;

  // Define the various planner flags
  // See Sec. 4.3.2 of FFTW manual "Planner Flags"

  //
  // TODO: Is it weird that we use FFTW_ names here, but not in
  // the routines?  Should we wrap the external flags so that
  // they can be ints or enums and avoid the all-caps C scheme
  // or is the familiarity better?
  //

  extern const FFTW_FORWARD : c_int;
  /* These two constants are symbolic aliases for making the sign of
     the exponent in the formula that defines the Fourier transform
     -1 or 1, respectively. */
  extern const FFTW_BACKWARD : c_int;

  extern const FFTW_ESTIMATE : c_uint;
  extern const FFTW_MEASURE : c_uint;
  extern const FFTW_PATIENT : c_uint;
  extern const FFTW_EXHAUSTIVE : c_uint;
  /*
     All of the above are planning-rigor flags.  See `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual
     for details.
  */
  extern const FFTW_WISDOM_ONLY : c_uint;
  extern const FFTW_DESTROY_INPUT : c_uint;
  extern const FFTW_PRESERVE_INPUT : c_uint;
  /*
     All of the above are algorithm-restriction flags.  See `Section
     4.3.2 <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW
     manual for details.
  */
  extern const FFTW_UNALIGNED : c_uint;

  // Define types
  // NOTE : Ideally, this should be mapped to complex(128), but I seem
  // to run into casting issues in the C code. This is a simple
  // workaround for now - although ugly.  Also note that if one used
  // complex types, then one would need to include complex.h at the
  // command line.
  //
  // TODO: What would it take to change these to complex rather than
  // fftw_complex?
  //
  /*
    At present, to send complex arrays to FFTW routines, they must be
    declared to be of type fftw_complex.  We anticipate adding support
    for using native Chapel complex types over time.
  */
  extern type fftw_complex = 2*real(64); // 4.1.1

  /*
    'fftw_plan' is an opaque type used by FFTW for storing and passing
    plans between routines.
  */
  extern type fftw_plan; // opaque type

  // Multi-threaded setup routines below
  /*
    Initialize the threads used for multi-threaded FFTW plans.

    :returns: Zero if there was an error, non-zero otherwise.
  */
  proc init_threads() : c_int {
    return C_FFTW.fftw_init_threads();
  }

  /*
    Registers the number of threads to use for multi-threaded FFTW
    plans.

    :arg nthreads: The number of threads to use.
    :type nthreads: int
   */
  proc plan_with_nthreads(nthreads: int) {
    C_FFTW.fftw_plan_with_nthreads(safe_cast(c_int, nthreads));
  }

  // Planner functions
  // Complex : 4.3.1
  // NOTE : We pass in arrays using ref

  // TODO: Can we have the plan_dft() routine below take in native
  // Chapel types without changing the external C constants from
  // type c_int?  Probably given that Chapel's int will be >=
  // C's int for the forseeable future.  See also the TODO above
  // about whether we'd want to represent those constants using
  // more native Chapel types anyway.

  /*
    Creates a plan for a complex->complex DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] :type:`fftw_complex`

    :arg output: The output array, with rank matching the input array's
    :type output: [] :type:`fftw_complex`

    :arg sign: The sign of the exponent in the DFT formula.
    :type sign: c_int

    :arg flags: planning-rigor and/or algorithm-restriction flags
    :type sign: c_int

    :returns: The fftw_plan representing the plan
  */
  proc plan_dft(input: [] fftw_complex, output: [] fftw_complex,
                 sign: c_int, flags: c_uint) : fftw_plan
  {
    param rank = input.rank;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = safe_cast(c_int, input.domain.dim(i).size);

    return C_FFTW.fftw_plan_dft(safe_cast(c_int, rank), dims, input,
                                     output, sign, flags);
  }

  // Real-to-complex and complex-to-real planning routines
  // There are two cases that we treat independently here : in-place and out-of-place transforms

  // Since the calls to FFTW are the same, pull the extern declarations out.
  // TODO : This should be cleaned up further and made consistent across the file

  // Out-of-place routines
  proc plan_dft_r2c(in1 : [] real(64), out1 : [] fftw_complex, flags :c_uint) : fftw_plan
  {
    param rank = in1.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = in1.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, in1, out1, flags);
  }

  proc plan_dft_c2r(in1 : [] fftw_complex, out1 : [] real(64),  flags :c_uint) : fftw_plan
  {
    param rank = out1.rank: c_int; // The dimensions are that of the real array

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = out1.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, in1, out1, flags);
  }

  // In-place routines, note that these take in the true leading dimension
  // TODO : We should put in checks to see that the sizes are consistent
  // TODO : We should check on types...
  proc plan_dft_r2c(realDom : domain, in1 : [] ?t, flags : c_uint) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, in1, in1, flags);
  }
  proc plan_dft_c2r(realDom : domain, in1: [] ?t, flags : c_uint) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, in1, in1, flags);
  }

  // Using plans
  /*
    Executes the given plan.

    :arg plan: The plan to execute, as computed by a plan*() routine.
    :type plan: fftw_plan
  */
  proc execute(const plan: fftw_plan) {
   C_FFTW.fftw_execute(plan);
  }

  /*
    Destroys the given plan.

    :arg plan: The plan to destroy
    :type plan: fftw_plan
  */
  proc destroy_plan(plan: fftw_plan) {
    C_FFTW.fftw_destroy_plan(plan);
  }

  /*
    Cleans up the threads if multi-threaded FFTW's were enabled using
    :proc:`init_threads()`.
  */
  proc cleanup_threads() {
    C_FFTW.fftw_cleanup_threads();
  }

  /*
    Cleans up FFTW overall.
  */
  proc cleanup() {
    C_FFTW.fftw_cleanup();
  }

  // Utilities -- not in FFTW
  /*
    Computes an absolute value on an :type:`fftw_complex` value.  This will
    be deprecated once we can replace :type:`fftw_complex` with Chapel's
    native complex type.

    :arg val: the input value
    :type val: :type:`fftw_complex`
  */
  proc abs(val: fftw_complex) : real(64) {
    const (r,c) = val;
    return sqrt(r*r + c*c);
  }

  /*
    Returns the real component of an :type:`fftw_complex` value.  This will be
    deprecated once we can replace :type:`fftw_complex` with Chapel's native
    complex type.

    :arg val: the input value
    :type val: :type:`fftw_complex`

    :returns: the real component of value as a `real(64)`
  */
  proc re(val: fftw_complex) : real(64) {
    return val(1);
  }

  //
  // TODO: Is there a world in which returning this as imaginary would
  // be preferable?
  //

  /*
    Returns the imaginary component of an :type:`fftw_complex` value.  This
    will be deprecated once we can replace :type:`fftw_complex` with Chapel's
    native complex type.

    :arg val: the input value
    :type val: :type:`fftw_complex`

    :returns: the imaginary component of value as a `real(64)`
  */
  proc im(val: fftw_complex) : real(64) {
    return val(2);
  }

  pragma "no doc"
  module C_FFTW {
    extern proc fftw_plan_dft(rank: c_int,
        n,  // BLC: having trouble being specific
        in1: [] fftw_complex,
        out1: [] fftw_complex,
        sign : c_int, c_flags : c_uint) : fftw_plan;

  extern proc fftw_plan_dft_r2c(rank: c_int,
      n,  // BLC: having trouble being specific
      in1: [],
      out1: [],
      c_flags : c_uint) : fftw_plan;

  extern proc fftw_plan_dft_c2r(rank: c_int,
      n,  // BLC: having trouble being specific
      in1: [],
      out1: [],
      c_flags : c_uint) : fftw_plan;

    extern proc fftw_execute(const plan : fftw_plan);

    extern proc fftw_destroy_plan(plan : fftw_plan);

    extern proc fftw_cleanup();

    extern proc fftw_init_threads() : c_int;

    extern proc fftw_cleanup_threads();

    extern proc fftw_plan_with_nthreads(n : c_int);
  }
}

// Wrapper for FFTW version 3.

/*
  Support for key FFTW version 3 routines in Chapel trappings

  This module defines a Chapel wrapper around key routines from FFTW
  version 3.  Over time, the intention is to expand this support to
  additional routines, prioritizing based on user feedback.

  As in general FFTW, the flow is to:

  0) Call init_threads() and plan_with_nthreads() if using the
     multithreaded FFTW interface.

  1) Create plan(s) using the plan_dft*() routines.

  2) Execute the plan(s) using the execute() routine.

  3) Destroy the plan(s) using the destroy_plan() routine.

  4) If using the multithreaded FFTW interface, call
     cleanup_threads().

  5) Call cleanup().

  At present, clients of the FFTW module must specify fftw3.h and -I,
  -L, -l flags sufficient to specify the required libraries and find
  the files on the 'chpl' command line.  We are currently
  investigating techniques to streamline this process.
*/

// TODOs for Brad (in priority order):
//
// - How can we get away from fftw_complex and use Chapel's complex?
//
// - How do we feel about FFTW_ALLCAPS names given that the routines
//   themselves don't have fftw_ prefixes, that they're defined as
//   c_int's, and that they're all-caps?
//
// - Related question about fftw_ types taking type c_int, though
//   maybe we don't care that much since symbolic values are taken
//   in?
//

module FFTW {

  use SysCTypes;

  // Define the various planner flags
  // See Sec. 4.3.2 of FFTW manual "Planner Flags"

  //
  // TODO: Is it weird that we use FFTW_ names here, but not in
  // the routines?  Should we wrap the external flags so that
  // they can be ints or enums and avoid the all-caps C scheme
  // or is the familiarity better?
  //

  extern const FFTW_FORWARD : c_int;
  /* These two constants are symbolic aliases for making the sign of
     the exponent in the formula that defines the Fourier transform
     -1 or 1, respectively. */
  extern const FFTW_BACKWARD : c_int;

  extern const FFTW_ESTIMATE : c_uint;
  extern const FFTW_MEASURE : c_uint;
  extern const FFTW_PATIENT : c_uint;
  extern const FFTW_EXHAUSTIVE : c_uint;
  /*
     All of the above are planning-rigor flags.  See `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual
     for details.
  */
  extern const FFTW_WISDOM_ONLY : c_uint;
  extern const FFTW_DESTROY_INPUT : c_uint;
  extern const FFTW_PRESERVE_INPUT : c_uint;
  /*
     All of the above are algorithm-restriction flags.  See `Section
     4.3.2 <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW
     manual for details.
  */
  extern const FFTW_UNALIGNED : c_uint;

  // Define types
  // NOTE : Ideally, this should be mapped to complex(128), but I seem
  // to run into casting issues in the C code. This is a simple
  // workaround for now - although ugly.  Also note that if one used
  // complex types, then one would need to include complex.h at the
  // command line.
  //
  // TODO: What would it take to change these to complex rather than
  // fftw_complex?
  //
  /*
    At present, to send complex arrays to FFTW routines, they must be
    declared to be of type fftw_complex.  We anticipate adding support
    for using native Chapel complex types over time.
  */
  extern type fftw_complex = 2*real(64); // 4.1.1

  /*
    'fftw_plan' is an opaque type used by FFTW for storing and passing
    plans between routines.
  */
  extern type fftw_plan; // opaque type

  // Multi-threaded setup routines below
  /*
    Initialize the threads used for multi-threaded FFTW plans.

    :returns: Zero if there was an error, non-zero otherwise.
  */
  proc init_threads() : c_int {
    return C_FFTW.fftw_init_threads();
  }

  /*
    Registers the number of threads to use for multi-threaded FFTW
    plans.

    :arg nthreads: The number of threads to use.
    :type nthreads: int
   */
  proc plan_with_nthreads(nthreads: int) {
    C_FFTW.fftw_plan_with_nthreads(safe_cast(c_int, nthreads));
  }

  // Planner functions
  // Complex : 4.3.1
  // NOTE : We pass in arrays using ref

  // TODO: Can we have the plan_dft() routine below take in native
  // Chapel types without changing the external C constants from
  // type c_int?  Probably given that Chapel's int will be >=
  // C's int for the forseeable future.  See also the TODO above
  // about whether we'd want to represent those constants using
  // more native Chapel types anyway.

  /*
    Creates a plan for a complex->complex DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] :type:`fftw_complex`

    :arg output: The output array, with rank matching the input array's
    :type output: [] :type:`fftw_complex`

    :arg sign: The sign of the exponent in the DFT formula.
    :type sign: c_int

    :arg flags: planning-rigor and/or algorithm-restriction flags
    :type sign: c_int

    :returns: The fftw_plan representing the plan
  */
  proc plan_dft(input: [] fftw_complex, output: [] fftw_complex,
                 sign: c_int, flags: c_uint) : fftw_plan
  {
    param rank = input.rank;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = safe_cast(c_int, input.domain.dim(i).size);

    return C_FFTW.fftw_plan_dft(safe_cast(c_int, rank), dims, input,
                                     output, sign, flags);
  }

  // Real-to-complex and complex-to-real planning routines
  // There are two cases that we treat independently here : in-place and out-of-place transforms

  // Since the calls to FFTW are the same, pull the extern declarations out.
  // TODO : This should be cleaned up further and made consistent across the file

  // Out-of-place routines
  proc plan_dft_r2c(in1 : [] real(64), out1 : [] fftw_complex, flags :c_uint) : fftw_plan
  {
    param rank = in1.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = in1.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, in1, out1, flags);
  }

  proc plan_dft_c2r(in1 : [] fftw_complex, out1 : [] real(64),  flags :c_uint) : fftw_plan
  {
    param rank = out1.rank: c_int; // The dimensions are that of the real array

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = out1.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, in1, out1, flags);
  }

  // In-place routines, note that these take in the true leading dimension
  // TODO : We should put in checks to see that the sizes are consistent
  // TODO : We should check on types...
  proc plan_dft_r2c(realDom : domain, in1 : [] ?t, flags : c_uint) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, in1, in1, flags);
  }
  proc plan_dft_c2r(realDom : domain, in1: [] ?t, flags : c_uint) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, in1, in1, flags);
  }

  // Using plans
  /*
    Executes the given plan.

    :arg plan: The plan to execute, as computed by a plan*() routine.
    :type plan: fftw_plan
  */
  proc execute(const plan: fftw_plan) {
   C_FFTW.fftw_execute(plan);
  }

  /*
    Destroys the given plan.

    :arg plan: The plan to destroy
    :type plan: fftw_plan
  */
  proc destroy_plan(plan: fftw_plan) {
    C_FFTW.fftw_destroy_plan(plan);
  }

  /*
    Cleans up the threads if multi-threaded FFTW's were enabled using
    :proc:`init_threads()`.
  */
  proc cleanup_threads() {
    C_FFTW.fftw_cleanup_threads();
  }

  /*
    Cleans up FFTW overall.
  */
  proc cleanup() {
    C_FFTW.fftw_cleanup();
  }

  // Utilities -- not in FFTW
  /*
    Computes an absolute value on an :type:`fftw_complex` value.  This will
    be deprecated once we can replace :type:`fftw_complex` with Chapel's
    native complex type.

    :arg val: the input value
    :type val: :type:`fftw_complex`
  */
  proc abs(val: fftw_complex) : real(64) {
    const (r,c) = val;
    return sqrt(r*r + c*c);
  }

  /*
    Returns the real component of an :type:`fftw_complex` value.  This will be
    deprecated once we can replace :type:`fftw_complex` with Chapel's native
    complex type.

    :arg val: the input value
    :type val: :type:`fftw_complex`

    :returns: the real component of value as a `real(64)`
  */
  proc re(val: fftw_complex) : real(64) {
    return val(1);
  }

  //
  // TODO: Is there a world in which returning this as imaginary would
  // be preferable?
  //

  /*
    Returns the imaginary component of an :type:`fftw_complex` value.  This
    will be deprecated once we can replace :type:`fftw_complex` with Chapel's
    native complex type.

    :arg val: the input value
    :type val: :type:`fftw_complex`

    :returns: the imaginary component of value as a `real(64)`
  */
  proc im(val: fftw_complex) : real(64) {
    return val(2);
  }

  pragma "no doc"
  module C_FFTW {
    // CHPLDOC FIXME: I don't believe the following routines should
    // need to get labeled as "no doc" routines...
  pragma "no doc"
    extern proc fftw_plan_dft(rank: c_int,
        n,  // BLC: having trouble being specific
        in1: [] fftw_complex,
        out1: [] fftw_complex,
        sign : c_int, c_flags : c_uint) : fftw_plan;

  pragma "no doc"
  extern proc fftw_plan_dft_r2c(rank: c_int,
      n,  // BLC: having trouble being specific
      in1: [],
      out1: [],
      c_flags : c_uint) : fftw_plan;

  pragma "no doc"
  extern proc fftw_plan_dft_c2r(rank: c_int,
      n,  // BLC: having trouble being specific
      in1: [],
      out1: [],
      c_flags : c_uint) : fftw_plan;

  pragma "no doc"
    extern proc fftw_execute(const plan : fftw_plan);

  pragma "no doc"
    extern proc fftw_destroy_plan(plan : fftw_plan);

  pragma "no doc"
    extern proc fftw_cleanup();

  pragma "no doc"
    extern proc fftw_init_threads() : c_int;

  pragma "no doc"
    extern proc fftw_cleanup_threads();

  pragma "no doc"
    extern proc fftw_plan_with_nthreads(n : c_int);
  }
}