Continued working on MPMC queue work stealing

use Queue;
use Random;
use Time;
use BigInteger;

// Segment statuses
const UNLOCKED : uint = 0;
const ENQUEUE : uint = 1;
const DEQUEUE : uint = 2;
const STEALING : uint = 3;
const STEALING_INIT : uint = 4;
const VICTIM : uint = 5;

// Work Stealing statuses
const UNINITIALIZED : int = -2;
const INITIALIZED : int = -1;
const CLAIMED : int = 0;
const FINISHED_WITH_NO_WORK : int = 1;
const FINISHED_WITH_WORK : int = 2;

config param ELEMS_PER_BLOCK = 1024;

class LocalWorkQueueSegmentBlock {
  type eltType;
  var idx = 1;
  var elems : ELEMS_PER_BLOCK * eltType;
  var next : LocalWorkQueueSegmentBlock(eltType);
}

record LocalWorkQueueSegment {
  type eltType;

  var status : atomic uint;

  var headBlock : LocalWorkQueueSegmentBlock(eltType);
  var tailBlock : LocalWorkQueueSegmentBlock(eltType);

  // For when we are stealing work... helper threads may attempt to
  // help during a steal by setting one of the values from -1 to 0, and
  // when they are finished they may set it to 1 indicating that they are finished.
  // The first part helps distribute work to tasks waiting and doing nothing, and the
  // last part helps us know when we can declare that work stealing is finished.
  var workStealingStatus : [{0 .. #here.maxTaskPar}] atomic int;
  var workStolen : [{0 .. #here.maxTaskPar}] LocalWorkQueueSegmentBlock(eltType);
  var workStealingIdx : atomic int;

  // Number of elements currently in the list to allow faster determinism of whether
  // or not an operation should attempt this bucket.
  var nElems : atomic uint;

  inline proc isEmpty {
    return nElems.read() == 0;
  }
}

class LocalWorkQueue : Queue {
  // Place in own cache line
  var startIdxEnq : atomic uint;
  var startIdxDeq : atomic uint;
  var workStealer : atomic int;

  var maxParallelSegmentSpace = {0 .. #here.maxTaskPar};
  var segments : [maxParallelSegmentSpace] LocalWorkQueueSegment(eltType);

  const maxIterationsPerPhase = here.maxTaskPar;

  proc LocalWorkQueue(type eltType) {
    workStealer.write(-1);
  }

  /*
    Enqueue...
  */

  proc enqueue(elt : eltType) {
    var startIdx = startIdxEnq.fetchAdd(1) % here.maxTaskPar : uint;
    var segmentIdx = enqueueAcquire(startIdx);
    ref seg = segments[segmentIdx];
    var block = seg.tailBlock;

    // Empty?
    if block == nil then {
      seg.tailBlock = new LocalWorkQueueSegmentBlock(eltType);
      seg.headBlock = seg.tailBlock;
    }

    // Full?
    if block.idx > ELEMS_PER_BLOCK {
      block.next = new LocalWorkQueueSegmentBlock(eltType);
      seg.tailBlock = block.next;
      block = block.next;
    }

    block.elems[block.idx] = elt;
    block.idx = block.idx + 1;

    seg.nElems.add(1);
    seg.status.write(UNLOCKED);
  }

  proc enqueueAcquire(startIdx) : int {
    var retval = enqueueAcquirePhase1(startIdx);
    if retval == -1 {
      retval = enqueueAcquirePhase2(startIdx);
    }

    return retval;
  }

  // In Phase 1, we attempt to acquire the first bucket that is unlocked
  inline proc enqueueAcquirePhase1(startIdx) : int {
    var idx = startIdx : int;
    var iterations = 0;

    // Find first free bucket
    while iterations < maxIterationsPerPhase {
      ref seg = segments[idx];

      // Attempt to acquire...
      if seg.status.compareExchangeStrong(UNLOCKED, ENQUEUE) {
        return idx;
      }

      iterations = iterations + 1;
      idx = (idx + 1) % here.maxTaskPar;
    }

    return -1;
  }

  // In Phase 2, we attempt to acquire the first bucket that is locked...
  inline proc enqueueAcquirePhase2(startIdx) : int {
    var idx = startIdx : int;

    // Find first free bucket
    while true {
      ref seg = segments[idx];

      // Attempt to acquire...
      while true {
        var currStatus = seg.status.read();

        select currStatus {
          when UNLOCKED do {
            if seg.status.compareExchangeStrong(UNLOCKED, ENQUEUE) {
              return idx;
            }
          }
          // If someone is stealing, then that means that we'd be waiting for a while
          // anyway, so we may as well attempt to find a node with a shorter wait time.
          when STEALING || STEALING_INIT {
            break;
          }
        }

        // TODO: Implement exponential backoff + yielding
        chpl_task_yield();
      }

      idx = (idx + 1) % here.maxTaskPar;
    }

    halt("Broke out of phase 2 without getting segment...");
  }

  /*
    Dequeue...
  */

  proc dequeue() : (bool, eltType) {
    var startIdx = startIdxDeq.fetchAdd(1) % here.maxTaskPar : uint;

  }

  // Phase 1: acquire unlocked bucket which contains elements
  inline proc dequeueAcquirePhase1(startIdx) : int {
    var idx = startIdx : int;
    var maxIterations = here.maxTaskPar;
    var iterations = 0;

    // Find first non-empty free bucket
    while iterations < maxIterations {
      ref seg = segments[idx];

      // Attempt to acquire...
      if seg.nElems.peek() != 0 && seg.status.compareExchangeStrong(UNLOCKED, DEQUEUE) {
        return idx;
      }

      iterations = iterations + 1;
      idx = (idx + 1) % here.maxTaskPar;
    }

    return -1;
  }

  // Phase 2: acquire locked bucket which contains elements
  inline proc dequeueAcquirePhase2(startIdx) : int {
    var idx = startIdx : int;
    var maxIterations = here.maxTaskPar;
    var iterations = 0;

    // Find first non-empty bucket
    while iterations < maxIterations {
      ref seg = segments[idx];

      // Attempt to acquire...
      while seg.nElems.read() != 0 {
        var currStatus = seg.status.read();

        select currStatus {
          // Quick acquire
          when UNLOCKED {
            if seg.status.compareExchangeStrong(UNLOCKED, DEQUEUE) {
              return idx;
            }
          }
          // This segment is being stolen from. Assist the current stealer.
          when VICTIM {
            // TODO
          }
          // If someone else is stealing work, we add ourself as a potential helper
          // to help speed this along, and hopefully we get an element for ourself.
          when STEALING || STEALING_INIT {
            // TODO
          }
        }

        // Backoff
        chpl_task_yield();
      }

      iterations = iterations + 1;
      idx = (idx + 1) % here.maxTaskPar;
    }

    return -1;
  }

  // Phase 3: Take any bucket whether it is locked or unlocked, empty or non-empty
  inline proc dequeueAcquirePhase3(startIdx) : (bool, eltType) {
    ref seg = segments[startIdx];

    // Attempt to acquire...
    while true {
      var backoff = 0;
      var currStatus = seg.status.read();

      select currStatus {
        // Quick acquire
        when UNLOCKED {
          if seg.status.compareExchangeStrong(UNLOCKED, DEQUEUE) {
            return idx;
          }
        }
        // This segment is being stolen from. Assist the current stealer.
        when VICTIM {
          // TODO
          backoff = 0;
        }
        // If someone else is stealing work, we add ourself as a potential helper
        // to help speed this along, and hopefully we get an element for ourself.
        when STEALING || STEALING_INIT {
          var (hasElem, elem) = helpStealWork(startIdx);

          // We have our element...
          if hasElem then return (hasElem, elem);

          // We don't have our element, but no element currently exists.
          if seg.isEmpty {
            return (false, _defaultOf(eltType));
          }
          backoff = 0;
        }
      }

      // Backoff to maximum...
      if backoff == 0 then chpl_task_yield();
      else sleep(max(1000, 2 ** backoff), TimeUnits.microseconds);
    }

    halt("Somehow exited loop in Phase 3...");
  }

  /*
    Work Stealing
  */

  // We have the lock but it is empty, try to set ourself as work stealer...
  // As our index can change based on whether or not we become the work stealer
  // or help someone else become the work stealer, we return our acquired index.
  proc stealWork(idx) : (bool, eltType) {
    ref seg = segments[idx];

    // If we set ourself as work stealer, we may proceed, otherwise if someone else
    // is set, we help them out instead...
    while true {
      if workStealer.compareExchangeStrong(-1, idx) then break;

      // Someone else is set...
      var currStealer = workStealer.read();
      if currStealer != -1 {
        seg.status.write(UNLOCKED);
        return helpStealWork(currStealer);
      }
    }

    // We are the current work stealer, so our responsibility is to initialize
    // all work statuses and signal that we may begin our work stealing
    for status in seg.workStealingStatus do status.write(INITIALIZED);
    seg.workStealingIdx.write(0);
    seg.status.write(STEALING_INIT);

    // Help steal work with everyone else
    var (hasElem, elem) = helpStealWork(idx);

    // We are finished, all work stolen (if any) are now available...
    // As work stealer, our next job is merge it all together...
    for i in 0..#here.maxTaskPar {
      // Spin until they are finished... if they finish but have no work, skip them
      while seg.workStealingStatus[i].read() == CLAIMED then chpl_task_yield();
      if i == idx || seg.workStealingStatus[i].read() != FINISHED_WITH_WORK then continue;

      // TODO: Find a way copy data from one tuple to another that is faster
      var blockElems = block.idx - 1;
      seg.nElems.add(blockElems);

      ref stolenBlock = seg.workStolen[i];
      ref tailBlock = seg.tailBlock;
      while stolenBlock.idx > 0 {
        // If we haven't stolen an element by now, do so
        if !hasElem {
          elem = stolenBlock.elems[stolenBlock.idx];
          hasElem = true;
          stolenBlock.idx = stolenBlock.idx - 1;
          continue;
        }

        if tailBlock.idx > ELEMS_PER_BLOCK {
          tailBlock.next = new LocalWorkQueueSegmentBlock(eltType);
          seg.tailBlock = tailBlock.next;
          tailBlock = tailBlock.next;
        }

        tailBlock.elems[tailBlock.idx] = stolenBlock.elems[stolenBlock.idx];
        stolenBlock.idx = stolenBlock.idx - 1;
        tailBlock.idx = tailBlock.idx + 1;
      }

      delete stolenBlock;
    }

    // No longer work stealer
    workStealer.write(-1);
    seg.status.write(UNLOCKED);

    // At this point, we have successfully stolen work. Furthermore, we have
    // stolen at least one element for ourself.
    return (hasElem, elem);
  }

  proc helpStealWork(idx) : (bool, eltType) {
    ref seg = segments[idx];

    while true {
      var currStatus = seg.status.read();

      select currStatus {
        // They are still initializing...
        when STEALING {
          // Do nothing...
        }
        // They are initialized, we can now attempt to help...
        when STEALING_INIT {
          break;
        }
        // We're not stealing work...
        otherwise do return (false, _defaultOf(eltType));
      }
    }

    var (hasElem, elem) : (bool, eltType);

    // We obtain our index which tells us which segment we need to steal from
    // It should be noted that we handle the edge cases where we get preempted long
    // enough here that the current work stealing endeavor is over. In the case where
    // work stealing is finished and the next has not started, 'ourIdx' will be greater
    // than or equal to here.maxTaskPar, meaning that we would spin anyway... If the next
    // work stealing endeavor has started, but it has not yet been initialized, then technically
    // the counter would be below here.maxTaskPar, but the workStealingStatus would likely be
    // 'FINISHED_WITH_WORK' or 'FINISHED_WITH_NO_WORK'. In the case, that it is neither then we have
    // already setup everything anyway, and we can partake in the next work stealing.
    while true {
      var ourIdx = seg.workStealingIdx.fetchAdd(1);
      ref wsStatus = seg.workStealingStatus[ourIdx];

      if ourIdx >= here.maxTaskPar {
        return (hasElem, elem);
      }
      if wsStatus.compareExchangeStrong(INITIALIZED, CLAIMED) == false {
        return (hasElem, elem);
      }
      if ourIdx == idx then continue;

      // Do our part by stealing work...
      ref otherSeg = segments[ourIdx];
      while !otherSeg.isEmpty {
        if otherSeg.status.read() != UNLOCKED {
          chpl_task_yield();
        } else if otherSeg.status.compareExchangeStrong(UNLOCKED, VICTIM) {
          break;
        }
      }

      if otherSeg.isEmpty() {
        wsStatus.write(FINISHED_WITH_NO_WORK);
        otherSeg.status.write(UNLOCKED);
        continue;
      }

      if !hasElem {
        // Steal an item... adjust index if it is full (went past end)
        if otherSeg.headBlock.idx > here.maxTaskPar {
          otherSeg.headBlock.idx = otherSeg.headBlock.idx - 1;
        }
        (hasElem, elem) = (true, otherSeg.headBlock.elems[otherSeg.headBlock.idx]);
        otherSeg.headBlock.idx = otherSeg.headBlock.idx - 1;
        otherSeg.nElems.sub(1);

        // Fix list if we consumed last one...
        if otherSeg.headBlock.idx == 0 {
          var block = otherSeg.headBlock;
          otherSeg.headBlock = otherSeg.headBlock.next;
          delete block;
        }

        if otherSeg.nElems.read() == 0 {
          wsStatus.write(FINISHED_WITH_NO_WORK);
          otherSeg.status.write(UNLOCKED);
          continue;
        }
      }

      // Report what has been stolen...
      seg.workStolen[ourIdx] = otherSeg.headBlock;
      otherSeg.headBlock = otherSeg.headBlock.next;
      wsStatus.write(FINISHED_WITH_WORK);
      otherSeg.status.write(UNLOCKED);
    }
  }
}

proc main() {
  var nJitter = 0;
  var nComputations = 0;
  var nElements = 1000000;
  var nTrials = 8;
  var enqueueTrialTime : [1 .. nTrials] real;

  // Obtain average time for enqueue followed by dequeued...
  for i in 1 .. nTrials {
    var queue = new LocalWorkQueue(int);
    // We only use one or the other, but we must declare both.
    // TODO: Have both implement some parent 'Queue' class/interface?
    var timer = new Timer();
    timer.start();

    coforall loc in Locales do on loc {
      var iterations = nElements;

      coforall tid in 0 .. #here.maxTaskPar {
        var x : atomic int;
        var randStr = makeRandomStream(int);
        for j in 1 .. iterations / here.maxTaskPar {
          queue.enqueue(j);
          var nComps = nComputations + (if nJitter then (randStr.getNext() % nJitter) else 0);
          for i in 1 .. nComps {
            // Hopefully compiler doesn't throw away?
            x.write(sin(i) : int);
          }
        }
      }
    }

    timer.stop();
    enqueueTrialTime[i] = nElements / timer.elapsed();
    writeln(i, "/", nTrials, ": ", (+ reduce enqueueTrialTime) / i);
    timer.clear();
    delete queue;
  }
}

use Queue;
use Random;
use Time;
use BigInteger;

const UNLOCKED : uint = 0;
const ENQUEUE : uint = 1;
const DEQUEUE : uint = 2;
const STEALING : uint = 3;
const VICTIM : uint = 4;

config param ELEMS_PER_BLOCK = 1024;

class LocalWorkQueueSegmentBlock {
  type eltType;
  var idx = 1;
  var elems : ELEMS_PER_BLOCK * eltType;
  var next : LocalWorkQueueSegmentBlock(eltType);
}

record LocalWorkQueueSegment {
  type eltType;

  var status : atomic uint;

  var blockSpace = { 0 .. 0 };
  var headBlock : LocalWorkQueueSegmentBlock(eltType);
  var tailBlock : LocalWorkQueueSegmentBlock(eltType);

  // For when we are stealing work... helper threads may attempt to
  // help during a steal by setting one of the values from -1 to 0, and
  // when they are finished they may set it to 1 indicating that they are finished.
  // The first part helps distribute work to tasks waiting and doing nothing, and the
  // last part helps us know when we can declare that work stealing is finished.
  var workStealingStatus : [{0 .. #here.maxTaskPar}] atomic int;

  // Number of elements currently in the list to allow faster determinism of whether
  // or not an operation should attempt this bucket. Also doubles as the helper work
  // stealing counter;
  var nElems : atomic uint;

  proc LocalWorkQueueSegment(type eltType) {
    headBlock = new LocalWorkQueueSegmentBlock(eltType);
    tailBlock = headBlock;
  }
}

class LocalWorkQueue : Queue {
  // Place in own cache line
  var startIdxEnq : atomic uint;
  var startIdxDeq : atomic uint;
  var workStealer : atomic int;

  var maxParallelSegmentSpace = {0 .. #here.maxTaskPar};
  var segments : [maxParallelSegmentSpace] LocalWorkQueueSegment(eltType);

  const maxIterationsPerPhase = here.maxTaskPar;

  /*
    Enqueue...
  */

  proc enqueue(elt : eltType) {
    var startIdx = startIdxEnq.fetchAdd(1) % here.maxTaskPar : uint;
    var segmentIdx = enqueueAcquire(startIdx);
    ref seg = segments[segmentIdx];
    var block = seg.tailBlock;

    // Full?
    if block.idx > ELEMS_PER_BLOCK {
      block.next = new LocalWorkQueueSegmentBlock(eltType);
      seg.tailBlock = block.next;
      block = block.next;
    }

    block.elems[block.idx] = elt;
    block.idx = block.idx + 1;

    seg.nElems.add(1);
    seg.status.write(UNLOCKED);
  }

  proc enqueueAcquire(startIdx) : int {
    var retval = enqueueAcquirePhase1(startIdx);
    if retval == -1 {
      retval = enqueueAcquirePhase2(startIdx);
    }

    return retval;
  }

  // In Phase 1, we attempt to acquire the first bucket that is unlocked
  inline proc enqueueAcquirePhase1(startIdx) : int {
    var idx = startIdx : int;
    var iterations = 0;

    // Find first free bucket
    while iterations < maxIterationsPerPhase {
      ref seg = segments[idx];

      // Attempt to acquire...
      if seg.status.compareExchangeStrong(UNLOCKED, ENQUEUE) {
        return idx;
      }

      iterations = iterations + 1;
      idx = (idx + 1) % here.maxTaskPar;
    }

    return -1;
  }

  // In Phase 2, we attempt to acquire the first bucket that is locked...
  inline proc enqueueAcquirePhase2(startIdx) : int {
    var idx = startIdx : int;

    // Find first free bucket
    while true {
      ref seg = segments[idx];

      // Attempt to acquire...
      while true {
        var currStatus = seg.status.read();

        select currStatus {
          when UNLOCKED do {
            if seg.status.compareExchangeStrong(UNLOCKED, ENQUEUE) {
              return idx;
            }
          }
          // If someone is stealing, then that means that we'd be waiting for a while
          // anyway, so we may as well attempt to find a node with a shorter wait time.
          when STEALING {
            break;
          }
        }

        // TODO: Implement exponential backoff + yielding
        chpl_task_yield();
      }

      idx = (idx + 1) % here.maxTaskPar;
    }

    halt("Broke out of phase 2 without getting segment...");
  }

  /*
    Dequeue...
  */

  proc dequeue() : (bool, eltType) {
    var startIdx = startIdxDeq.fetchAdd(1) % here.maxTaskPar : uint;

  }

  // Phase 1: acquire unlocked bucket which contains elements
  inline proc dequeueAcquirePhase1(startIdx) : int {
    var idx = startIdx : int;
    var maxIterations = here.maxTaskPar;
    var iterations = 0;

    // Find first non-empty free bucket
    while iterations < maxIterations {
      ref seg = segments[idx];

      // Attempt to acquire...
      if seg.nElems.peek() != 0 && seg.status.compareExchangeStrong(UNLOCKED, DEQUEUE) {
        return idx;
      }

      iterations = iterations + 1;
      idx = (idx + 1) % here.maxTaskPar;
    }

    return -1;
  }

  // Phase 2: acquire locked bucket which contains elements
  inline proc dequeueAcquirePhase2(startIdx) : int {
    var idx = startIdx : int;
    var maxIterations = here.maxTaskPar;
    var iterations = 0;

    // Find first non-empty bucket
    while iterations < maxIterations {
      ref seg = segments[idx];

      // Attempt to acquire...
      while seg.nElems.read() != 0 {
        var currStatus = seg.status.read();

        select currStatus {
          // Quick acquire
          when UNLOCKED {
            if seg.status.compareExchangeStrong(UNLOCKED, DEQUEUE) {
              return idx;
            }
          }
          // This segment is being stolen from. Assist the current stealer.
          when VICTIM {
            // TODO
          }
          // If someone else is stealing work, we add ourself as a potential helper
          // to help speed this along, and hopefully we get an element for ourself.
          when STEALING {
            // TODO
          }
        }

        // Backoff
        chpl_task_yield();
      }

      iterations = iterations + 1;
      idx = (idx + 1) % here.maxTaskPar;
    }

    return -1;
  }
}

// Phase 3: Take any bucket whether it is locked or unlocked, empty or non-empty
inline proc dequeueAcquirePhase3(startIdx) : int {
  ref seg = segments[startIdx];

  // Attempt to acquire...
  while true {
    var backoff = 0;
    var currStatus = seg.status.read();

    select currStatus {
      // Quick acquire
      when UNLOCKED {
        if seg.status.compareExchangeStrong(UNLOCKED, DEQUEUE) {
          return idx;
        }
      }
      // This segment is being stolen from. Assist the current stealer.
      when VICTIM {
        // TODO
        backoff = 0;
      }
      // If someone else is stealing work, we add ourself as a potential helper
      // to help speed this along, and hopefully we get an element for ourself.
      when STEALING {
        // TODO
        backoff = 0;
      }
    }

    // Backoff to maximum...
    if backoff == 0 then chpl_task_yield();
    else sleep(max(1000, 2 ** backoff), TimeUnits.microseconds);
  }

  halt("Somehow exited loop in Phase 3...");
}

proc main() {
  var nJitter = 0;
  var nComputations = 0;
  var nElements = 1000000;
  var nTrials = 8;
  var enqueueTrialTime : [1 .. nTrials] real;

  // Obtain average time for enqueue followed by dequeued...
  for i in 1 .. nTrials {
    var queue = new LocalWorkQueue(int);
    // We only use one or the other, but we must declare both.
    // TODO: Have both implement some parent 'Queue' class/interface?
    var timer = new Timer();
    timer.start();

    coforall loc in Locales do on loc {
      var iterations = nElements;

      coforall tid in 0 .. #here.maxTaskPar {
        var x : atomic int;
        var randStr = makeRandomStream(int);
        for j in 1 .. iterations / here.maxTaskPar {
          queue.enqueue(j);
          var nComps = nComputations + (if nJitter then (randStr.getNext() % nJitter) else 0);
          for i in 1 .. nComps {
            // Hopefully compiler doesn't throw away?
            x.write(sin(i) : int);
          }
        }
      }
    }

    timer.stop();
    enqueueTrialTime[i] = nElements / timer.elapsed();
    writeln(i, "/", nTrials, ": ", (+ reduce enqueueTrialTime) / i);
    timer.clear();
    delete queue;
  }
}


Removed the 'executable' property from a .chpl file.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20790 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
Suggested TODO, in addition to implementing the actual computations:
- add a printout of grid/problem sizes etc. at start of program
- add verification that our cached slices survived after we have
  finished initializing everything.
  E.g. by dsiAccess-indexing into them and querying their domains.
- v1 messages can be now turned off (see util.chpl)
*/

use BlockDist;
use util;

config var chk = true;

/////////// grid of locales ///////////

// "x" -> first dimension
// from grid node (ix,iy): north (ix-1,iy); west (ix,iy-1)

config const gx=3, gy=3;
const gridDom = {1..gx, 1..gy};
const gridBig = {0..gx+1, 0..gy+1};

var gridLocales: [gridDom] locale;
setupGridLocales();
const gridDist = gridDom dmapped Block(gridDom, gridLocales);

/////////// the computation data and index sets ///////////

// the size of node-local data
config const ldx=2, ldy=3;
type elType = real;

// the size of all data
const allx = ldx * gx,
      ally = ldy * gy;

// for neighbor-cache pointers
var auxArr: [1..1] elType;
type cacheType = auxArr._value.type; // a class type, so it can be nil

///////////

// Node-local domains.
class LocalInfo {
  // our coordinates on the grid (perhaps not strictly needed)
  const mygx, mygy: int;
  // our coordinates in the data space
  const myhighx = mygx * ldx,
        mylowx  = myhighx - ldx + 1,
        myhighy = mygy * ldy,
        mylowy  = myhighy - ldy + 1;

  // frequently-used domains
  const domAlloc = {mylowx-1 .. myhighx+1, mylowy-1 .. myhighy+1};
  const domCompute = {mylowx..myhighx, mylowy..myhighy};
}

// A class for all the node-local domains.
class GlobalInfo {
  var infos: [gridDist] LocalInfo;
}

// constructor for GlobalInfo
proc GlobalInfo.GlobalInfo() {
  forall ((ix,iy), inf) in zip(gridDist, infos) {
    inf = new LocalInfo(mygx=ix, mygy=iy);
  }
}

// Here are all our local domains. WI <- "Working Indices".
const WI = new GlobalInfo();

assert(allx == WI.infos[gx,gy].myhighx, "a6 ");
assert(ally == WI.infos[gx,gy].myhighy, "a7");

///////////

// Node-local computation data.
class LocalData {
  const linfo: LocalInfo;

  // the locale share of the compute data, plus neighbor cache
  var ldata: [linfo.domAlloc] elType;

  // pointers to slices of neighbor data
  // from which we will fetch into our neighbor cache
  var cnorth, csouth, cwest, ceast: cacheType;
}

// A class for all node-local data.
class GlobalData {
  const name: string;
  var datas: [gridDist] LocalData;
}

// constructor for GlobalData
proc GlobalData.GlobalData(nameArg: string) {
  name=nameArg;
  forall (inf, dat, loc) in zip(WI.infos, datas, gridLocales) {
    dat = new LocalData(inf);
    // sanity checks
    assert(dat.locale == loc);
    assert(dat.linfo.locale == loc);
    assert(dat.linfo == inf);
  }

  /// get and store pointers to neighbor data slices ///

  serial(v1) do forall ((ix,iy), dat, inf) in zip(gridDist, datas, WI.infos) {
    msg1("neighbors for ", name, (ix, iy), "  domCompute=", inf.domCompute);
    dat.cnorth = storecache(-1,0, inf.mylowx-1,  inf.mylowy..inf.myhighy);
    dat.csouth = storecache(+1,0, inf.myhighx+1, inf.mylowy..inf.myhighy);
    dat.cwest = storecache(0,-1, inf.mylowx..inf.myhighx, inf.mylowy-1);
    dat.ceast = storecache(0,+1, inf.mylowx..inf.myhighx, inf.myhighy+1);

    proc storecache(dx, dy, slicex, slicey) {
      const ind = (ix+dx, iy+dy);
      if !gridDist.member(ind) {
	msg1("  ", ind, "  no neighbor");
	return nil;
      }
      const nbr = datas[ind]; // our neighbor
      var result: cacheType;
      on nbr {
	msg1("  ", ind, "  slice at [", slicex, ",", slicey, "]");
	var slice => nbr.ldata[slicex, slicey];
	result = slice._value;
	if !noRefCount then result._arrCnt.add(1);  // this is a bit low-level
      }
      return result;
    }  // storecache()

  }  // forall
}  // GlobalData constructor

// Our two global arrays, to switch between.
const WA = new GlobalData("WA"),
      WB = new GlobalData("WB");

// Reuse the name for an indexing operation.
// This does not access neighbor caches.
proc GlobalData.dsiAccess(ix,iy) var {
  if chk {
    if 1 <= ix && ix <= ldx * gx && 1 <= iy && iy <= ldy * gy
      then {/*OK*/}
      else halt("GlobalData.dsiAccess: index out of bounds ",
                (ix,ldx,gx), (iy,ldy,gy));
  }
  const gridx = (ix-1) / gx + 1, gridy = (iy-1) / gy + 1;
  return datas[gridx, gridy].ldata[ix,iy];
}

/////////// reference data ///////////

const refAlloc = {0..allx+1, 0..ally+1},
    refCompute = {1..allx, 1..ally},
  refAllocDist = refAlloc dmapped Block(refAlloc, gridLocales);

// Our reference data arrays, corresponding to WA, WB.
var RA, RB: [refAllocDist] elType;


Added labels to classes in BradsBlock1DPar classes

As we discussed during the last code review, I added some
high-level labels in the form of comments to my six classes
so that the different parts of the classes would be clearer.
In particular:

  // GENERICS:     the types/params that make the class generic
  // STATE:        the local state of the class
  // LINKAGE:      linkage to other classes that make up the distribution
                     DOWN = link from global class to local
                     UP   = link from local class to global
                     LEFT = link from dom->dist or from arr->dom
  // CONSTRUCTORS: constructors/initialize calls
  // INTERFACE:    different aspects of the interface
                     INTERNAL = specific to my block distribution/use of it
                     <other>  = interface for the named distribution class

This should help us read these classes, and should also help me
try to massage them into an official distribution, with any luck.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14473 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newDomain(inds, type domIdxType = idxType) where domIdxType == idxType {
    return new Block1DDom(idxType, this, whole=inds);
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom {

  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  // CONSTRUCTORS:

  def initialize() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocBlock1DDom(idxType, this,
                                           dist.getChunk(whole, localeIdx));
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));
  }

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def newArray(type elemType) {
    return new Block1DArr(idxType, elemType, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type elemType;

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, elemType);

  // CONSTRUCTORS:

  def initialize() {
    for localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, elemType, dom.locDoms(localeIdx));
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {
    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type elemType;

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] elemType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: Label which routines are the compiler's interface vs. an
// internal interface.

// TODO: Label the linkage fields.

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

// TODO: This would need to be moved somewhere more standard
enum IteratorType { leader, follower };

//
// The distribution class
//
class Block1DDist {
  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;
  //
  // NOTE: originally I wrote this as:
  //
  //   const targetLocs = Locales;
  //   const targetLocDom = targetLocs.domain;
  //
  // but this had two interesting problems:
  //
  // (1) the capture of the targetLocs array only makes a copy of the
  //     array, not of its domain, so if someone were to resize the
  //     LocaleSpace domain outside of the class, this would inadvertantly
  //     get resized as well.  We really want to make a copy of both the
  //     domain and array (which I now do in the constructor).
  //
  // (2) Initializing targetLocs with Locales prevents us from assigning
  //     anything other than a numLocales-ary array of locales to it
  //     which is stricter than I wanted -- I only wanted to restrict
  //     us to a 1D array of locales.  In our current model, this would
  //     require us to have some sort of [domain(1)] locale declaration
  //     to make it work.  Or we could think about whether a model in
  //     which only the array's static type information is inferred
  //     for a field?

  //
  // an array of local distribution class descriptors -- set up in
  // the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);
  }

  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newDomain(inds, type domIdxType = idxType) where domIdxType == idxType {
    return new Block1DDom(idxType, this, inds);
  }

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }

  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {
  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);
  const localeIdx: int;
  const loc: locale;

  //
  // Compute what chunk of index(1) is owned by the current locale
  // Arguments:
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real));

  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom {
  //
  // The index types of the domain
  //
  type idxType;

  //
  // a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  //
  // an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType);

  def initialize() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocBlock1DDom(idxType, this,
                                           dist.getChunk(whole, localeIdx));
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));
  }

  //
  // the iterator for the domain -- currently sequential
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    for locDom in locDoms do
      yield locDom.myBlock - whole.low;
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in followThis {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def newArray(type elemType) {
    return new Block1DArr(idxType, elemType, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }
}

//
// the local domain class
//
class LocBlock1DDom {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType);

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis)
        where iterator == IteratorType.follower {
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type elemType;

  //
  // the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType);

  //
  // an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, elemType);

  def initialize() {
    for localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, elemType, dom.locDoms(localeIdx));
  }

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
    for blk in dom.newThese(IteratorType.leader) do
      yield blk;
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
    for i in followThis {
      yield this(i + dom.low);
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {
    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {
  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type elemType;

  //
  // a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType);

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] elemType;

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def newThese(param iterator: IteratorType)
        where iterator == IteratorType.leader {
  }

  def newThese(param iterator: IteratorType, followThis) var
        where iterator == IteratorType.follower {
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }
}