forward is now operable, need to do backwards

/*
 Is this a package level comment?

 Algorithm 6.3 (Feed Forward) from Bengio, et al.
 Require: Network depth l
 Require: W for 1...l
 Require: b for 1...l
 Require: x, input data
 Require: y, target output (e.g. labels)
 h_0 = x  // dims batchSize x observationDim
 for k = 1..l do
  a_k = b_k + W_k h_{k-1}  // Notice, this is equiv to adding b to each ROW
                           // W_k units_k x batchSize
                           // b_k 1 x units_k
  h_k = f(a_k) // f = activation function
 end for
 y = h_l
 J = L(y,y) + lambda * Omega(theta)  // described later

 */
module NNModels {
  //use MLP;
  use LinearAlgebra;
  use Random;

  class Sequential {
    var epochs: int,
        lr: real;

    var layerDom: domain(1) = {1..0}; // Is now 1 long
    var layers: [layerDom] Layer;
    var needSomeLayers = true;
    proc add(d: Dense) {
      // Is it the first?
      var curidx = layerDom.size;
      writeln("curidx ", curidx);
      // layerDom has a default size of 1, even if empty
      // So curidx will be 0 twice

      if curidx == 0 {
        layerDom = {1..#layerDom.size+1};
        writeln("  Very first dense layer %i".format(layerDom.size));
        var l = new Layer(dataLayer=d
          , inputDim=d.inputDim
          , units=d.units
          , batchSize=d.batchSize
          , layerId=curidx
          , activation = new Activation(name="linear")
          );
        layers[layerDom.size] = l;
        layers[layerDom.size].summarize();
      } else {
        layerDom = {1..#layerDom.size+1};
        writeln(" Adding Dense layer %i".format(layerDom.size));
        var l = new Layer(dataLayer=d
          , inputDim=layers[layerDom.size-1].inputDim
          , units=d.units
          , batchSize=layers[layerDom.size-1].batchSize
          , layerId=curidx
          , activation=new Activation(name="linear"));
        layers[layerDom.size] = l;
        layers[layerDom.size].summarize();
      }
    }
    /*
     Must be added sequentially
     */
    proc add(a: Activation) {
      writeln("  Adding activation to layer %i".format(layerDom.size));
      layers[layerDom.size].activation = a;
    }

    /*
     */
    proc fit(xTrain:[], yTrain:[], epochs:int, lr: real) {
      for l in layerDom {
        if l == 1 {
          writeln(" layers[l].h domain " , layers[l].h.domain);
          layers[l].h = xTrain.T;
          continue;
        }
        writeln("Prepping layer %i".format(l));
        fillRandom(layers[l].W);
        fillRandom(layers[l].b);
        writeln(layers[l].bias.domain.dim(2));
        for i in layers[l].bias.domain.dim(2) {
          layers[l].bias[..,i] = layers[l].b;
        }

      }
      for e in 1..#epochs {
          writeln("epoch %i".format(e));
          writeln("Going FORWARD");
          for l in layerDom {
            if l == 1 {
              continue;
            }
            //writeln("%i: I need to \n\t1. multiply W_t, h_t-1, W_t\n\t2. Add the bias to each row\n\t3. Apply the activation function".format(l));
            // a = b + wh_{l-1}
            layers[l].a = matPlus(layers[l].bias, dot(layers[l].W, layers[l-1].h));
            layers[l].h = layers[l].activation.sigmoid(layers[l].a);
          }
          writln("Going BACKWARD");
      }
      return 0;
    }
  }

  /*
  inputDim: dimension of the input
  units: dimension of the output
   */
  class Layer {
    var dataLayer: Dense,
        activation: Activation,
        batchSize: int,
        inputDim: int,
        units: int,
        layerId: int,
        W: [{0..#units, 0..#inputDim}] real,
        a: [{0..#units, 0..#batchSize}] real,
        h: [{0..#inputDim, 0..#batchSize}] real,
        b: [{0..#units}] real, // The single column of bias
        bias: [{0..#units, 0..#batchSize}] real; // bias = [b,b,..]

        proc summarize() {
          writeln("\tinputDim: %i\n\tunits: %i\n\tbatchSize: %i".format(inputDim,units, batchSize ));
          writeln("\tW size ", W.shape);
          writeln("\th size ", h.shape);
          writeln("\ta size ", a.shape);
          writeln("\tb size ", b.shape);
          writeln("\tbias size ", bias.shape);
        }
  }

  /*
   Holds a layer of data.
   */
  class Dense {
    var units: int,  // dimension of the output space, to be fed to the next layer
        inputDim: int,  // Size of the vectors being considered
        batchSize: int;  // How many vectors are being sent it
  }

  class Activation {
    var name: string;

    /* @TODO abstract function and derivative to f() and df() */
    proc sigmoid(x: real) {
      return (1/(1 + exp(-x)));
    }

    proc derivativesSigmoid(x) {
      return x * (1-x);
    }
  }
}

/*
 Is this a package level comment?

 Algorithm 6.3 (Feed Forward) from Bengio, et al.
 Require: Network depth l
 Require: W for 1...l
 Require: b for 1...l
 Require: x, input data
 Require: y, target output (e.g. labels)
 h_0 = x  // dims batchSize x observationDim
 for k = 1..l do
  a_k = b_k + W_k h_{k-1}  // Notice, this is equiv to adding b to each ROW
                           // W_k units_k x batchSize
                           // b_k 1 x units_k
  h_k = f(a_k) // f = activation function
 end for
 y = h_l
 J = L(y,y) + lambda * Omega(theta)  // described later

 */
module NNModels {
  //use MLP;
  use LinearAlgebra;
  use Random;

  class Sequential {
    var epochs: int,
        lr: real;

    var layerDom: domain(1) = {1..0}; // Is now 1 long
    var layers: [layerDom] Layer;
    var needSomeLayers = true;
    proc add(d: Dense) {
      // Is it the first?
      var curidx = layerDom.size;
      writeln("curidx ", curidx);
      // layerDom has a default size of 1, even if empty
      // So curidx will be 0 twice

      if curidx == 0 {
        layerDom = {1..#layerDom.size+1};
        writeln("  Very first dense layer %i".format(layerDom.size));
        var l = new Layer(dataLayer=d
          , inputDim=d.inputDim
          , units=d.units
          , batchSize=d.batchSize
          , layerId=curidx
          , activation = new Activation(name="linear")
          );
        layers[layerDom.size] = l;
        layers[layerDom.size].summarize();
      } else {
        layerDom = {1..#layerDom.size+1};
        writeln(" Adding Dense layer %i".format(layerDom.size));
        var l = new Layer(dataLayer=d
          , inputDim=layers[layerDom.size-1].inputDim
          , units=d.units
          , batchSize=layers[layerDom.size-1].batchSize
          , layerId=curidx
          , activation=new Activation(name="linear"));
        layers[layerDom.size] = l;
        layers[layerDom.size].summarize();
      }
    }
    /*
     Must be added sequentially
     */
    proc add(a: Activation) {
      writeln("  Adding activation to layer %i".format(layerDom.size));
      //writeln(layers[layerDom.size-1]);
      //writeln(layers[layerDom.size]);
      layers[layerDom.size].activation = a;
    }

    /*
     */
    proc fit(xTrain:[], yTrain:[], epochs:int, lr: real) {
      for l in layerDom {
        if l == 1 {
          writeln(" layers[l].h domain " , layers[l].h.domain);
          layers[l].h = xTrain.T;
          continue;
        }
        writeln("Prepping layer %i".format(l));
        //writeln(layers[l].W.domain);
        fillRandom(layers[l].W);
        fillRandom(layers[l].bias);
        //writeln("W:");
        //writeln(layers[l].W);
        //writeln("\nb:");
        //writeln(layers[l].bias);
      }
      for e in 1..#epochs {
          writeln("epoch %i".format(e));
          writeln("Going FORWARD");
          for l in layerDom {
            if l == 1 {
              /*
              writeln(xTrain.domain);
              writeln(layers[1].W.domain);
              var wt = dot(layers[1].W, in);
              writeln(wt.domain);
              */
              continue;
            }
            writeln("%i: I need to \n\t1. multiply W_t, h_t-1, W_t\n\t2. Add the bias to each row\n\t3. Apply the activation function".format(l));
            writeln("W_now domain ", layers[l].W.domain);
            writeln("h_last domain ", layers[l-1].h.domain);
            var wl = dot(layers[l].W, layers[l-1].h);

          }
      }
      return 0;
    }
  }

  /*
  inputDim: dimension of the input
  units: dimension of the output
   */
  class Layer {
    var dataLayer: Dense,
        activation: Activation,
        batchSize: int,
        inputDim: int,
        units: int,
        layerId: int,
        W: [{0..#units, 0..#inputDim}] real,
        a: [{0..0, 0..0}] real,
        h: [{0..#inputDim, 0..#batchSize}] real,
        bias: [{0..#inputDim}] real;

        proc summarize() {
          writeln("\tinputDim: %i\n\tunits: %i\n\tbatchSize: %i".format(inputDim,units, batchSize ));
          writeln("\tW size ", W.shape);
          writeln("\th size ", h.shape);
          writeln("\ta size ", a.shape);
        }
  }

  /*
   Holds a layer of data.
   */
  class Dense {
    var units: int,  // dimension of the output space, to be fed to the next layer
        inputDim: int,  // Size of the vectors being considered
        batchSize: int;  // How many vectors are being sent it
  }

  class Activation {
    var name: string;

    /* @TODO abstract function and derivative to f() and df() */
    proc sigmoid(x: real) {
      return (1/(1 + exp(-x)));
    }

    proc derivativesSigmoid(x) {
      return x * (1-x);
    }
  }
}


Working more on Deque overhaul, need testing

use Collection;

// TODO: Make most methods convoy avoidant (randomization and deferred processing)...
// TODO: Convert into a Deque; we perform bounds checking at the door, so head and tail
// are guaranteed to not switch sides.

/*
  Frozen states... If we are FREEZE_UNFROZEN, we are mutable. If we are FREEZE_FROZEN,
  we are immutable. If we are FREEZE_MARKED, we are in the middle of a state change.
*/
private const FREEZE_UNFROZEN = 0;
private const FREEZE_MARKED = 1;
private const FREEZE_FROZEN = 2;

// Default assumption is that the cache line size is 64 bytes and that most elements
// are within 8 bytes.
private param DEQUE_BLOCK_SIZE = 8;

// For each node we manage an unroll block. This block needs to also support Deque
// operations, and as such we maintain our own mini headIdx and tailIdx. Since we can
// add and remove from either direction, we must also maintain the size ourselves...
class LocalDequeNode {
  type eltType;
  var elements : DEQUE_BLOCK_SIZE * eltType;
  var headIdx : int = 1;
  var tailIdx : int = 1;
  var size : int;
  var next : LocalDequeNode(eltType);
  var prev : LocalDequeNode(eltType);

  inline proc isFull {
    return size == DEQUE_BLOCK_SIZE;
  }

  inline proc isEmpty {
    return size == 0;
  }

  inline proc pushBack(elt : eltType) {
    tailIdx += 1;
    if tailIdx > DEQUE_BLOCK_SIZE {
      tailIdx = 1;
    }

    elements[tailIdx] = elt;
    size += 1;
  }

  inline proc popBack() : elt {
    tailIdx -= 1;
    if tailIdx == 0 {
      tailIdx = DEQUE_BLOCK_SIZE
    }

    size -= 1;
    return elements[tailIdx];
  }

  inline proc pushFront(elt : eltType) {
    headIdx -= 1;
    if headIdx == 0 {
      headIdx = DEQUE_BLOCK_SIZE;
    }

    elements[headIdx] = elt;
    size += 1;
  }

  inline proc popFront() : elt {
    headIdx += 1;
    if headIdx > DEQUE_BLOCK_SIZE {
      headIdx = 1;
    }

    size -= 1;
    return elements[headIdx];
  }
}

/*
  For each slot we maintain a deque.
*/
class LocalDeque {
  type eltType;

  // Locks are atomic to allow us to remotely contest for the lock.
  var lock$ : sync bool;

  var head : LocalDequeNode(eltType);
  var tail : LocalDequeNode(eltType);

  // We cache the last deleted node to handle cases where we have rapid mixed push/pop
  var cached : LocalDequeNode(eltType);

  // The size of a segment. This is used as both a means of knowing when an element
  // gets added, as well as a barrier to prevent the head and tail from being cached.
  var size : atomic int;

  inline proc recycleNode() {
    // If we have cached a previous used node, reuse it here...
    if cached != nil {
      cached = nil;
      return cached;
    }

    // Create a new one...
    return  new LocalDequeNode(eltType);
  }

  inline proc retireNode(node) {
    if cached != nil {
      delete cached;
    }

    cached = node;
  }

  proc pushBack(elt : eltType) {
    on this {
      var _elt = elt;
      local {
        lock$ = true;

        // Its empty...
        if tail == nil {
          tail = recycleNode();
          head = tail;
        }

        // Its full...
        if tail.isFull {
          tail.next = recycleNode();
          tail.next.prev = tail;
          tail = tail.next;
        }

        // Push...
        tail.pushBack(_elt);
        size.add(1);

        lock$;
      }
    }
  }

  // TODO: Make 'local'
  proc popBack() : eltType {
    var elt : eltType;
    on this {
      var _elt : eltType;
      local {
        while true {
          // Check if there is an element for us...
          if size.read() == 0 {
            while size.read() == 0 {
              chpl_task_yield();
            }
          }

          lock$ = true;

          // Someone else came in and took a value, wait for the next one...
          if size.read() == 0 {
            lock$;
            continue;
          }

          // Pop...
          _elt = tail.popBack();
          if tail.isEmpty {
            var node = tail;
            tail = tail.prev;

            // If not empty, remove reference to retired node...
            // If it is empty, fix the head...
            if tail != nil {
              tail.next = nil;
            } else {
              head = nil;
            }

            retireNode(node);
          }

          size.sub(1);
          lock$;
        }
      }
      elt = _elt;
    }

    return elt;
  }

  proc pushFront(elt : eltType) {
    on this {
      var _elt = elt;
      local {
        lock$ = true;

        // Its empty...
        if head == nil {
          head = recycleNode();
          tail = head;
        }

        // Its full...
        if head.isFull {
          head.prev = recycleNode();
          head.prev.next = head;
          head = head.prev;
        }

        // Push...
        tail.pushFront(_elt);
        size.add(1);

        lock$;
      }
    }
  }

  proc popFront() : eltType {
    var elt : eltType;
    on this {
      var _elt : eltType;
      local {
        while true {
          // Check if there is an element for us...
          if size.read() == 0 {
            while size.read() == 0 {
              chpl_task_yield();
            }
          }

          lock$ = true;

          // Someone else came in and took a value, wait for the next one...
          if size.read() == 0 {
            lock$;
            continue;
          }

          // Pop...
          _elt = head.popFront();
          if head.isEmpty {
            var node = head;
            head = head.next;

            // If not empty, remove reference to retired node...
            // If it is empty, fix the tail...
            if head != nil {
              head.prev = nil;
            } else {
              tail = nil;
            }

            retireNode(node);
          }

          size.sub(1);
          lock$;
        }
      }
      elt = _elt;
    }

    return elt;
  }

  proc ~LocalDeque() {
    on this {
      var curr = head;

      while curr != nil {
        var tmp = curr.next;
        delete curr;
        curr = tmp;
      }

      if cached != nil {
        delete cached;
      }
    }
  }
}

/*
  An unbounded queue that is distributed across nodes.
*/
class DistributedDeque : Collection {
  var cap;
  var targetLocales;

  // Privatization id
  var pid : int;

  // Keeps track of which slot we are on...
  var globalHead : atomic int;
  var globalTail : atomic int;
  var queueSize : atomic int;

  // Freezing the queue consists of two phases: The 'marked' phase, where the queue
  // is marked for a state change, which prevents any new tasks for a particular state
  // from entering, followed by a 'waiting' phase where we wait any concurrent tasks
  // to find across all nodes (in case they did not notice the state change). This
  // applies to state changes to frozen and unfrozen state.
  var concurrentTasks : atomic int;
  var frozenState : atomic int;

  // We maintain an array of slots, wherein each slot is a pointer into a node's
  // address space. To maximize parallelism, we maintain numLocales * maxTaskPar
  // to reduce the amount of contention.
  var nSlots = here.maxTaskPar * targetLocales.size;
  var slotSpace = {0..#nSlots};
  var slots : [slotSpace] LocalDeque(eltType);

  proc DistributedDeque(type eltType, cap=-1, targetLocales=Locales) {
    nSlots = here.maxTaskPar * targetLocales.size;
    slotSpace = {0..#nSlots};

    // Initialize each slot. We use a round-robin algorithm.
    var idx : atomic int;
    for 0 .. #here.maxTaskPar {
      for loc in targetLocales do on loc {
        var i = idx.fetchAdd(1);
        slots[i] = new LocalDeque(eltType);
      }
    }

    pid = _newPrivatizedClass(this);
  }

  proc DistributedDeque(other, privData, type eltType = other.eltType, cap=other.cap, targetLocales=other.targetLocales) {
    slots = other.slots;
  }

  proc dsiPrivatize(privData) {
      return new DistributedDeque(this, privData);
  }

  proc dsiGetPrivatizeData() {
    return pid;
  }

  inline proc getPrivatizedThis {
    return chpl_getPrivatizedCopy(this.type, pid);
  }

  inline proc enterRemoveBarrier(localThis) {
    // Enter freeze barrier...
    local {
      localThis.concurrentTasks.add(1);

      // Check if the queue is now 'immutable'.
      if localThis.frozenState.read() > FREEZE_UNFROZEN {
        localThis.concurrentTasks.sub(1);
        return false;
      }
    }

    // If we have a capacity of 0, then we can't really remove anything anyway.
    if localThis.cap == 0 {
      local { localThis.concurrentTasks.sub(1); }
      return false;
    }

    // Similar to add, we use a wait-free method for updating the queue size,
    // using fetch-sub instead. Like enqueue, we make attempts to fix the queue
    // size in a lock-free manner (for as long as it is negative). It should be
    // noted that if we see that the queue size is over capacity, meaning an
    // enqueue operation has over committed, we consequentially help them as well
    // during the fetch-sub loop.
    while true {
      var size = queueSize.fetchSub(1);

      // Negative, fix before returning...
      if size <= 0 {
        on this {
          var readSize = queueSize.read();
          // Attempt to fix, but yield to reduce potential contention and CPU hogging.
          while readSize < 0 && queueSize.compareExchangeWeak(readSize, 0) {
            chpl_task_yield();
            readSize = queueSize.read();
          }
        }

        // Fixed, return early...
        local { localThis.concurrentTasks.sub(1); }
        return false;
      }
      // If we have a cap, then at this point, we have something between [0, cap],
      // if we do not have a cap then there is no need to 'fix' the queue size;
      // in both cases we are safe.
      else if localThis.cap < 0 || size <= localThis.cap {
        break;
      }
      // If the size is over capacity, a enqueuer has overcommitted. Since there can
      // be an element for us, we loop again.
    }

    return true;
  }

  inline proc enterAddBarrier(localThis) {
    // Enter freeze barrier...
    local {
      localThis.concurrentTasks.add(1);

      // Check if the queue is now 'immutable'.
      if localThis.frozenState.read() > FREEZE_UNFROZEN {
        localThis.concurrentTasks.sub(1);
        return false;
      }
    }

    // If we have a capacity of 0, then we can't really add anything anyway.
    if localThis.cap == 0 {
      local { localThis.concurrentTasks.sub(1); }
      return false;
    }

    // If we have a capacity, we must ensure that there is enough space for our
    // operation ahead of time. We take a wait-free approach by performing a
    // fetch-add on the queue size. If there is space, then we succeed within one
    // communication round-trip. If we fail, then we make attempts to fix the queue
    // size in a lock-free manner (for as long as it is over capacity). It should be
    // noted that if we see that the queue size is negative, meaning a dequeue operation
    // has over committed, we consequentially help them as well during the fetch-add loop.
    if localThis.cap > 0 {
      while true {
        var size = queueSize.fetchAdd(1);

        // Over capacity, fix before returning...
        if size >= localThis.cap {
          on this {
            var readSize = queueSize.read();
            // Attempt to fix, but yield to reduce potential contention and CPU hogging.
            while readSize > this.cap && queueSize.compareExchangeWeak(readSize, this.cap) {
              chpl_task_yield();
              readSize = queueSize.read();
            }
          }

          // Fixed, return early...
          local { localThis.concurrentTasks.sub(1); }
          return false;
        }
        // At this point, we either have a capacity and something between [0, cap],
        // or we do not have a capacity and we have something greater than 0,
        // either of which means we are safe.
        else if size >= 0 {
          break;
        }
        // If the size is negative, a dequeuer has overcommitted. Since there can
        // be an element for us, we loop again.
      }
    }
    // If we have a 'negative' capacity, we do not have one and can hold an
    // infinite amount; this gives us a more optimized code path.
    else {
      while true {
        var size = queueSize.fetchAdd(1);

        // We have reserved a spot for our operation.
        if size >= 0 {
          break;
        }
        // If the size is negative, a dequeuer has overcommitted. Since there can
        // be an element for us, we loop again.
      }
    }

    return true;
  }

  proc add(elt : eltType) : bool {
    return pushBack(elt);
  }

  proc remove() : (bool, eltType) {
    return popFront();
  }

  proc enqueue(elt : eltType) : bool {
    return pushBack(elt);
  }

  proc dequeue() : (bool, eltType) {
    return popFront();
  }

  proc push(elt : eltType) : bool {
    return pushBack(elt);
  }

  proc pop() : (bool, eltType) {
    return popBack();
  }

  proc pushBack(elt : eltType) : bool {
    var localThis = getPrivatizedThis;

    // Test for if we can continue...
    if enterAddBarrier(localThis) == false {
      return false;
    }

    // At this point, we know we have a space for us. We find our slot based on another
    // fetch-add counter, making this wait-free as well.
    var tail = globalTail.fetchAdd(1) % localThis.nSlots;
    localThis.slots[abs(tail)].pushBack(elt);
    local { localThis.concurrentTasks.sub(1); }
    return true;
  }

  proc popBack() : (bool, eltType) {
    var localThis = getPrivatizedThis;

    // Test for if we can continue...
    if enterRemoveBarrier(localThis) == false {
      return (false, _defaultOf(eltType));
    }

    // We find our slot based on another fetch-add counter, making this wait-free as well.
    var tail = globalTail.fetchSub(1) % localThis.nSlots;
    var elt = localThis.slots[abs(tail)].popBack();
    local { localThis.concurrentTasks.sub(1); }
    return (true, elt);
  }

  proc pushFront(elt : eltType) : bool {
    var localThis = getPrivatizedThis;

    // Test if we can continue...
    if enterAddBarrier(localThis) == false {
      return false;
    }

    // We find our slot based on another fetch-add counter, making this wait-free as well.
    var head = globalHead.fetchSub(1) % localThis.nSlots;
    localThis.slots[abs(head)].pushFront(elt);
    local { localThis.concurrentTasks.sub(1); }
    return true;
  }

  proc popFront() : (bool, eltType) {
    var localThis = getPrivatizedThis;

    // Test for if we can continue...
    if enterRemoveBarrier(localThis) == false {
      return (false, _defaultOf(eltType));
    }

    // We find our slot based on another fetch-add counter, making this wait-free as well.
    var head = globalHead.fetchAdd(1) % localThis.nSlots;
    var elt = localThis.slots[abs(head)].popFront();
    local { localThis.concurrentTasks.sub(1); }
    return (true, elt);
  }

  proc canFreeze() : bool {
    return true;
  }

  proc isFrozen() : bool {
    var localThis = getPrivatizedThis;
    var state = localThis.frozenState.read();

    // Current transitioning state, wait it out...
    while state == FREEZE_MARKED {
      chpl_task_yield();
      state = localThis.frozenState.read();
    }

    return state == FREEZE_FROZEN;
  }

  proc freeze() : bool {
    var localThis = getPrivatizedThis;

    // Check if already frozen
    if localThis.frozenState.read() == FREEZE_FROZEN {
      return true;
    }

    // Mark as transient state...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_MARKED);
      localThis.concurrentTasks.waitFor(0);
    }

    // Mark as frozen...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_FROZEN);
    }

    return true;
  }

  proc unfreeze() : bool {
    var localThis = getPrivatizedThis;

    // Check if already unfrozen
    if localThis.frozenState.read() == FREEZE_UNFROZEN {
      return true;
    }

    // Mark as transient state...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_MARKED);
      localThis.concurrentTasks.waitFor(0);
    }

    // Mark as unfrozen...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_UNFROZEN);
    }

    return true;
  }

  // To clear the data structure, we must be sure to update the counter while
  // maintaining concurrent safety. The easiest way to do this is to just call remove.
  proc clear() {
    while remove()[1] do ;
  }

  proc size() : int {
    return queueSize.read();
  }

  proc isEmpty() : bool {
      return size() == 0;
  }

  // TODO: Make Convoy Avoidant
  proc contains(elt : eltType) : bool {
    // Frozen lookups can be done concurrently
    if isFrozen() {
      var containsElem : atomic bool;
      forall elem in this {
        if elem == elt {
          containsElem.write(true);
        }
      }
      return containsElem.read();
    }

    // Non-frozen lookups require us to obtain the lock to ensure mutual exclusion
    var localThis = getPrivatizedThis;
    var foundItem : atomic bool;
    for slot in localThis.slots {
      on slot {
        const targetElt = elt;
        slot.lock$ = true;

        var prev = slot.head;
        var node = slot.head.next;
        while node != nil {
          if node.elt == targetElt {
            foundItem.write(true);
            break;
          }
        }

        // Release...
        slot.lock$;
      }

      if foundItem.read() {
        break;
      }
    }

    return foundItem.read();
  }

  /*
    Iterate in FIFO order.

    TODO: Once the issue of serial iteration leaking state is fixed, a simple
    one-way channel implementation may prove very useful here to get an item
    in a round-robin manner.
  */
  iter these() : eltType {
    if !isFrozen() {
      halt("Iteration only supported while frozen...");
    }

    var localThis = getPrivatizedThis;

    // Fill our slots to visit in FIFO order.
    var head = globalHead.read();
    var tail = globalTail.read();
    var size = queueSize.read();

    // Check if empty...
    if size == 0 {
      return;
    }

    // We iterate directly over the heads of each slot, so we capture them in advance.
    var nodes : [{0..#nSlots}] LocalDequeNode(eltType);
    for i in 0 .. #nSlots {
      nodes[i] = slots[i].head;
    }

    // Iterate over captured head nodes; each time we read them we advance them
    for i in head..#tail {
      var idx = i % nSlots;
      yield nodes[idx].elt;
      nodes[idx] = nodes[idx].next;
    }
  }

  iter these(param tag : iterKind) where tag == iterKind.leader {
    if !isFrozen() {
      halt("Iteration only supported while frozen...");
    }
    coforall slot in getPrivatizedThis.slots do on slot do yield slot;
  }

  iter these(param tag : iterKind, followThis) where tag == iterKind.follower {
    var node = followThis.head.next;
    while node != nil {
      yield node.elt;
      node = node.next;
    }
  }

  proc ~DistributedDeque() {
    var localThis = getPrivatizedThis;
    for slot in localThis.slots do delete slot;
  }
}

use Collection;

// TODO: Make most methods convoy avoidant (randomization and deferred processing)...
// TODO: Convert into a Deque; we perform bounds checking at the door, so head and tail
// are guaranteed to not switch sides.

/*
  Frozen states... If we are FREEZE_UNFROZEN, we are mutable. If we are FREEZE_FROZEN,
  we are immutable. If we are FREEZE_MARKED, we are in the middle of a state change.
*/
private const FREEZE_UNFROZEN = 0;
private const FREEZE_MARKED = 1;
private const FREEZE_FROZEN = 2;

// Default assumption is that the cache line size is 64 bytes and that most elements
// are within 8 bytes.
private param DEQUE_BLOCK_SIZE = 8;

// For each node we manage an unroll block. This block needs to also support Deque
// operations, and as such we maintain our own mini headIdx and tailIdx. Since we can
// add and remove from either direction, we must also maintain the size ourselves...
class LocalDequeNode {
  type eltType;
  var elements : DEQUE_BLOCK_SIZE * eltType;
  var headIdx : int = 1;
  var tailIdx : int = 1;
  var size : int;
  var next : LocalDequeNode(eltType);
  var prev : LocalDequeNode(eltType);

  inline proc isFull {
    return size == DEQUE_BLOCK_SIZE;
  }

  inline proc isEmpty {
    return size == 0;
  }

  inline proc pushBack(elt : eltType) {
    tailIdx += 1;
    if tailIdx > DEQUE_BLOCK_SIZE {
      tailIdx = 1;
    }

    elements[tailIdx] = elt;
    size += 1;
  }

  inline proc popBack() : elt {
    tailIdx -= 1;
    if tailIdx == 0 {
      tailIdx = DEQUE_BLOCK_SIZE
    }

    size -= 1;
    return elements[tailIdx];
  }

  inline proc pushFront(elt : eltType) {
    headIdx -= 1;
    if headIdx == 0 {
      headIdx = DEQUE_BLOCK_SIZE;
    }

    elements[headIdx] = elt;
    size += 1;
  }

  inline proc popFront() : elt {
    headIdx += 1;
    if headIdx > DEQUE_BLOCK_SIZE {
      headIdx = 1;
    }

    size -= 1;
    return elements[headIdx];
  }
}

/*
  For each slot we maintain a deque.
*/
class LocalDeque {
  type eltType;

  // Locks are atomic to allow us to remotely contest for the lock.
  var lock$ : sync bool;

  var head : LocalDequeNode(eltType);
  var tail : LocalDequeNode(eltType);

  // We cache the last deleted node to handle cases where we have rapid mixed push/pop
  var cached : LocalDequeNode(eltType);

  // The size of a segment. This is used as both a means of knowing when an element
  // gets added, as well as a barrier to prevent the head and tail from being cached.
  var size : atomic int;

  inline proc recycleNode() {
    // If we have cached a previous used node, reuse it here...
    if cached != nil {
      cached = nil;
      return cached;
    }

    // Create a new one...
    return  new LocalDequeNode(eltType);
  }

  inline proc retireNode(node) {
    if cached != nil {
      delete cached;
    }

    cached = node;
  }

  proc pushBack(elt : eltType) {
    on this {
      lock$ = true;

      // Its empty...
      if tail == nil {
        tail = recycleNode();
        head = tail;
      }

      // Its full...
      if tail.isFull {
        tail.next = recycleNode();
        tail = tail.next;
      }

      // Push...
      tail.headIdx = ((tail.headIdx - 1) % DEQUE_BLOCK_SIZE);
      tail.elements[tail.headIdx]

      if tail != nil {
        tail.next = node;
      }
      if head == nil {
        head = node;
      }
      node.prev = tail;
      tail = node;
      size.add(1);

      lock$;
    }
  }

  // TODO: Make 'local'
  proc popBack() : eltType {
    var elt : eltType;
    on this {
      while true {
        // Wait until there is an element for us...
        while size.read() == 0 do chpl_task_yield();
        lock$ = true;

        // Someone else came in and took a value, wait for the next one...
        if size.read() == 0 {
          lock$;
          continue;
        }

        var node = tail;
        if node == head {
          head = nil;
        }
        tail = node.prev;
        size.sub(1);

        lock$;
        elt = node.elt;
        delete node;
        break;
      }
    }

    return elt;
  }

  proc pushFront(elt : eltType) {
    on this {
      var node = new LocalDequeNode(eltType, elt=elt);
      lock$ = true;

      if head != nil {
        head.prev = node;
      }
      if tail == nil {
        tail = node;
      }
      node.next = head;
      head = node;
      size.add(1);

      lock$;
    }
  }

  proc popFront() : eltType {
    var elt : eltType;
    on this {
      while true {
        // Wait until there is an element for us...
        while size.read() == 0 do chpl_task_yield();
        lock$ = true;

        // Someone else came in and took a value, wait for the next one...
        if size.read() == 0 {
          lock$;
          continue;
        }

        var node = head;
        if node == tail {
          tail = nil;
        }
        head = node.next;
        size.sub(1);

        lock$;
        elt = node.elt;
        delete node;
        break;
      }
    }

    return elt;
  }

  proc ~LocalDeque() {
    on this {
      var curr = head;

      while curr != nil {
        var tmp = curr.next;
        delete curr;
        curr = tmp;
      }
    }
  }
}

/*
  An unbounded queue that is distributed across nodes.
*/
class DistributedDeque : Collection {
  var cap;
  var targetLocales;

  // Privatization id
  var pid : int;

  // Keeps track of which slot we are on...
  var globalHead : atomic int;
  var globalTail : atomic int;
  var queueSize : atomic int;

  // Freezing the queue consists of two phases: The 'marked' phase, where the queue
  // is marked for a state change, which prevents any new tasks for a particular state
  // from entering, followed by a 'waiting' phase where we wait any concurrent tasks
  // to find across all nodes (in case they did not notice the state change). This
  // applies to state changes to frozen and unfrozen state.
  var concurrentTasks : atomic int;
  var frozenState : atomic int;

  // We maintain an array of slots, wherein each slot is a pointer into a node's
  // address space. To maximize parallelism, we maintain numLocales * maxTaskPar
  // to reduce the amount of contention.
  var nSlots = here.maxTaskPar * targetLocales.size;
  var slotSpace = {0..#nSlots};
  var slots : [slotSpace] LocalDeque(eltType);

  proc DistributedDeque(type eltType, cap=-1, targetLocales=Locales) {
    nSlots = here.maxTaskPar * targetLocales.size;
    slotSpace = {0..#nSlots};

    // Initialize each slot. We use a round-robin algorithm.
    var idx : atomic int;
    for 0 .. #here.maxTaskPar {
      for loc in targetLocales do on loc {
        var i = idx.fetchAdd(1);
        slots[i] = new LocalDeque(eltType);
      }
    }

    pid = _newPrivatizedClass(this);
  }

  proc DistributedDeque(other, privData, type eltType = other.eltType, cap=other.cap, targetLocales=other.targetLocales) {
    slots = other.slots;
  }

  proc dsiPrivatize(privData) {
      return new DistributedDeque(this, privData);
  }

  proc dsiGetPrivatizeData() {
    return pid;
  }

  inline proc getPrivatizedThis {
    return chpl_getPrivatizedCopy(this.type, pid);
  }

  inline proc enterRemoveBarrier(localThis) {
    // Enter freeze barrier...
    local {
      localThis.concurrentTasks.add(1);

      // Check if the queue is now 'immutable'.
      if localThis.frozenState.read() > FREEZE_UNFROZEN {
        localThis.concurrentTasks.sub(1);
        return false;
      }
    }

    // If we have a capacity of 0, then we can't really remove anything anyway.
    if localThis.cap == 0 {
      local { localThis.concurrentTasks.sub(1); }
      return false;
    }

    // Similar to add, we use a wait-free method for updating the queue size,
    // using fetch-sub instead. Like enqueue, we make attempts to fix the queue
    // size in a lock-free manner (for as long as it is negative). It should be
    // noted that if we see that the queue size is over capacity, meaning an
    // enqueue operation has over committed, we consequentially help them as well
    // during the fetch-sub loop.
    while true {
      var size = queueSize.fetchSub(1);

      // Negative, fix before returning...
      if size <= 0 {
        on this {
          var readSize = queueSize.read();
          // Attempt to fix, but yield to reduce potential contention and CPU hogging.
          while readSize < 0 && queueSize.compareExchangeWeak(readSize, 0) {
            chpl_task_yield();
            readSize = queueSize.read();
          }
        }

        // Fixed, return early...
        local { localThis.concurrentTasks.sub(1); }
        return false;
      }
      // If we have a cap, then at this point, we have something between [0, cap],
      // if we do not have a cap then there is no need to 'fix' the queue size;
      // in both cases we are safe.
      else if localThis.cap < 0 || size <= localThis.cap {
        break;
      }
      // If the size is over capacity, a enqueuer has overcommitted. Since there can
      // be an element for us, we loop again.
    }

    return true;
  }

  inline proc enterAddBarrier(localThis) {
    // Enter freeze barrier...
    local {
      localThis.concurrentTasks.add(1);

      // Check if the queue is now 'immutable'.
      if localThis.frozenState.read() > FREEZE_UNFROZEN {
        localThis.concurrentTasks.sub(1);
        return false;
      }
    }

    // If we have a capacity of 0, then we can't really add anything anyway.
    if localThis.cap == 0 {
      local { localThis.concurrentTasks.sub(1); }
      return false;
    }

    // If we have a capacity, we must ensure that there is enough space for our
    // operation ahead of time. We take a wait-free approach by performing a
    // fetch-add on the queue size. If there is space, then we succeed within one
    // communication round-trip. If we fail, then we make attempts to fix the queue
    // size in a lock-free manner (for as long as it is over capacity). It should be
    // noted that if we see that the queue size is negative, meaning a dequeue operation
    // has over committed, we consequentially help them as well during the fetch-add loop.
    if localThis.cap > 0 {
      while true {
        var size = queueSize.fetchAdd(1);

        // Over capacity, fix before returning...
        if size >= localThis.cap {
          on this {
            var readSize = queueSize.read();
            // Attempt to fix, but yield to reduce potential contention and CPU hogging.
            while readSize > this.cap && queueSize.compareExchangeWeak(readSize, this.cap) {
              chpl_task_yield();
              readSize = queueSize.read();
            }
          }

          // Fixed, return early...
          local { localThis.concurrentTasks.sub(1); }
          return false;
        }
        // At this point, we either have a capacity and something between [0, cap],
        // or we do not have a capacity and we have something greater than 0,
        // either of which means we are safe.
        else if size >= 0 {
          break;
        }
        // If the size is negative, a dequeuer has overcommitted. Since there can
        // be an element for us, we loop again.
      }
    }
    // If we have a 'negative' capacity, we do not have one and can hold an
    // infinite amount; this gives us a more optimized code path.
    else {
      while true {
        var size = queueSize.fetchAdd(1);

        // We have reserved a spot for our operation.
        if size >= 0 {
          break;
        }
        // If the size is negative, a dequeuer has overcommitted. Since there can
        // be an element for us, we loop again.
      }
    }

    return true;
  }

  proc add(elt : eltType) : bool {
    return pushBack(elt);
  }

  proc remove() : (bool, eltType) {
    return popFront();
  }

  proc enqueue(elt : eltType) : bool {
    return pushBack(elt);
  }

  proc dequeue() : (bool, eltType) {
    return popFront();
  }

  proc push(elt : eltType) : bool {
    return pushBack(elt);
  }

  proc pop() : (bool, eltType) {
    return popBack();
  }

  proc pushBack(elt : eltType) : bool {
    var localThis = getPrivatizedThis;

    // Test for if we can continue...
    if enterAddBarrier(localThis) == false {
      return false;
    }

    // At this point, we know we have a space for us. We find our slot based on another
    // fetch-add counter, making this wait-free as well.
    var tail = globalTail.fetchAdd(1) % localThis.nSlots;
    localThis.slots[abs(tail)].pushBack(elt);
    local { localThis.concurrentTasks.sub(1); }
    return true;
  }

  proc popBack() : (bool, eltType) {
    var localThis = getPrivatizedThis;

    // Test for if we can continue...
    if enterRemoveBarrier(localThis) == false {
      return (false, _defaultOf(eltType));
    }

    // We find our slot based on another fetch-add counter, making this wait-free as well.
    var tail = globalTail.fetchSub(1) % localThis.nSlots;
    var elt = localThis.slots[abs(tail)].popBack();
    local { localThis.concurrentTasks.sub(1); }
    return (true, elt);
  }

  proc pushFront(elt : eltType) : bool {
    var localThis = getPrivatizedThis;

    // Test if we can continue...
    if enterAddBarrier(localThis) == false {
      return false;
    }

    // We find our slot based on another fetch-add counter, making this wait-free as well.
    var head = globalHead.fetchSub(1) % localThis.nSlots;
    localThis.slots[abs(head)].pushFront(elt);
    local { localThis.concurrentTasks.sub(1); }
    return true;
  }

  proc popFront() : (bool, eltType) {
    var localThis = getPrivatizedThis;

    // Test for if we can continue...
    if enterRemoveBarrier(localThis) == false {
      return (false, _defaultOf(eltType));
    }

    // We find our slot based on another fetch-add counter, making this wait-free as well.
    var head = globalHead.fetchAdd(1) % localThis.nSlots;
    var elt = localThis.slots[abs(head)].popFront();
    local { localThis.concurrentTasks.sub(1); }
    return (true, elt);
  }

  proc canFreeze() : bool {
    return true;
  }

  proc isFrozen() : bool {
    var localThis = getPrivatizedThis;
    var state = localThis.frozenState.read();

    // Current transitioning state, wait it out...
    while state == FREEZE_MARKED {
      chpl_task_yield();
      state = localThis.frozenState.read();
    }

    return state == FREEZE_FROZEN;
  }

  proc freeze() : bool {
    var localThis = getPrivatizedThis;

    // Check if already frozen
    if localThis.frozenState.read() == FREEZE_FROZEN {
      return true;
    }

    // Mark as transient state...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_MARKED);
      localThis.concurrentTasks.waitFor(0);
    }

    // Mark as frozen...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_FROZEN);
    }

    return true;
  }

  proc unfreeze() : bool {
    var localThis = getPrivatizedThis;

    // Check if already unfrozen
    if localThis.frozenState.read() == FREEZE_UNFROZEN {
      return true;
    }

    // Mark as transient state...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_MARKED);
      localThis.concurrentTasks.waitFor(0);
    }

    // Mark as unfrozen...
    coforall loc in targetLocales do on loc {
      localThis.frozenState.write(FREEZE_UNFROZEN);
    }

    return true;
  }

  // To clear the data structure, we must be sure to update the counter while
  // maintaining concurrent safety. The easiest way to do this is to just call remove.
  proc clear() {
    while remove()[1] do ;
  }

  proc size() : int {
    return queueSize.read();
  }

  proc isEmpty() : bool {
      return size() == 0;
  }

  // TODO: Make Convoy Avoidant
  proc contains(elt : eltType) : bool {
    // Frozen lookups can be done concurrently
    if isFrozen() {
      var containsElem : atomic bool;
      forall elem in this {
        if elem == elt {
          containsElem.write(true);
        }
      }
      return containsElem.read();
    }

    // Non-frozen lookups require us to obtain the lock to ensure mutual exclusion
    var localThis = getPrivatizedThis;
    var foundItem : atomic bool;
    for slot in localThis.slots {
      on slot {
        const targetElt = elt;
        slot.lock$ = true;

        var prev = slot.head;
        var node = slot.head.next;
        while node != nil {
          if node.elt == targetElt {
            foundItem.write(true);
            break;
          }
        }

        // Release...
        slot.lock$;
      }

      if foundItem.read() {
        break;
      }
    }

    return foundItem.read();
  }

  /*
    Iterate in FIFO order.

    TODO: Once the issue of serial iteration leaking state is fixed, a simple
    one-way channel implementation may prove very useful here to get an item
    in a round-robin manner.
  */
  iter these() : eltType {
    if !isFrozen() {
      halt("Iteration only supported while frozen...");
    }

    var localThis = getPrivatizedThis;

    // Fill our slots to visit in FIFO order.
    var head = globalHead.read();
    var tail = globalTail.read();
    var size = queueSize.read();

    // Check if empty...
    if size == 0 {
      return;
    }

    // We iterate directly over the heads of each slot, so we capture them in advance.
    var nodes : [{0..#nSlots}] LocalDequeNode(eltType);
    for i in 0 .. #nSlots {
      nodes[i] = slots[i].head;
    }

    // Iterate over captured head nodes; each time we read them we advance them
    for i in head..#tail {
      var idx = i % nSlots;
      yield nodes[idx].elt;
      nodes[idx] = nodes[idx].next;
    }
  }

  iter these(param tag : iterKind) where tag == iterKind.leader {
    if !isFrozen() {
      halt("Iteration only supported while frozen...");
    }
    coforall slot in getPrivatizedThis.slots do on slot do yield slot;
  }

  iter these(param tag : iterKind, followThis) where tag == iterKind.follower {
    var node = followThis.head.next;
    while node != nil {
      yield node.elt;
      node = node.next;
    }
  }

  proc ~DistributedDeque() {
    var localThis = getPrivatizedThis;
    for slot in localThis.slots do delete slot;
  }
}


Changed names around in _chpl_seq.chpl to match names from before I
lost it.  This removes a few current regressions.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@3016 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

class _chplSeqNode {
  type _chplSeqElementType;

  var _chplSeqElement : _chplSeqElementType;
  var _chplSeqNext : _chplSeqNode;
}

class seq2 {
  type _chplSeqElementType;

  var length : integer;
  var _chplSeqFirst : _chplSeqNode;
  var _chplSeqLast : _chplSeqNode;

  function _chplSeqAppend(e : _chplSeqElementType) : seq2 {
    var new : _chplSeqNode = _chplSeqNode(_chplSeqElementType);
    new._chplSeqElement = e;
    if length > 0 {
      _chplSeqLast._chplSeqNext = new;
      _chplSeqLast = new;
    } else {
      _chplSeqFirst = new;
      _chplSeqLast = new;
    }
    length += 1;
    return this;
  }

  function _chplSeqPrepend(e : _chplSeqElementType) : seq2 {
    var new : _chplSeqNode = _chplSeqNode(_chplSeqElementType);
    new._chplSeqElement = e;
    if length > 0 {
      new._chplSeqNext = _chplSeqFirst;
      _chplSeqFirst = new;
    } else {
      _chplSeqFirst = new;
      _chplSeqLast = new;
    }
    length += 1;
    return this;
  }

  function _chplSeqConcat(s : seq2) : seq2 {
    _chplSeqLast._chplSeqNext = s._chplSeqFirst;
    _chplSeqLast = s._chplSeqLast;
    length += s.length;
    return this;
  }

  function copy() : seq2 {
    var new : seq2 = seq2(_chplSeqElementType);
    var tmp : _chplSeqNode = _chplSeqFirst;
    while tmp != nil {
      new = new._chplSeqAppend(tmp._chplSeqElement);
      tmp = tmp._chplSeqNext;
    }
    return new;
  }
}

function #(s : seq2, e : seq2._chplSeqElementType) {
  return s._chplSeqAppend(e);
}

function #(e : seq2._chplSeqElementType, s : seq2) {
  return s._chplSeqPrepend(e);
}

function #(s1 : seq2, s2 : seq2) {
  return s1._chplSeqConcat(s2);
}

function write(s : seq2) {
  write("(/");
  var tmp = s._chplSeqFirst;
  while tmp != nil {
    write(tmp._chplSeqElement);
    tmp = tmp._chplSeqNext;
    if (tmp != nil) {
      write(", ");
    }
  }
  write("/)");
}

class _chplSeqNode {
  type _chplElementType;

  var _chplElement : _chplElementType;
  var _chplNext : _chplSeqNode;
}

class seq2 {
  type _chplElementType;

  var length : integer;
  var _chplFirst : _chplSeqNode;
  var _chplLast : _chplSeqNode;

  function append(e : _chplElementType) : seq2 {
    var new : _chplSeqNode = _chplSeqNode(_chplElementType);
    new._chplElement = e;
    if length > 0 {
      _chplLast._chplNext = new;
      _chplLast = new;
    } else {
      _chplFirst = new;
      _chplLast = new;
    }
    length += 1;
    return this;
  }

  function prepend(e : _chplElementType) : seq2 {
    var new : _chplSeqNode = _chplSeqNode(_chplElementType);
    new._chplElement = e;
    if length > 0 {
      new._chplNext = _chplFirst;
      _chplFirst = new;
    } else {
      _chplFirst = new;
      _chplLast = new;
    }
    length += 1;
    return this;
  }

  function concat(s : seq2) : seq2 {
    _chplLast._chplNext = s._chplFirst;
    _chplLast = s._chplLast;
    length += s.length;
    return this;
  }

  function copy() : seq2 {
    var new : seq2 = seq2(_chplElementType);
    var tmp : _chplSeqNode = _chplFirst;
    while tmp != nil {
      new = new.append(tmp._chplElement);
      tmp = tmp.next;
    }
    return new;
  }
}

function #(s : seq2, e : seq2._chplElementType) {
  return s.append(e);
}

function #(e : seq2._chplElementType, s : seq2) {
  return s.prepend(e);
}

function #(s1 : seq2, s2 : seq2) {
  return s1.concat(s2);
}

function write(s : seq2) {
  var tmp = s._chplFirst;
  write("(/");
  while tmp != nil {
    write(tmp._chplElement);
    tmp = tmp._chplNext;
    if (tmp != nil) {
      write(", ");
    }
  }
  write("/)");
}


Fix capacityParSafe memory allocation failure by enabling parSafe updates. [Reviewed by koenigjm]

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20884 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// capacityParSafe.chpl
//
// Test that calls to set capacity work correctly in parallel.
//

// Safety under parallel execution must be requested explicitly (!).
var D : domain(string, parSafe=true);

coforall i in 1..1000 {
    D.requestCapacity(i);
}

writeln("SUCCESS");

var D : domain(string);

coforall i in 1..1000 {
    D.requestCapacity(i);
}

writeln("SUCCESS");


Remove outdated comment pointed out by Elliot

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial and single-locale, though the ultimate intention
// is to also support parallel and multi-locale options.
//
// These interfaces should be considered experimental and subject to
// change for the 1.10 release.  For that reason, send in feedback if
// you would like to see changes or additional arguments/features.
// And, as always, send in bug reports if you think we got something
// wrong.
//

/* iter listdir(path: string, hidden=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * hidden: should hidden files/directories be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, hidden=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (hidden || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          //
          // use FileSystem;  // Doesn't work, see comment below
          //
          const fullpath = path + "/" + filename;
          {
            //
            // The use of this compound statement to restrict the
            // impact of the 'use' of FileSystem is unfortunate
            // (compared to placing it in the more logical place
            // above), yet seemingly required at present; otherwise
            // the 'path' argument gets shadowed by a
            // (compiler-introduced?) method coming from one of the
            // standard or internal modules.  See
            // test/modules/bradc/useFileSystemShadowsPath.chpl for
            // a smaller standalone test exhibiting the issue (or
            // uncomment the 'use' above to see it here).
            //
            use FileSystem;

            if (listlinks || !isLink(fullpath)) {
              if (dirs && isDir(fullpath)) then
                yield filename;
              else if (files && isFile(fullpath)) then
                yield filename;
            }
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 hidden=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.

     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * hidden: indicates whether to enter hidden directories
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden,
                             followlinks, sort) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

//
// Here's a parallel version
//
iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false, param tag: iterKind): string
       where tag == iterKind.standalone {

  if (sort) then
    warning("sorting has no effect for parallel invocations of walkdirs()");

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    forall subdir in subdirs {
      const fullpath = path + "/" + subdir;
      //
      // Call standalone walkdirs() iterator recursively; set sort=false since it is
      // not useful and we've already printed the warning
      //
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden, followlinks, sort=false, iterKind.standalone) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()

     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

module Chpl_rt_glob {
  extern type glob_t;
  extern proc chpl_glob(pattern:c_string, flags: c_int,
                        ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);
}

iter glob(pattern="*") {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    for i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.standalone {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    forall i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: The following leader/follower iterator assumes that
// the file system state won't change between calls because
// it calls the glob routines twice and assumes the same
// result will come back.  In leader-follower 2.0, we would
// like to store such state between calls in which case this
// should be rewritten to do so (and would require freeing
// the state at the end of the call).
//
iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.leader {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  //
  // cast is used here to ensure we create an int-based leader
  //
  const num = Chpl_rt_glob.chpl_glob_num(glb).safeCast(int);
  if (num) {
    Chpl_rt_glob.globfree(glb);

    //
    // Forward to the range type's leader
    //
    for followThis in (0..num-1).these(tag) do
      yield followThis;
  }
}

iter glob(pattern:string="*", followThis, param tag: iterKind)
       where tag == iterKind.follower {
  var glb : Chpl_rt_glob.glob_t;
  if (followThis.size != 1) then
    compilerError("glob() iterator can only be zipped with 1D iterators");
  var (r,) = followThis;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (r.high > num.safeCast(int)) then
    halt("glob() iterator zipped with something too big");
  if (num) then
    for i in r do
      //
      // safe cast is used here to turn an int into a size_t
      //
      yield Chpl_rt_glob.chpl_glob_index(glb, i.safeCast(size_t)): string;

  Chpl_rt_glob.globfree(glb);
}

/* iter findfiles(startdir = ".", recursive=false, hidden=false)

   findfiles() is a simple find-like utility implemented using the
   above routines

     * startdir: where to start when looking for files
     * recursive: tells whether or not to descend recurisvely
     * hidden: tells whether or not to yield hidden
*/

iter findfiles(startdir = ".", recursive=false, hidden=false) {
  if (recursive) then
    for subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

iter findfiles(startdir = ".", recursive=false, hidden=false, param tag: iterKind) where tag == iterKind.standalone {
  if (recursive) then
    forall subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use Sort;

//
// This module defines several iterators that are designed to help
// support reasoning about directory and file contents.  At present,
// all are serial and single-locale, though the ultimate intention
// is to also support parallel and multi-locale options.
//
// These interfaces should be considered experimental and subject to
// change for the 1.10 release.  For that reason, send in feedback if
// you would like to see changes or additional arguments/features.
// And, as always, send in bug reports if you think we got something
// wrong.
//

/* iter listdir(path: string, hidden=false, dirs=true, files=true,
                listlinks=true): string

    listdir() lists the contents of a directory, similar to 'ls'
      * path: the directory whose contents should be listed
      * hidden: should hidden files/directories be listed?
      * dirs: should dirs be listed?
      * files: should files be listed?
      * listlinks: should symbolic links be listed?

   By default this routine lists all files and directories in the
   current directory, including symbolic links, as long as they don't
   start with '.'
*/

iter listdir(path: string, hidden=false, dirs=true, files=true,
             listlinks=true): string {
  extern type DIRptr;
  extern type direntptr;
  extern proc opendir(name: c_string): DIRptr;
  extern proc readdir(dirp: DIRptr): direntptr;
  extern proc closedir(dirp: DIRptr): c_int;

  proc direntptr.d_name(): c_string {
    extern proc chpl_rt_direntptr_getname(d: direntptr): c_string;

    return chpl_rt_direntptr_getname(this);
  }

  var dir: DIRptr;
  var ent: direntptr;
  dir = opendir(path:c_string);
  if (!is_c_nil(dir)) {
    ent = readdir(dir);
    while (!is_c_nil(ent)) {
      const filename = ent.d_name();
      if (hidden || filename.substring(1) != '.') {
        if (filename != "." && filename != "..") {
          //
          // use FileSystem;  // Doesn't work, see comment below
          //
          const fullpath = path + "/" + filename;
          {
            //
            // The use of this compound statement to restrict the
            // impact of the 'use' of FileSystem is unfortunate
            // (compared to placing it in the more logical place
            // above), yet seemingly required at present; otherwise
            // the 'path' argument gets shadowed by a
            // (compiler-introduced?) method coming from one of the
            // standard or internal modules.  See
            // test/modules/bradc/useFileSystemShadowsPath.chpl for
            // a smaller standalone test exhibiting the issue (or
            // uncomment the 'use' above to see it here).
            //
            use FileSystem;

            if (listlinks || !isLink(fullpath)) {
              if (dirs && isDir(fullpath)) then
                yield filename;
              else if (files && isFile(fullpath)) then
                yield filename;
            }
          }
        }
      }
      ent = readdir(dir);
    }
    closedir(dir);
  } else {
    extern proc perror(s: c_string);
    perror("error in listdir(): ");
  }
}

/* iter walkdirs(path: string=".", topdown=true, depth=max(int),
                 hidden=false, followlinks=false, sort=false): string

   walkdirs() recursively walks a directory structure, yielding
   directory names.  The strings that are generated will be rooted
   from 'path'.

     * path: the directory to start from
     * topdown: indicates whether to yield the directories using a
       preorder (vs. postorder) traversal
     * depth: indicates the maximal depth of recursion to use
     * hidden: indicates whether to enter hidden directories
     * followlinks: indicates whether to follow symbolic links or not
     * sort: indicates whether to consider subdirectories in sorted
       order or not

   by default, walkdirs() will start in the current directory, process
   directories in preorder; recursively traverse subdirectories; and
   neither follow dotfile directories nor symbolic links.  It will not
   sort the directories by default.
*/

iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false): string {

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    if (sort) then
      QuickSort(subdirs);
    for subdir in subdirs {
      const fullpath = path + "/" + subdir;
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden,
                             followlinks, sort) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

//
// Here's a parallel version
//
iter walkdirs(path: string=".", topdown=true, depth=max(int), hidden=false,
              followlinks=false, sort=false, param tag: iterKind): string
       where tag == iterKind.standalone {

  if (sort) then
    warning("sorting has no effect for parallel invocations of walkdirs()");

  if (topdown) then
    yield path;

  if (depth) {
    var subdirs = listdir(path, hidden=hidden, files=false, listlinks=followlinks);
    forall subdir in subdirs {
      const fullpath = path + "/" + subdir;
      //
      // Call standalone walkdirs() iterator recursively; set sort=false since it is
      // not useful and we've already printed the warning
      //
      for subdir in walkdirs(fullpath, topdown, depth-1, hidden, followlinks, sort=false, iterKind.standalone) do
        yield subdir;
    }
  }

  if (!topdown) then
    yield path;
}

/* iter glob(pattern="*")

   glob() gives glob() capabilities and is implemented using C's glob()

     * pattern: the glob pattern to match against

   By default, it will list all files/directories in the current directory
*/

module Chpl_rt_glob {
  extern type glob_t;
  extern proc chpl_glob(pattern:c_string, flags: c_int,
                        ref ret_glob:glob_t):c_int;
  extern proc chpl_glob_num(x:glob_t): size_t;
  extern proc chpl_glob_index(x:glob_t, idx:size_t): c_string;
  extern proc globfree(ref glb:glob_t);
}

iter glob(pattern="*") {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    for i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.standalone {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (num) then
    forall i in 0..num-1 do
      yield Chpl_rt_glob.chpl_glob_index(glb, i): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: The following leader/follower iterator assumes that
// the file system state won't change between calls because
// it calls the glob routines twice and assumes the same
// result will come back.  In leader-follower 2.0, we would
// like to store such state between calls in which case this
// should be rewritten to do so (and would require freeing
// the state at the end of the call).
//
iter glob(pattern:string="*", param tag: iterKind)
       where tag == iterKind.leader {
  var glb : Chpl_rt_glob.glob_t;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  //
  // cast is used here to ensure we create an int-based leader
  //
  const num = Chpl_rt_glob.chpl_glob_num(glb).safeCast(int);
  if (num) {
    Chpl_rt_glob.globfree(glb);

    //
    // Forward to the range type's leader
    //
    for followThis in (0..num-1).these(tag) do
      yield followThis;
  }
}

iter glob(pattern:string="*", followThis, param tag: iterKind)
       where tag == iterKind.follower {
  var glb : Chpl_rt_glob.glob_t;
  if (followThis.size != 1) then
    compilerError("glob() iterator can only be zipped with 1D iterators");
  var (r,) = followThis;

  const err = Chpl_rt_glob.chpl_glob(pattern:c_string, 0, glb);
  const num = Chpl_rt_glob.chpl_glob_num(glb);
  if (r.high > num.safeCast(int)) then
    halt("glob() iterator zipped with something too big");
  if (num) then
    for i in r do
      //
      // safe cast is used here to turn an int into a size_t
      //
      yield Chpl_rt_glob.chpl_glob_index(glb, i.safeCast(size_t)): string;

  Chpl_rt_glob.globfree(glb);
}

//
// TODO: It'd be nice if we could do a leader-follower version of
// glob() -- and we almost could -- except that the glob_t state
// has to be communicated from leader to follower.
//

/* iter findfiles(startdir = ".", recursive=false, hidden=false)

   findfiles() is a simple find-like utility implemented using the
   above routines

     * startdir: where to start when looking for files
     * recursive: tells whether or not to descend recurisvely
     * hidden: tells whether or not to yield hidden
*/

iter findfiles(startdir = ".", recursive=false, hidden=false) {
  if (recursive) then
    for subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}

iter findfiles(startdir = ".", recursive=false, hidden=false, param tag: iterKind) where tag == iterKind.standalone {
  if (recursive) then
    forall subdir in walkdirs(startdir, hidden=hidden) do
      for file in listdir(subdir, hidden=hidden, dirs=false, files=true, listlinks=true) do
        yield subdir+"/"+file;
  else
    for file in listdir(startdir, hidden=hidden, dirs=false, files=true, listlinks=false) do
      yield startdir+"/"+file;
}