Clean up a few bugs in the bytes implementation

A few functions were renamed but not updated in the bytes implementation in
Buffers.chpl. Give them the right names.

A bytes constructor was using the wrong name for one of its fields.  It was
also assigning c_nil to the same field with type "qbytes_ptr_t". Fix the name
and change the nil assignment to use QBYTES_PTR_NULL.

[Reviewed by mppf]

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20917 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module BufferInternals {
  use SysBasic;
  use Error;

  extern type qbytes_ptr_t;
  extern type qbuffer_ptr_t;
  extern type qbuffer_iter_t;
  extern const QBYTES_PTR_NULL:qbytes_ptr_t;
  extern const QBUFFER_PTR_NULL:qbuffer_ptr_t;

  extern proc qbytes_retain(qb:qbytes_ptr_t);
  extern proc qbytes_release(qb:qbytes_ptr_t);
  extern proc qbytes_len(qb:qbytes_ptr_t):int(64);
  extern proc qbytes_data(qb:qbytes_ptr_t):c_ptr;

  extern proc qbytes_create_iobuf(inout ret:qbytes_ptr_t):syserr;
  extern proc qbytes_create_calloc(inout ret:qbytes_ptr_t, len:int(64)):syserr;

  extern proc qbuffer_iter_null():qbuffer_iter_t;

  /*
  extern proc qbuffer_init(inout buf:qbuffer_t):syserr;
  extern proc qbuffer_destroy(inout buf:qbuffer_t);
  */
  extern proc qbuffer_create(inout buf:qbuffer_ptr_t):syserr;
  extern proc qbuffer_retain(buf:qbuffer_ptr_t);
  extern proc qbuffer_release(buf:qbuffer_ptr_t);

  extern proc qbuffer_append(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):syserr;
  extern proc qbuffer_append_buffer(buf:qbuffer_ptr_t, src:qbuffer_ptr_t, src_start:qbuffer_iter_t, src_end:qbuffer_iter_t):syserr;
  extern proc qbuffer_prepend(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):syserr;
  extern proc qbuffer_flatten(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout bytes_out):syserr;
  //extern proc qbuffer_clone(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout buf_out:qbuffer_t):syserr;
  extern proc qbuffer_copyout(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):syserr;
  extern proc qbuffer_copyin(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):syserr;

  extern proc qbuffer_begin(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_end(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_iter_next_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_prev_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_advance(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t, amt:int(64));

  extern proc qbuffer_iter_get(it: qbuffer_iter_t, end:qbuffer_iter_t,
                                inout bytes_out:qbytes_ptr_t,
                                inout skip_out:int(64),
                                inout len_out:int(64));
  extern proc qbuffer_iter_num_bytes(start:qbuffer_iter_t, end:qbuffer_iter_t):int(64);

  extern proc debug_print_qbuffer_iter(inout it:qbuffer_iter_t);

  extern proc qbuffer_start_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_end_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_reposition(buf:qbuffer_ptr_t, new_offset_start:int(64));

  extern proc bulk_get_bytes(src_locale:int, src_addr:qbytes_ptr_t):qbytes_ptr_t;

  extern proc bulk_put_buffer(dst_locale:int, dst_addr:c_ptr, dst_len:int(64), buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t):syserr;

}

module Buffers {
  use BufferInternals; // TODO -- non-exporting use

  // Now define the Chapel types using the originals..
  record bytes {
    var home: locale;
    var _bytes_internal:qbytes_ptr_t;
  }

  record buffer_iterator {
    var home: locale;
    var _bufit_internal:qbuffer_iter_t;
  }
  record buffer_range {
    var start:buffer_iterator;
    var end:buffer_iterator;
  }

  proc buffer_range.len:int(64) {
    var ret:int(64);
    on this.start.home {
      ret = qbuffer_iter_num_bytes(this.start._bufit_internal,
                                   this.end._bufit_internal);
    }
    return ret;
  }

  // FUTURE -- we could create const buffers
  // with a boolean type param
  record buffer {
    var home:locale;
    var _buf_internal:qbuffer_ptr_t;
  }

  // bytes methods.
  proc bytes.bytes() {
    this._bytes_internal = QBYTES_PTR_NULL;
    this.home = here;
  }
  proc bytes.bytes(len:int(64), out error:syserr) {
    error = qbytes_create_calloc(this._bytes_internal, len);
    this.home = here;
  }
  proc bytes.bytes(len:int(64)) {
    var error:syserr = ENOERR;
    error = qbytes_create_calloc(this._bytes_internal, len);
    if error then ioerror(error, "in bytes constructor");
    this.home = here;
  }

  proc create_iobuf(out error:syserr):bytes {
    var ret:bytes;
    error = qbytes_create_iobuf(ret._bytes_internal);
    ret.home = here;
    return ret;
  }
  proc create_iobuf():bytes {
    var err:syserr = ENOERR;
    var ret = create_iobuf(err);
    if err then ioerror(err, "in create_iobuf");
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: bytes) {
    if x.home == here {
      qbytes_retain(x._bytes_internal);
      return x;
    } else {
      var ret:bytes;
      ret.home = here;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home.id, x._bytes_internal);
      return ret;
    }
  }

  proc =(ret:bytes, x:bytes) {
    // retain -- release
    if( x.home == here ) {
      on x.home {
        qbytes_retain(x._bytes_internal);
      }

      on ret.home {
        qbytes_release(ret._bytes_internal);
      }

      ret.home = x.home;
      ret._bytes_internal = x._bytes_internal;
    } else {
      on ret.home {
        qbytes_release(ret._bytes_internal);
      }
      ret.home = here;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home.id, x._bytes_internal);
    }
    return ret;
  }

  proc bytes.~bytes() {
    on this.home {
      qbytes_release(this._bytes_internal);
      this._bytes_internal = QBYTES_PTR_NULL;
    }
  }
  proc bytes.len:int(64) {
    var ret:int(64);
    on home {
     ret = qbytes_len(this._bytes_internal);
    }
    return ret;
  }

  // buffer iterator
  proc buffer_iterator.buffer_iterator() {
    this.home = here;
    this._bufit_internal = qbuffer_iter_null();
  }
  /* don't believe this is necessary...
  proc chpl__initCopy(x: buffer_iterator) {
    return x;
  }
  */

  // buffer methods.
  proc buffer.buffer(out error:syserr) {
    this.home = here;
    error = qbuffer_create(this._buf_internal);
  }
  proc buffer.buffer() {
    var error:syserr = ENOERR;
    this.home = here;
    error = qbuffer_create(this._buf_internal);
    if error then ioerror(error, "in buffer constructor");
  }

  // buffer.flatten is special in that it creates a
  // bytes object wherever it is called (and communicates it
  // intelligently). It is use in buffer's initCopy.
  proc buffer.flatten(range:buffer_range, out error:syserr):bytes {
    var ret:bytes;
    if this.home == here {
      error = qbuffer_flatten(this._buf_internal, range.start._bufit_internal, range.end._bufit_internal, ret);
    } else {
      var dst_locale = here;
      var dst_len:int(64) = range.len;
      ret = new bytes(dst_len);
      var dst_addr = qbytes_data(ret._bytes_internal);
      on this.home {
        // Copy the buffer to the bytes...
        error = bulk_put_buffer(dst_locale.id, dst_addr, dst_len,
                                this._buf_internal,
                                range.start._bufit_internal,
                                range.end._bufit_internal);
      }
    }
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: buffer) {
    if x.home == here {
      qbuffer_retain(x._buf_internal);
      return x;
    } else {
      var ret:buffer;
      var start_offset:int(64);
      var end_offset:int(64);

      on x.home {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here.id;

      on x.home {
        var err:syserr = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer init copy");
      }

      ret.append(b);
      return ret;
    }
  }

  proc =(ret:buffer, x:buffer) {
    ret.home = here;
    // retain -- release
    if( x.home == ret.home ) {
      on x.home {
        qbuffer_retain(x._buf_internal);
      }

      on ret.home {
        qbuffer_release(ret._buf_internal);
      }

      ret.home = x.home;
      ret._buf_internal = x._buf_internal;
    } else {
      on ret.home {
        qbuffer_release(ret._buf_internal);
      }

      var start_offset:int(64);
      var end_offset:int(64);

      on x.home {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here.id;

      on x.home {
        var err:syserr = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer assignment");
      }

      ret.append(b);
    }
  }

  proc buffer.~buffer() {
    on this.home {
      qbuffer_release(this._buf_internal);
      this._buf_internal = QBUFFER_PTR_NULL;
    }
  }

  proc buffer.len {
    var ret:int(64);
    on this.home {
      ret = qbuffer_len(this._buf_internal);
    }
    return ret;
  }

  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, out error:syserr) {
    on this.home {
      error = qbuffer_append(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:syserr = ENOERR;
    this.append(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.append(buf:buffer, part:buffer_range = buf.all(), out error:syserr) {
    on this.home {
      error = qbuffer_append_buffer(this._buf_internal, buf._buf_internal, part.start._bufit_internal, part.end._bufit_internal);
    }
  }
  proc buffer.append(buf:buffer, part:buffer_range = buf.all()) {
    var err:syserr = ENOERR;
    this.append(buf, part, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, out error:syserr) {
    on this.home {
      qbuffer_prepend(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:syserr = ENOERR;
    this.prepend(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.prepend");
  }

  proc buffer.start():buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      ret._bufit_internal = qbuffer_begin(this._buf_internal);
    }
    return ret;
  }
  proc buffer.end():buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      ret._bufit_internal = qbuffer_end(this._buf_internal);
    }
    return ret;
  }
  proc buffer.all():buffer_range {
    return new buffer_range(this.start(), this.end());
  }

  proc buffer.next_part(inout it:buffer_iterator) {
    on this.home {
      var x = it;
      qbuffer_iter_next_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.prev_part(inout it:buffer_iterator) {
    on this.home {
      var x = it;
      qbuffer_iter_prev_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.advance(inout it:buffer_iterator, amount:int(64)) {
    on this.home {
      var x = it;
      qbuffer_iter_advance(this._buf_internal, x._bufit_internal, amount);
      it = x;
    }
  }

  proc buffer_iterator.debug_print()
  {
    on this.home {
      debug_print_qbuffer_iter(this._bufit_internal);
    }
  }

  /*
  proc buffer.clone(range:buffer_range, err:ErrorHandler = nil):buffer {
    on this.home {
      var ret:buffer;
      seterr(qbuffer_clone(this._buf_internal, range.start, range.end, ret._buf_internal));
      return ret;
    }
  }*/

  // methods to read/write basic types.
  proc buffer.copyout(it:buffer_iterator, out value, out error:syserr):buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      var end:buffer_iterator = it;
      var tmp:value.type;
      var sz = numBytes(value.type);
      this.advance(end, sz);
      error = qbuffer_copyout(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      value = tmp;
      ret = end;
    }
    return ret;
  }
  proc buffer.copyout(it:buffer_iterator, out value):buffer_iterator {
    var err:syserr = ENOERR;
    this.copyout(it, value, err);
    if err then ioerror(err, "in buffer.copyout");
  }

  proc buffer.copyin( it:buffer_iterator, value, out error:syserr):buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      //writeln("iterator on way in");
      //debug_print_qbuffer_iter(it._bufit_internal);
      var end = it;
      var tmp = value;
      var sz = numBytes(value.type);
      //writeln("iterator copied to end");
      //debug_print_qbuffer_iter(end._bufit_internal);

      //writeln("sz is ", sz);
      this.advance(end, sz);
      //debug_print_qbuffer_iter(end._bufit_internal);
      error = qbuffer_copyin(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      ret = end;
    }
    return ret;
  }
  proc buffer.copyin( it:buffer_iterator, value):buffer_iterator {
    var err:syserr = ENOERR;
    this.copyin(it, value, err);
    if err then ioerror(err, "in buffer.copyin");
  }
}

module BufferInternals {
  use SysBasic;
  use Error;

  extern type qbytes_ptr_t;
  extern type qbuffer_ptr_t;
  extern type qbuffer_iter_t;
  extern const QBYTES_PTR_NULL:qbytes_ptr_t;
  extern const QBUFFER_PTR_NULL:qbuffer_ptr_t;

  extern proc qbytes_retain(qb:qbytes_ptr_t);
  extern proc qbytes_release(qb:qbytes_ptr_t);
  extern proc qbytes_len(qb:qbytes_ptr_t):int(64);
  extern proc qbytes_data(qb:qbytes_ptr_t):c_ptr;

  extern proc qbytes_create_iobuf(inout ret:qbytes_ptr_t):syserr;
  extern proc qbytes_create_calloc(inout ret:qbytes_ptr_t, len:int(64)):syserr;

  extern proc qbuffer_iter_null():qbuffer_iter_t;

  /*
  extern proc qbuffer_init(inout buf:qbuffer_t):syserr;
  extern proc qbuffer_destroy(inout buf:qbuffer_t);
  */
  extern proc qbuffer_create(inout buf:qbuffer_ptr_t):syserr;
  extern proc qbuffer_retain(buf:qbuffer_ptr_t);
  extern proc qbuffer_release(buf:qbuffer_ptr_t);

  extern proc qbuffer_append(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):syserr;
  extern proc qbuffer_append_buffer(buf:qbuffer_ptr_t, src:qbuffer_ptr_t, src_start:qbuffer_iter_t, src_end:qbuffer_iter_t):syserr;
  extern proc qbuffer_prepend(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):syserr;
  extern proc qbuffer_flatten(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout bytes_out):syserr;
  //extern proc qbuffer_clone(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout buf_out:qbuffer_t):syserr;
  extern proc qbuffer_copyout(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):syserr;
  extern proc qbuffer_copyin(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):syserr;

  extern proc qbuffer_begin(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_end(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_iter_next_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_prev_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_advance(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t, amt:int(64));

  extern proc qbuffer_iter_get(it: qbuffer_iter_t, end:qbuffer_iter_t,
                                inout bytes_out:qbytes_ptr_t,
                                inout skip_out:int(64),
                                inout len_out:int(64));
  extern proc qbuffer_iter_num_bytes(start:qbuffer_iter_t, end:qbuffer_iter_t):int(64);

  extern proc debug_print_qbuffer_iter(inout it:qbuffer_iter_t);

  extern proc qbuffer_start_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_end_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_reposition(buf:qbuffer_ptr_t, new_offset_start:int(64));

  extern proc bulk_get_bytes(src_locale:int, src_addr:qbytes_ptr_t):qbytes_ptr_t;

  extern proc bulk_put_buffer(dst_locale:int, dst_addr:c_ptr, dst_len:int(64), buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t):syserr;

}

module Buffers {
  use BufferInternals; // TODO -- non-exporting use

  // Now define the Chapel types using the originals..
  record bytes {
    var home: locale;
    var _bytes_internal:qbytes_ptr_t;
  }

  record buffer_iterator {
    var home: locale;
    var _bufit_internal:qbuffer_iter_t;
  }
  record buffer_range {
    var start:buffer_iterator;
    var end:buffer_iterator;
  }

  proc buffer_range.len:int(64) {
    var ret:int(64);
    on this.start.home {
      ret = qbuffer_iter_num_bytes(this.start._bufit_internal,
                                   this.end._bufit_internal);
    }
    return ret;
  }

  // FUTURE -- we could create const buffers
  // with a boolean type param
  record buffer {
    var home:locale;
    var _buf_internal:qbuffer_ptr_t;
  }

  // bytes methods.
  proc bytes.bytes() {
    this._internal = c_nil;
    this.home = here;
  }
  proc bytes.bytes(len:int(64), out error:syserr) {
    error = qbytes_create_calloc(this._bytes_internal, len);
    this.home = here;
  }
  proc bytes.bytes(len:int(64)) {
    var error:syserr = ENOERR;
    error = qbytes_create_calloc(this._bytes_internal, len);
    if error then ioerror(error, "in bytes constructor");
    this.home = here;
  }

  proc create_iobuf(out error:syserr):bytes {
    var ret:bytes;
    error = qbytes_create_iobuf(ret._bytes_internal);
    ret.home = here;
    return ret;
  }
  proc create_iobuf():bytes {
    var err:syserr = ENOERR;
    var ret = create_iobuf(err);
    if err then ioerror(err, "in create_iobuf");
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: bytes) {
    if x.home == here {
      qbytes_retain(x._bytes_internal);
      return x;
    } else {
      var ret:bytes;
      ret.home = here;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home.id, x._bytes_internal);
      return ret;
    }
  }

  proc =(ret:bytes, x:bytes) {
    // retain -- release
    if( x.home == here ) {
      on x.home {
        qio_bytes_retain(x._bytes_internal);
      }

      on ret.home {
        qio_bytes_release(ret._bytes_internal);
      }

      ret.home = x.home;
      ret._bytes_internal = x._bytes_internal;
    } else {
      on ret.home {
        qio_bytes_release(ret._bytes_internal);
      }
      ret.home = here;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home.id, x._bytes_internal);
    }
    return ret;
  }

  proc bytes.~bytes() {
    on this.home {
      qbytes_release(this._bytes_internal);
      this._bytes_internal = QBYTES_PTR_NULL;
    }
  }
  proc bytes.len:int(64) {
    var ret:int(64);
    on home {
     ret = qbytes_len(this._bytes_internal);
    }
    return ret;
  }

  // buffer iterator
  proc buffer_iterator.buffer_iterator() {
    this.home = here;
    this._bufit_internal = qbuffer_iter_null();
  }
  /* don't believe this is necessary...
  proc chpl__initCopy(x: buffer_iterator) {
    return x;
  }
  */

  // buffer methods.
  proc buffer.buffer(out error:syserr) {
    this.home = here;
    error = qbuffer_create(this._buf_internal);
  }
  proc buffer.buffer() {
    var error:syserr = ENOERR;
    this.home = here;
    error = qbuffer_create(this._buf_internal);
    if error then ioerror(error, "in buffer constructor");
  }

  // buffer.flatten is special in that it creates a
  // bytes object wherever it is called (and communicates it
  // intelligently). It is use in buffer's initCopy.
  proc buffer.flatten(range:buffer_range, out error:syserr):bytes {
    var ret:bytes;
    if this.home == here {
      error = qbuffer_flatten(this._buf_internal, range.start._bufit_internal, range.end._bufit_internal, ret);
    } else {
      var dst_locale = here;
      var dst_len:int(64) = range.len;
      ret = new bytes(dst_len);
      var dst_addr = qbytes_data(ret._bytes_internal);
      on this.home {
        // Copy the buffer to the bytes...
        error = bulk_put_buffer(dst_locale.id, dst_addr, dst_len,
                                this._buf_internal,
                                range.start._bufit_internal,
                                range.end._bufit_internal);
      }
    }
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: buffer) {
    if x.home == here {
      qbuffer_retain(x._buf_internal);
      return x;
    } else {
      var ret:buffer;
      var start_offset:int(64);
      var end_offset:int(64);

      on x.home {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here.id;

      on x.home {
        var err:syserr = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer init copy");
      }

      ret.append(b);
      return ret;
    }
  }

  proc =(ret:buffer, x:buffer) {
    ret.home = here;
    // retain -- release
    if( x.home == ret.home ) {
      on x.home {
        qbuffer_retain(x._buf_internal);
      }

      on ret.home {
        qbuffer_release(ret._buf_internal);
      }

      ret.home = x.home;
      ret._buf_internal = x._buf_internal;
    } else {
      on ret.home {
        qbuffer_release(ret._buf_internal);
      }

      var start_offset:int(64);
      var end_offset:int(64);

      on x.home {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here.id;

      on x.home {
        var err:syserr = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer assignment");
      }

      ret.append(b);
    }
  }

  proc buffer.~buffer() {
    on this.home {
      qbuffer_release(this._buf_internal);
      this._buf_internal = QBUFFER_PTR_NULL;
    }
  }

  proc buffer.len {
    var ret:int(64);
    on this.home {
      ret = qbuffer_len(this._buf_internal);
    }
    return ret;
  }

  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, out error:syserr) {
    on this.home {
      error = qbuffer_append(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:syserr = ENOERR;
    this.append(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.append(buf:buffer, part:buffer_range = buf.all(), out error:syserr) {
    on this.home {
      error = qbuffer_append_buffer(this._buf_internal, buf._buf_internal, part.start._bufit_internal, part.end._bufit_internal);
    }
  }
  proc buffer.append(buf:buffer, part:buffer_range = buf.all()) {
    var err:syserr = ENOERR;
    this.append(buf, part, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, out error:syserr) {
    on this.home {
      qbuffer_prepend(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:syserr = ENOERR;
    this.prepend(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.prepend");
  }

  proc buffer.start():buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      ret._bufit_internal = qbuffer_begin(this._buf_internal);
    }
    return ret;
  }
  proc buffer.end():buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      ret._bufit_internal = qbuffer_end(this._buf_internal);
    }
    return ret;
  }
  proc buffer.all():buffer_range {
    return new buffer_range(this.start(), this.end());
  }

  proc buffer.next_part(inout it:buffer_iterator) {
    on this.home {
      var x = it;
      qbuffer_iter_next_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.prev_part(inout it:buffer_iterator) {
    on this.home {
      var x = it;
      qbuffer_iter_prev_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.advance(inout it:buffer_iterator, amount:int(64)) {
    on this.home {
      var x = it;
      qbuffer_iter_advance(this._buf_internal, x._bufit_internal, amount);
      it = x;
    }
  }

  proc buffer_iterator.debug_print()
  {
    on this.home {
      debug_print_qbuffer_iter(this._bufit_internal);
    }
  }

  /*
  proc buffer.clone(range:buffer_range, err:ErrorHandler = nil):buffer {
    on this.home {
      var ret:buffer;
      seterr(qbuffer_clone(this._buf_internal, range.start, range.end, ret._buf_internal));
      return ret;
    }
  }*/

  // methods to read/write basic types.
  proc buffer.copyout(it:buffer_iterator, out value, out error:syserr):buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      var end:buffer_iterator = it;
      var tmp:value.type;
      var sz = numBytes(value.type);
      this.advance(end, sz);
      error = qbuffer_copyout(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      value = tmp;
      ret = end;
    }
    return ret;
  }
  proc buffer.copyout(it:buffer_iterator, out value):buffer_iterator {
    var err:syserr = ENOERR;
    this.copyout(it, value, err);
    if err then ioerror(err, "in buffer.copyout");
  }

  proc buffer.copyin( it:buffer_iterator, value, out error:syserr):buffer_iterator {
    var ret:buffer_iterator;
    ret.home = this.home;
    on this.home {
      //writeln("iterator on way in");
      //debug_print_qbuffer_iter(it._bufit_internal);
      var end = it;
      var tmp = value;
      var sz = numBytes(value.type);
      //writeln("iterator copied to end");
      //debug_print_qbuffer_iter(end._bufit_internal);

      //writeln("sz is ", sz);
      this.advance(end, sz);
      //debug_print_qbuffer_iter(end._bufit_internal);
      error = qbuffer_copyin(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      ret = end;
    }
    return ret;
  }
  proc buffer.copyin( it:buffer_iterator, value):buffer_iterator {
    var err:syserr = ENOERR;
    this.copyin(it, value, err);
    if err then ioerror(err, "in buffer.copyin");
  }
}


Rearranged a comment for better English flow.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@23121 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// This example shows how to use leader-follower iterators in Chapel.
// leader-follower iterators are used to implement zippered (and
// currently, non-zippered) forall loops in Chapel over data
// structures or standalone iterators.  We expect the leader-follower
// interface to change and improve over time, so this should be
// considered a snapshot of their use in the current implementation.
//
// For a more thorough introduction to leader-follower iterators,
// refer to our PGAS 2011 paper, "User-Defined Parallel Zippered
// Iterators in Chapel".
//

//
// Any zippered forall loop in Chapel will be implemented using
// leader-follower iterators.  Generally speaking, a forall loop
// of the following form:
//
//   forall (a, b, c) in zip(A, B, C) do
//     ...loop body...
//
// is semantically defined such that the first thing being iterated
// over -- in this case, A -- is designated the 'leader.'  All things
// being iterated over are 'followers' (so for this loop, A, B, and C
// would be).
//
// Given such a loop, the compiler will roughly translate it into
// the following loop structure:
//
//   for work in A.lead() do   // implemented by inlining the leader
//     for (a, b, c) in zip(A.follow(work), B.follow(work), C.follow(work)) do
//       ...loop body...
//
// where .lead() and .follow() represent the leader-follower iterators
// using a simplified naming scheme.
//
// Note that since Chapel's implicitly parallel semantics are defined
// in terms of zippered iteration, such cases are also implemented using
// leader-follower iterators.  For example:
//
//   A = B + C;
//
// and:
//
//   foo(A, B, C)
//
// (where foo() is defined to take scalar arguments and is promoted
// in this function call) will be converted to their equivalent
// zippered parallel loops and then to the leader-follower idiom shown
// above.

//
// At a high level, the role of a leader iterator is to:
//
//   (a) create the parallel tasks used to implement the forall loop,
//
//   (b) associate the parallel tasks with specific locales as
//       required/desired.
//
//   (c) assign work (e.g., iterations) to each parallel task
//

// The leader typically creates the parallelism using task parallel
// features like coforall loops; and it associates tasks with locales
// using locality features like on-clauses.  The leader specifies work
// for tasks by having each task it creates yield some representation
// of the work it owns.
//
// The role of the follower iterator is to take as an input argument
// a chunk of work (as yielded by a leader) and to serially iterate
// over and yield the elements/values corresponding to those
// iterations in order.
//

// Let's go to some code:

//
// For this example, we're going to create a simple iterator named
// 'count' that will be able to be invoked in 'for' or 'forall' loops.
// Count will be defined to take an argument 'n' as input and an
// optional argument 'low' (set to 1 by default), and it will yield n
// integers starting with low.
//

//
// In this example, we'll use the following config const 'numTasks' to
// indicate the degree of parallelism to use in the leader to
// implement forall loops.  By default we've set it to the number of
// cores on the current locale, but it can be overridden on the
// executable command-line using the --numTasks=<n> option.
//
config const numTasks = here.numCores;

//
// numTasks must be at least 1 for this to make sense
//
if (numTasks < 1) then
  halt("numTasks must be a positive integer");

//
// If compiled with verbose set to true, the leader and follower will
// print out some messages indicating what they're doing under the
// covers.
//
config param verbose = false;

//
// Declare a problem size for this test.  By default we use a small
// problem size to make the output readable.  Of course, to use the
// parallelism effectively you'd want to use a much larger problem
// size (override on the execution command-line using the
// --probSize=<n> option).
//
config const probSize = 15;

//
// declare an array to work with
//
var A: [1..probSize] real;

//
// when defining a leader-follower iterator pair, our current
// implementation requires that you also define a serial iterator
// of the same name and that it yields the same type as the follower
// iterator does.  In this case, the serial iterator is simple:  We
// simply yield the integers low..low+n-1 (computed using the count
// operator, #):
//
iter count(n: int, low: int=1) {
  for i in low..#n do
    yield i;
}

//
// Here are some simple loops using this iterator to demonstrate
// it.  First we iterate over all indices in our problem size to
// initialize A:
//

for i in count(probSize) do
  A[i] = i:real;

writeln("After serial initialization, A is:");
writeln(A);
writeln();

//
// Then we override the default value of low in order to negate
// the "middle" elements of A:
//
for i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A:");
writeln(A);
writeln();

//
// For serial zippered iteration, nothing is required other than
// this single iterator:
//

for (i, a) in zip(count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using zippering:");
writeln(A);
writeln();

//
// The leader and follower iterators are defined as overloads of the
// serial version of the iterator, distinguished by an initial
// param argument of the built-in enumerated type 'iterKind'.  To
// invoke the leader iterator and differentiate it from the other
// overloads, the compiler will pass in the value 'iterKind.leader' to
// this argument.  The author of the leader iterator should use a
// 'where' clause to distinguish this overload from the others.  After
// this 'tag' argument, the rest of the argument list should match
// that of the serial iterator exactly.  For our example, this means
// providing the same n and low arguments as before.
//
// The implementation of this leader iterator is relatively simple and
// static.  It uses a coforall loop to create a number of tasks equal
// to the number specified by our 'numTasks' config const and then has
// each yield a subset of the total work to perform.
//
// We compute the work that a task should yield by calling into the
// computeChunk() helper function (defined at the bottom of this file)
// to compute its subrange of the range low..#n owned by the task,
// storing it in a variable called 'myIters.'
//
// To be a legal leader iterator, we could simply yield this range as
// a representation of the work we want the follower to perform.
// However, to support zippering our leader with follower iterators
// written by others, we typically take the convention of having
// iterators over 1D or dense rectangular index spaces yield tuples
// of ranges shifted to a 0-based coordinate system.  In this way, the
// leader-follower iterators have a common representation for the
// work even though each may use its own indexing system.  This
// permits, for example, arrays of the same size/shape to be zippered
// together even if they have different domains.
//
// To this end, rather than yielding subranges of low..#n, we'll yield
// subranges of 0..n-1 and rely on the follower to shift it back to
// the original coordinate system.  For this reason, we translate the
// range by -low to shift it from low-based coordinates to 0-based
// coordinates; and then we make a 1-tuple out of it.
//
// Note the debugging output inserted into this iterator.  While
// learning about leader-follower iterators, it's useful to turn
// this debugging output on by compiling  with -sverbose=true
//
iter count(param tag: iterKind, n: int, low: int=1)
       where tag == iterKind.leader {

  if (verbose) then
    writeln("In count() leader, creating ", numTasks, " tasks");

  coforall tid in 0..#numTasks {
    const myIters = computeChunk(low..#n, tid, numTasks);
    const zeroBasedIters = myIters.translate(-low);

    if (verbose) then
      writeln("task ", tid, " owns ", myIters, " yielded as: ", zeroBasedIters);

    yield (zeroBasedIters,);
  }
}
//
// As mentioned at the outset, this leader is fairly static and
// simple.  More generally, a leader can introduce tasks more
// dynamically, partition work between the tasks more dynamically,
// etc.  See $CHPL_HOME/modules/standard/AdvancedIters.chpl for
// some more interesting examples of leader iterators, including
// those that use dynamic partitioning.
//

//
// The follower is another overload of the same iterator name, this
// time taking the iterKind.follower param enumeration as its first
// argument.  The next arguments should match the leader and serial
// iterators exactly again (so, n and low for our example).  The
// final argument must be called 'followThis' which represents the data
// yielded by the leader (in our case, the 1-tuple of ranges).
//
// The goal of the follower is to do the iteration specified by the
// 'followThis' argument, serially yielding the elements corresponding
// to those iterations.  In our case, this involves plucking the
// range back out of the tuple of ranges, and shifting it back to
// our low-based coordinate system.  We then use a standard for loop
// to iterate over that range and yield the corresponding indices.
// Followers, as the name suggests, tend not to be very sophisticated,
// and simply do what the leader tells them to.
//
// As with the leader, this follower has been authored to support
// debugging output when compiled with -sverbose=true.
//
iter count(param tag: iterKind, n: int, low: int=1, followThis)
       where tag == iterKind.follower && followThis.size == 1 {
  const lowBasedIters = followThis(1).translate(low);

  if (verbose) then
    writeln("Follower received ", followThis, " as work chunk; shifting to ",
            lowBasedIters);

  for i in lowBasedIters do
    yield i;
}

//
// Now that we've defined leader-follower iterators, we can execute
// the same loops we did before, only this time using forall loops
// to make the execution parallel.  We start with some simple
// invocations as before.  In these invocations, the count() iterator
// serves as the only leader and follower iterator being invoked
// since it is the only thing being iterated over (A is being randomly
// accessed within the loop.
//
forall i in count(probSize) do
  A[i] = i:real;

writeln("After parallel initialization, A is:");
writeln(A);
writeln();

//
// Invoking it again with a different low value:
//
forall i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A in parallel:");
writeln(A);
writeln();

//
// Zippered iteration is now a bit more interesting.  In this
// first loop, count() serves as the leader and follower while
// the A array is a follower.  This works because A is a
// rectangular array whose follower iterator accepts tuples of
// 0-based ranges like the ones count()'s leader is yielding.
// If we were to have count() yield something else (like a raw
// subrange of low..#n), then the two things could not be
// zippered correctly because they wouldn't be speaking the
// same language -- either in terms of the type of work being
// yielded (range vs. 1-tuple of range), nor the description of
// the work (low-based indices vs. 0-based indices).
//
forall (i, a) in zip(count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using parallel zippering:");
writeln(A);
writeln();

//
// We can also zipper in the opposite order, making A the leader,
// in which case count() no longer controls the degree of parallelism
// and work assignment since it is no longer the leader.  Instead,
// A's leader iterator (defined as part of its domain map) is invoked.
// For standard Chapel arrays and domain maps, these leader-follower
// iterators are controlled by the dataPar* configuration constants
// as described in doc/README.executing.
//
forall (a, i) in zip(A, count(probSize)) do
  a = i/100.0;

writeln("After reassigning A using parallel zippering and A as the leader:");
writeln(A);
writeln();

//
// Finally, as mentioned at the outset, operations that are equivalent
// to zippering also use leader-follower iterators, so for example
// the following whole-array assignment will use A's leader and
// count()'s follower:
//
A = count(probSize, low=100);

writeln("After reassigning A using whole-array assignment:");
writeln(A);
writeln();

//
// Some closing notes:
//
// * Chapel data types like records and classes can support iteration
//   by defining iterator methods (invoked by name) or these() iterators
//   which support iterating over variables of that type directly.  Such
//   iterator methods can be overloaded to support leader-follower
//   versions as well to permit parallel iteration over the variable.
//
// * As mentioned at the outset, our leader-follower scheme has a number
//   of improvements planned for it, including the ability to specify
//   a standalone parallel iterator for non-zippered forall loops to
//   avoid the overhead of using a leader-follower pair and better error
//   checking.  We'll update this primer as we improve these features.
//

//
// This is a poor-man's partitioning algorithm.  It gives
// floor(numElements/NumChunks) work items to the first numChunks-1
// chunks and the remainder to the last chunk.  For simplicity it only
// works for non-strided, default index type ranges.  More work would
// be required to generalize it for strided or unbounded ranges.
//
proc computeChunk(r: range, myChunk, numChunks) where r.stridable == false {
  const numElems = r.length;
  const elemsPerChunk = numElems/numChunks;
  const mylow = r.low + elemsPerChunk*myChunk;
  if (myChunk != numChunks - 1) {
    return mylow..#elemsPerChunk;
  } else {
    return mylow..r.high;
  }
}

//
// This example shows how to use leader-follower iterators in Chapel.
// leader-follower iterators are used to implement zippered (and
// currently, non-zippered) forall loops in Chapel over data
// structures or standalone iterators.  We expect the leader-follower
// interface to change and improve over time, so this should be
// considered a snapshot of their use in the current implementation.
//
// For a more thorough introduction to leader-follower iterators,
// refer to our PGAS 2011 paper, "User-Defined Parallel Zippered
// Iterators in Chapel".
//

//
// Any zippered forall loop in Chapel will be implemented using
// leader-follower iterators.  Generally speaking, a forall loop
// of the following form:
//
//   forall (a, b, c) in zip(A, B, C) do
//     ...loop body...
//
// is semantically defined such that the first thing being iterated
// over -- in this case, A -- is designated the 'leader.'  All things
// being iterated over are 'followers' (so for this loop, A, B, and C
// would be).
//
// Given such a loop, the compiler will roughly translate it into
// the following loop structure:
//
//   for work in A.lead() do   // implemented by inlining the leader
//     for (a, b, c) in zip(A.follow(work), B.follow(work), C.follow(work)) do
//       ...loop body...
//
// where .lead() and .follow() represent the leader-follower iterators
// using a simplified naming scheme.
//
// Note that since Chapel's implicitly parallel semantics are defined
// in terms of zippered iteration, such cases are also implemented using
// leader-follower iterators.  For example:
//
//   A = B + C;
//
// and:
//
//   foo(A, B, C)
//
// (where foo() is defined to take scalar arguments and is promoted
// in this function call) will be converted to their equivalent
// zippered parallel loops and then to the leader-follower idiom shown
// above.

//
// At a high level, the role of a leader iterator is to:
//
//   (a) create the parallel tasks used to implement the forall loop,
//
//   (b) associate the parallel tasks with specific locales as
//       required/desired.
//
//   (c) assign work (e.g., iterations) to each parallel task
//

// The leader typically creates the parallelism using task parallel
// features like coforall loops; and it associates tasks with locales
// using locality features like on-clauses.  The leader specifies work
// for tasks by having each task it creates yield some representation
// of the work it owns.
//
// The role of the follower iterator is to take as an input argument
// a chunk of work (as yielded by a leader) and to serially iterate
// over and yield the elements/values corresponding to those
// iterations in order.
//

// Let's go to some code:

//
// For this example, we're going to create a simple iterator named
// 'count' that will be able to be invoked in 'for' or 'forall' loops.
// Count will be defined to take an argument 'n' as input and an
// optional argument 'low' (set to 1 by default), and it will yield n
// integers starting with low.
//

//
// In this example, we'll use the following config const 'numTasks' to
// indicate the degree of parallelism to use in the leader to
// implement forall loops.  By default we've set it to the number of
// cores on the current locale, but it can be overridden on the
// executable command-line using the --numTasks=<n> option.
//
config const numTasks = here.numCores;

//
// numTasks must be at least 1 for this to make sense
//
if (numTasks < 1) then
  halt("numTasks must be a positive integer");

//
// If compiled with verbose set to true, the leader and follower will
// print out some messages indicating what they're doing under the
// covers.
//
config param verbose = false;

//
// Declare a problem size for this test.  By default we use a small
// problem size to make the output readable.  To use the parallelism
// effectively, you'd want to use a much larger problem size of
// course (override on the execution command-line using the
// --probSize=<n> option).
//
config const probSize = 15;

//
// declare an array to work with
//
var A: [1..probSize] real;

//
// when defining a leader-follower iterator pair, our current
// implementation requires that you also define a serial iterator
// of the same name and that it yields the same type as the follower
// iterator does.  In this case, the serial iterator is simple:  We
// simply yield the integers low..low+n-1 (computed using the count
// operator, #):
//
iter count(n: int, low: int=1) {
  for i in low..#n do
    yield i;
}

//
// Here are some simple loops using this iterator to demonstrate
// it.  First we iterate over all indices in our problem size to
// initialize A:
//

for i in count(probSize) do
  A[i] = i:real;

writeln("After serial initialization, A is:");
writeln(A);
writeln();

//
// Then we override the default value of low in order to negate
// the "middle" elements of A:
//
for i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A:");
writeln(A);
writeln();

//
// For serial zippered iteration, nothing is required other than
// this single iterator:
//

for (i, a) in zip(count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using zippering:");
writeln(A);
writeln();

//
// The leader and follower iterators are defined as overloads of the
// serial version of the iterator, distinguished by an initial
// param argument of the built-in enumerated type 'iterKind'.  To
// invoke the leader iterator and differentiate it from the other
// overloads, the compiler will pass in the value 'iterKind.leader' to
// this argument.  The author of the leader iterator should use a
// 'where' clause to distinguish this overload from the others.  After
// this 'tag' argument, the rest of the argument list should match
// that of the serial iterator exactly.  For our example, this means
// providing the same n and low arguments as before.
//
// The implementation of this leader iterator is relatively simple and
// static.  It uses a coforall loop to create a number of tasks equal
// to the number specified by our 'numTasks' config const and then has
// each yield a subset of the total work to perform.
//
// We compute the work that a task should yield by calling into the
// computeChunk() helper function (defined at the bottom of this file)
// to compute its subrange of the range low..#n owned by the task,
// storing it in a variable called 'myIters.'
//
// To be a legal leader iterator, we could simply yield this range as
// a representation of the work we want the follower to perform.
// However, to support zippering our leader with follower iterators
// written by others, we typically take the convention of having
// iterators over 1D or dense rectangular index spaces yield tuples
// of ranges shifted to a 0-based coordinate system.  In this way, the
// leader-follower iterators have a common representation for the
// work even though each may use its own indexing system.  This
// permits, for example, arrays of the same size/shape to be zippered
// together even if they have different domains.
//
// To this end, rather than yielding subranges of low..#n, we'll yield
// subranges of 0..n-1 and rely on the follower to shift it back to
// the original coordinate system.  For this reason, we translate the
// range by -low to shift it from low-based coordinates to 0-based
// coordinates; and then we make a 1-tuple out of it.
//
// Note the debugging output inserted into this iterator.  While
// learning about leader-follower iterators, it's useful to turn
// this debugging output on by compiling  with -sverbose=true
//
iter count(param tag: iterKind, n: int, low: int=1)
       where tag == iterKind.leader {

  if (verbose) then
    writeln("In count() leader, creating ", numTasks, " tasks");

  coforall tid in 0..#numTasks {
    const myIters = computeChunk(low..#n, tid, numTasks);
    const zeroBasedIters = myIters.translate(-low);

    if (verbose) then
      writeln("task ", tid, " owns ", myIters, " yielded as: ", zeroBasedIters);

    yield (zeroBasedIters,);
  }
}
//
// As mentioned at the outset, this leader is fairly static and
// simple.  More generally, a leader can introduce tasks more
// dynamically, partition work between the tasks more dynamically,
// etc.  See $CHPL_HOME/modules/standard/AdvancedIters.chpl for
// some more interesting examples of leader iterators, including
// those that use dynamic partitioning.
//

//
// The follower is another overload of the same iterator name, this
// time taking the iterKind.follower param enumeration as its first
// argument.  The next arguments should match the leader and serial
// iterators exactly again (so, n and low for our example).  The
// final argument must be called 'followThis' which represents the data
// yielded by the leader (in our case, the 1-tuple of ranges).
//
// The goal of the follower is to do the iteration specified by the
// 'followThis' argument, serially yielding the elements corresponding
// to those iterations.  In our case, this involves plucking the
// range back out of the tuple of ranges, and shifting it back to
// our low-based coordinate system.  We then use a standard for loop
// to iterate over that range and yield the corresponding indices.
// Followers, as the name suggests, tend not to be very sophisticated,
// and simply do what the leader tells them to.
//
// As with the leader, this follower has been authored to support
// debugging output when compiled with -sverbose=true.
//
iter count(param tag: iterKind, n: int, low: int=1, followThis)
       where tag == iterKind.follower && followThis.size == 1 {
  const lowBasedIters = followThis(1).translate(low);

  if (verbose) then
    writeln("Follower received ", followThis, " as work chunk; shifting to ",
            lowBasedIters);

  for i in lowBasedIters do
    yield i;
}

//
// Now that we've defined leader-follower iterators, we can execute
// the same loops we did before, only this time using forall loops
// to make the execution parallel.  We start with some simple
// invocations as before.  In these invocations, the count() iterator
// serves as the only leader and follower iterator being invoked
// since it is the only thing being iterated over (A is being randomly
// accessed within the loop.
//
forall i in count(probSize) do
  A[i] = i:real;

writeln("After parallel initialization, A is:");
writeln(A);
writeln();

//
// Invoking it again with a different low value:
//
forall i in count(n=probSize/2, low=probSize/4) do
  A[i] = -A[i];

writeln("After negating the middle of A in parallel:");
writeln(A);
writeln();

//
// Zippered iteration is now a bit more interesting.  In this
// first loop, count() serves as the leader and follower while
// the A array is a follower.  This works because A is a
// rectangular array whose follower iterator accepts tuples of
// 0-based ranges like the ones count()'s leader is yielding.
// If we were to have count() yield something else (like a raw
// subrange of low..#n), then the two things could not be
// zippered correctly because they wouldn't be speaking the
// same language -- either in terms of the type of work being
// yielded (range vs. 1-tuple of range), nor the description of
// the work (low-based indices vs. 0-based indices).
//
forall (i, a) in zip(count(probSize), A) do
  a = i/10.0;

writeln("After reassigning A using parallel zippering:");
writeln(A);
writeln();

//
// We can also zipper in the opposite order, making A the leader,
// in which case count() no longer controls the degree of parallelism
// and work assignment since it is no longer the leader.  Instead,
// A's leader iterator (defined as part of its domain map) is invoked.
// For standard Chapel arrays and domain maps, these leader-follower
// iterators are controlled by the dataPar* configuration constants
// as described in doc/README.executing.
//
forall (a, i) in zip(A, count(probSize)) do
  a = i/100.0;

writeln("After reassigning A using parallel zippering and A as the leader:");
writeln(A);
writeln();

//
// Finally, as mentioned at the outset, operations that are equivalent
// to zippering also use leader-follower iterators, so for example
// the following whole-array assignment will use A's leader and
// count()'s follower:
//
A = count(probSize, low=100);

writeln("After reassigning A using whole-array assignment:");
writeln(A);
writeln();

//
// Some closing notes:
//
// * Chapel data types like records and classes can support iteration
//   by defining iterator methods (invoked by name) or these() iterators
//   which support iterating over variables of that type directly.  Such
//   iterator methods can be overloaded to support leader-follower
//   versions as well to permit parallel iteration over the variable.
//
// * As mentioned at the outset, our leader-follower scheme has a number
//   of improvements planned for it, including the ability to specify
//   a standalone parallel iterator for non-zippered forall loops to
//   avoid the overhead of using a leader-follower pair and better error
//   checking.  We'll update this primer as we improve these features.
//

//
// This is a poor-man's partitioning algorithm.  It gives
// floor(numElements/NumChunks) work items to the first numChunks-1
// chunks and the remainder to the last chunk.  For simplicity it only
// works for non-strided, default index type ranges.  More work would
// be required to generalize it for strided or unbounded ranges.
//
proc computeChunk(r: range, myChunk, numChunks) where r.stridable == false {
  const numElems = r.length;
  const elemsPerChunk = numElems/numChunks;
  const mylow = r.low + elemsPerChunk*myChunk;
  if (myChunk != numChunks - 1) {
    return mylow..#elemsPerChunk;
  } else {
    return mylow..r.high;
  }
}


Removing bugs acknowledging that this test locks in incorrect output

use DimensionalDist2D, ReplicatedDim, BlockDim, BlockCycDim;
use u;

config const s1 = 2;
config const s2 = 2;
setupLocales(s1, s2);

var phase = 0;
proc leapphase() { phase += 20; fphase(phase); }

config const selector = 0;

proc test(message, sel, dd1, dd2) {
  leapphase();

  // ignore 'message'
  hd(dd1.toString(), " + ", dd2.toString(), "  sel=", sel); tl();

  if selector == 0 || selector == sel {
      const dm = new dmap(new DimensionalDist2D(mylocs, dd1, dd2, "dm"));

      const Dbase = {1..3, 1..5};
      const Ddm = Dbase dmapped dm;

      var A: [Ddm] int = [(i,j) in Dbase] i*10+j;
      hd("A: base"); msgserial(A); tl();

      var B: [Ddm by (2,2)] int = [(i,j) in Dbase by (2,2)] i*100+j;
      hd("B: by (2,2)"); msgserial(B); tl();

      const rc1 = 0..5 by 2, rc2 = 2..4;
      var C: [Ddm(rc1,rc2)] int = [(i,j) in Dbase(rc1,rc2)] i*1000+j;
      hd("C: slice", (rc1,rc2)); msgserial(C); tl();

      const rd1 = 0..6 by 3, rd2 = 0..6 by 3 align 2;
      var D: [Ddm(rd1,rd2)] int = [(i,j) in Dbase(rd1,rd2)] i*10+j;
      hd("D: aligned ", (rd1,rd2)); msgserial(D); tl();
//    hd("testing"); msg(Dbase(rd1,rd2)); msgserial(Dbase(rd1,rd2)); tl();

  } else {
    hd("not selected"); tl();
  }
}

test("ReplicatedDim,ReplicatedDim", 1, new ReplicatedDim(s1), new ReplicatedDim(s2));
test("BlockDim,BlockDim", 2, new BlockDim(s1, 1, 4), new BlockDim(s2, 1, 4));
test("BlockCyclicDim,BlockCyclicDim", 3, new BlockCyclicDim(s1, 0, 2), new BlockCyclicDim(s2, 2, 3));

test("ReplicatedDim,BlockDim", 4, new ReplicatedDim(s1),       new BlockDim(s2, 1, 4));
test("ReplicatedDim,BlockCyclicDim", 5, new ReplicatedDim(s1),       new BlockCyclicDim(s2, 2, 3));
test("BlockDim,ReplicatedDim", 6, new BlockDim(s1, 1, 4), new ReplicatedDim(s2));
test("BlockDim,BlockCyclicDim", 7, new BlockDim(s1, 1, 4), new BlockCyclicDim(s2, 2, 3));
test("BlockCyclicDim,ReplicatedDim", 8, new BlockCyclicDim(s1, 0, 2), new ReplicatedDim(s2));
test("BlockCyclicDim,BlockDim", 9, new BlockCyclicDim(s1, 0, 2), new BlockDim(s2, 1, 4));

use DimensionalDist2D, ReplicatedDim, BlockDim, BlockCycDim;
use u;

config const s1 = 2;
config const s2 = 2;
setupLocales(s1, s2);

var phase = 0;
proc leapphase() { phase += 20; fphase(phase); }

config const selector = 0;

proc test(message, sel, dd1, dd2) {
  leapphase();

  // ignore 'message'
  hd(dd1.toString(), " + ", dd2.toString(), "  sel=", sel); tl();

  if selector == 0 || selector == sel {
      const dm = new dmap(new DimensionalDist2D(mylocs, dd1, dd2, "dm"));

      const Dbase = {1..3, 1..5};
      const Ddm = Dbase dmapped dm;

      var A: [Ddm] int = [(i,j) in Dbase] i*10+j;
      hd("A: base"); msgserial(A); tl();

      var B: [Ddm by (2,2)] int = [(i,j) in Dbase by (2,2)] i*100+j;
      hd("B: by (2,2)"); msgserial(B); tl();

// NOTE: .good for this reflect the current bug in domain leader iterator.
// C should have 2002, 2003, 2004.
// See test/domains/vass/aligned-domain-forall.chpl
      const rc1 = 0..5 by 2, rc2 = 2..4;
      var C: [Ddm(rc1,rc2)] int = [(i,j) in Dbase(rc1,rc2)] i*1000+j;
      hd("C: slice", (rc1,rc2)); msgserial(C); tl();

// NOTE: likewise, D should have 32, 35.
      const rd1 = 0..6 by 3, rd2 = 0..6 by 3 align 2;
      var D: [Ddm(rd1,rd2)] int = [(i,j) in Dbase(rd1,rd2)] i*10+j;
      hd("D: aligned ", (rd1,rd2)); msgserial(D); tl();
//    hd("testing"); msg(Dbase(rd1,rd2)); msgserial(Dbase(rd1,rd2)); tl();

  } else {
    hd("not selected"); tl();
  }
}

test("ReplicatedDim,ReplicatedDim", 1, new ReplicatedDim(s1), new ReplicatedDim(s2));
test("BlockDim,BlockDim", 2, new BlockDim(s1, 1, 4), new BlockDim(s2, 1, 4));
test("BlockCyclicDim,BlockCyclicDim", 3, new BlockCyclicDim(s1, 0, 2), new BlockCyclicDim(s2, 2, 3));

test("ReplicatedDim,BlockDim", 4, new ReplicatedDim(s1),       new BlockDim(s2, 1, 4));
test("ReplicatedDim,BlockCyclicDim", 5, new ReplicatedDim(s1),       new BlockCyclicDim(s2, 2, 3));
test("BlockDim,ReplicatedDim", 6, new BlockDim(s1, 1, 4), new ReplicatedDim(s2));
test("BlockDim,BlockCyclicDim", 7, new BlockDim(s1, 1, 4), new BlockCyclicDim(s2, 2, 3));
test("BlockCyclicDim,ReplicatedDim", 8, new BlockCyclicDim(s1, 0, 2), new ReplicatedDim(s2));
test("BlockCyclicDim,BlockDim", 9, new BlockCyclicDim(s1, 0, 2), new BlockDim(s2, 1, 4));


Modified some type declarations to work for a different indexType,
namely int(64) and added a comment for the following restrictions:

- 'indexType' cannot be smaller than int(32) because
  the compiler would not let [1..n, 1..n+1] be a domain(2, indexType).

- 'elemType' can only be real or complex because (a) fillRandom
  allows only real, complex or imag and (b) the assignment Ab = Ab * 2.0 - 1.0
  does not work for imag.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@18002 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of matrices and the element type of those matrices
// indexType can be int or int(64), elemType can be real or complex
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 5;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  const MatVectSpace: domain(2, indexType) = [1..n, 1..n+1],
        MatrixSpace = MatVectSpace[.., ..n];

  var Ab : [MatVectSpace] elemType,  // the matrix A and vector b
      piv: [1..n] indexType,         // a vector of pivot values
      x  : [1..n] elemType;          // the solution vector, x

  var A => Ab[MatrixSpace],          // an alias for the Matrix part of Ab
      b => Ab[.., n+1];              // an alias for the last column of Ab

  initAB(Ab);

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, Ab, piv);                 // compute the LU factorization

  x = backwardSub(n, A, b);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(Ab, MatrixSpace, x);
  printResults(validAnswer, execTime);
}

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
def LUFactorize(n: indexType, Ab: [1..n, 1..n+1] elemType,
                piv: [1..n] indexType) {
  const AbD = Ab.domain;    // alias Ab.domain to save typing

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(Ab, l, piv);
    if (tr.numIndices > 0) then
      updateBlockRow(Ab, tl, tr);

    //
    // update trailing submatrix (if any)
    //
    if (br.numIndices > 0) then
      schurComplement(Ab, blk);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The vertex labeled [1] is location
//     |aaaaa|.....|.....|.....|  (ptOp, ptOp) in the code below.
//     +-----+-----+-----+-----+
//     |aaaaa|.....|.....|.....|  The vertex labeled [2] is location
//     |aaaaa|.....|.....|.....|  (ptSol, ptSol)
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
def schurComplement(Ab: [1..n, 1..n+1] elemType, ptOp: indexType) {
  const AbD = Ab.domain;

  //
  // Calculate location of ptSol (see diagram above)
  //
  const ptSol = ptOp+blkSize;

  //
  // Copy data into replicated array so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.  These
  // replicated distributions aren't implemented yet, but imagine that
  // they look something like the following:
  //
  //var replAbD: domain(2)
  //            dmapped new Dimensional(BlkCyc(blkSize), Replicated))
  //          = AbD[ptSol.., 1..#blkSize];
  //
  const replAD: domain(2, indexType) = AbD[ptSol.., ptOp..#blkSize],
        replBD: domain(2, indexType) = AbD[ptOp..#blkSize, ptSol..];

  const replA : [replAD] elemType = Ab[ptSol.., ptOp..#blkSize],
        replB : [replBD] elemType = Ab[ptOp..#blkSize, ptSol..];

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in AbD[ptSol.., ptSol..] by (blkSize, blkSize) {
    //
    // At this point, the dgemms should all be local, so assert that
    // fact
    //
    local {
      const aBlkD = replAD[row..#blkSize, ptOp..#blkSize],
            bBlkD = replBD[ptOp..#blkSize, col..#blkSize],
            cBlkD = AbD[row..#blkSize, col..#blkSize];

      dgemm(aBlkD.dim(1).length,
            aBlkD.dim(2).length,
            bBlkD.dim(2).length,
            replA(aBlkD),
            replB(bBlkD),
            Ab(cBlkD));
    }
  }
}

//
// calculate C = C - A * B.
//
def dgemm(p: indexType,       // number of rows in A
          q: indexType,       // number of cols in A, number of rows in B
          r: indexType,       // number of cols in B
          A: [1..p, 1..q] ?t,
          B: [1..q, 1..r] t,
          C: [1..p, 1..r] t) {
  // Calculate (i,j) using a dot product of a row of A and a column of B.
  for i in 1..p do
    for j in 1..r do
      for k in 1..q do
        C[i,j] -= A[i, k] * B[k, j];
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
def panelSolve(Ab: [] ?t,
               panel: domain(2, indexType),
               piv: [] indexType) {
  const pnlRows = panel.dim(1),
        pnlCols = panel.dim(2);

  //
  // Ideally some type of assertion to ensure panel is embedded in Ab's
  // domain
  //
  assert(piv.domain.dim(1) == Ab.domain.dim(1));

  if (pnlCols.length == 0) then return;

  for k in pnlCols {             // iterate through the columns
    var col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.dim(1).length == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const ( , (pivotRow, )) = maxloc reduce(abs(Ab(col)), col),
          pivot = Ab[pivotRow, k];

    // Swap the current row with the pivot row
    piv[k] <=> piv[pivotRow];

    Ab[k, ..] <=> Ab[pivotRow, ..];

    if (pivot == 0) then
      halt("Matrix can not be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot
    if k+1 <= pnlRows.high then
      Ab(col)[k+1.., k..k] /= pivot;

    // update all other values below the pivot
    if k+1 <= pnlRows.high && k+1 <= pnlCols.high then
      forall (i,j) in panel[k+1.., k+1..] do
        Ab[i,j] -= Ab[i,k] * Ab[k,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
def updateBlockRow(Ab: [] ?t,
                   tl: domain(2, indexType),
                   tr: domain(2, indexType)) {
  const tlRows = tl.dim(1),
        tlCols = tl.dim(2),
        trRows = tr.dim(1),
        trCols = tr.dim(2);

  assert(tlCols == trRows);

  //
  // Ultimately, we will probably want to do some replication of the
  // tl block in order to make this operation completely localized as
  // in the dgemm.  We have not yet undertaken that optimization.
  //
  for i in trRows do
    forall j in trCols do
      for k in tlRows.low..i-1 do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
}

//
// compute the backwards substitution
//
def backwardSub(n: indexType,
                A: [1..n, 1..n] elemType,
                b: [1..n] elemType) {
  var x: [b.domain] elemType;

  for i in [b.domain by -1] {
    x[i] = b[i];

    for j in [i+1..b.domain.high] do
      x[i] -= A[i,j] * x[j];

    x[i] /= A[i,i];
  }

  return x;
}

//
// print out the problem size and block size if requested
//
def printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
def initAB(Ab: [] elemType) {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
def verifyResults(Ab, MatrixSpace, x) {
  var A => Ab[MatrixSpace],
      b => Ab[.., n+1];

  initAB(Ab);

  const axmbNorm = norm(gaxpyMinus(n, n, A, x, b), normType.normInf);

  const a1norm   = norm(A, normType.norm1),
        aInfNorm = norm(A, normType.normInf),
        x1Norm   = norm(x, normType.norm1),
        xInfNorm = norm(x, normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
def gaxpyMinus(n: indexType,
               m: indexType,
               A: [1..n, 1..m],
               x: [1..m],
               y: [1..n]) {
  var res: [1..n] elemType;

  for i in 1..n do
    for j in 1..m do
      res[i] += A[i,j]*x[j];

  for i in 1..n do
    res[i] -= y[i];

  return res;
}

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of matrices and the element type of those matrices
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 5;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  const MatVectSpace: domain(2, indexType) = [1..n, 1..n+1],
        MatrixSpace = MatVectSpace[.., ..n];

  var Ab : [MatVectSpace] elemType,  // the matrix A and vector b
      piv: [1..n] indexType,         // a vector of pivot values
      x  : [1..n] elemType;          // the solution vector, x

  var A => Ab[MatrixSpace],          // an alias for the Matrix part of Ab
      b => Ab[.., n+1];              // an alias for the last column of Ab

  initAB(Ab);

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, Ab, piv);                 // compute the LU factorization

  x = backwardSub(n, A, b);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(Ab, MatrixSpace, x);
  printResults(validAnswer, execTime);
}

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
def LUFactorize(n: indexType, Ab: [1..n, 1..n+1] elemType,
                piv: [1..n] indexType) {
  const AbD = Ab.domain;    // alias Ab.domain to save typing

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(Ab, l, piv);
    if (tr.numIndices > 0) then
      updateBlockRow(Ab, tl, tr);

    //
    // update trailing submatrix (if any)
    //
    if (br.numIndices > 0) then
      schurComplement(Ab, blk);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The vertex labeled [1] is location
//     |aaaaa|.....|.....|.....|  (ptOp, ptOp) in the code below.
//     +-----+-----+-----+-----+
//     |aaaaa|.....|.....|.....|  The vertex labeled [2] is location
//     |aaaaa|.....|.....|.....|  (ptSol, ptSol)
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
def schurComplement(Ab: [1..n, 1..n+1] elemType, ptOp: indexType) {
  const AbD = Ab.domain;

  //
  // Calculate location of ptSol (see diagram above)
  //
  const ptSol = ptOp+blkSize;

  //
  // Copy data into replicated array so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.  These
  // replicated distributions aren't implemented yet, but imagine that
  // they look something like the following:
  //
  //var replAbD: domain(2)
  //            dmapped new Dimensional(BlkCyc(blkSize), Replicated))
  //          = AbD[ptSol.., 1..#blkSize];
  //
  const replAD: domain(2) = AbD[ptSol.., ptOp..#blkSize],
        replBD: domain(2) = AbD[ptOp..#blkSize, ptSol..];

  const replA : [replAD] elemType = Ab[ptSol.., ptOp..#blkSize],
        replB : [replBD] elemType = Ab[ptOp..#blkSize, ptSol..];

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in AbD[ptSol.., ptSol..] by (blkSize, blkSize) {
    //
    // At this point, the dgemms should all be local, so assert that
    // fact
    //
    local {
      const aBlkD = replAD[row..#blkSize, ptOp..#blkSize],
            bBlkD = replBD[ptOp..#blkSize, col..#blkSize],
            cBlkD = AbD[row..#blkSize, col..#blkSize];

      dgemm(aBlkD.dim(1).length,
            aBlkD.dim(2).length,
            bBlkD.dim(2).length,
            replA(aBlkD),
            replB(bBlkD),
            Ab(cBlkD));
    }
  }
}

//
// calculate C = C - A * B.
//
def dgemm(p: indexType,       // number of rows in A
          q: indexType,       // number of cols in A, number of rows in B
          r: indexType,       // number of cols in B
          A: [1..p, 1..q] ?t,
          B: [1..q, 1..r] t,
          C: [1..p, 1..r] t) {
  // Calculate (i,j) using a dot product of a row of A and a column of B.
  for i in 1..p do
    for j in 1..r do
      for k in 1..q do
        C[i,j] -= A[i, k] * B[k, j];
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
def panelSolve(Ab: [] ?t,
               panel: domain(2, indexType),
               piv: [] indexType) {
  const pnlRows = panel.dim(1),
        pnlCols = panel.dim(2);

  //
  // Ideally some type of assertion to ensure panel is embedded in Ab's
  // domain
  //
  assert(piv.domain.dim(1) == Ab.domain.dim(1));

  if (pnlCols.length == 0) then return;

  for k in pnlCols {             // iterate through the columns
    var col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.dim(1).length == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const ( , (pivotRow, )) = maxloc reduce(abs(Ab(col)), col),
          pivot = Ab[pivotRow, k];

    // Swap the current row with the pivot row
    piv[k] <=> piv[pivotRow];

    Ab[k, ..] <=> Ab[pivotRow, ..];

    if (pivot == 0) then
      halt("Matrix can not be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot
    if k+1 <= pnlRows.high then
      Ab(col)[k+1.., k..k] /= pivot;

    // update all other values below the pivot
    if k+1 <= pnlRows.high && k+1 <= pnlCols.high then
      forall (i,j) in panel[k+1.., k+1..] do
        Ab[i,j] -= Ab[i,k] * Ab[k,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
def updateBlockRow(Ab: [] ?t, tl: domain(2), tr: domain(2)) {
  const tlRows = tl.dim(1),
        tlCols = tl.dim(2),
        trRows = tr.dim(1),
        trCols = tr.dim(2);

  assert(tlCols == trRows);

  //
  // Ultimately, we will probably want to do some replication of the
  // tl block in order to make this operation completely localized as
  // in the dgemm.  We have not yet undertaken that optimization.
  //
  for i in trRows do
    forall j in trCols do
      for k in tlRows.low..i-1 do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
}

//
// compute the backwards substitution
//
def backwardSub(n: int,
                A: [1..n, 1..n] elemType,
                b: [1..n] elemType) {
  var x: [b.domain] elemType;

  for i in [b.domain by -1] {
    x[i] = b[i];

    for j in [i+1..b.domain.high] do
      x[i] -= A[i,j] * x[j];

    x[i] /= A[i,i];
  }

  return x;
}

//
// print out the problem size and block size if requested
//
def printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
def initAB(Ab: [] elemType) {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
def verifyResults(Ab, MatrixSpace, x) {
  var A => Ab[MatrixSpace],
      b => Ab[.., n+1];

  initAB(Ab);

  const axmbNorm = norm(gaxpyMinus(n, n, A, x, b), normType.normInf);

  const a1norm   = norm(A, normType.norm1),
        aInfNorm = norm(A, normType.normInf),
        x1Norm   = norm(x, normType.norm1),
        xInfNorm = norm(x, normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

//
// print success/failure, the execution time and the Gflop/s value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
def gaxpyMinus(n: indexType,
               m: indexType,
               A: [1..n, 1..m],
               x: [1..m],
               y: [1..n]) {
  var res: [1..n] elemType;

  for i in 1..n do
    for j in 1..m do
      res[i] += A[i,j]*x[j];

  for i in 1..n do
    res[i] -= y[i];

  return res;
}


Update to jacobi.chpl to keep lines with comments to 80 characters.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@8837 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
 *  Jacobi Method Example
 *
 *  This program computes the solution of a Laplace equation using
 *  the Jacobi method.
 *
 */

// Configuration variables for program:
config var n = 5,                       // size of nxn grid
           epsilon = 0.00001,           // convergence tolerance
           verbose = false;             // printing control

def main() {
  const ProblemSpace = [1..n, 1..n],    // domain for interior grid points
        BigDomain = [0..n+1, 0..n+1];   // domain with boundary points

  var X, XNew: [BigDomain] real = 0.0;  // X stores approximate solution
                                        // XNew is work array
  X[n+1, 1..n] = 1.0;

  if (verbose) {
    writeln("Initial configuration:");
    writeln(X, "\n");
  }

  var iteration = 0,                    // iteration counter
      delta: real;                      // measure of convergence

  do {
    // compute next approximation using Jacobi method and store in XNew
    forall (i,j) in ProblemSpace do
      XNew(i,j) = (X(i-1,j) + X(i+1,j) + X(i,j-1) + X(i,j+1)) / 4.0;

    // compute difference between next and current approximations
    delta = max reduce abs(XNew[ProblemSpace] - X[ProblemSpace]);

    // update X with next approximation
    X[ProblemSpace] = XNew[ProblemSpace];

    // advance iteration counter
    iteration += 1;

    if (verbose) {
      writeln("iteration: ", iteration);
      writeln(X);
      writeln("delta: ", delta, "\n");
    }
  } while (delta > epsilon);

  writeln("Jacobi computation complete.");
  writeln("Delta is ", delta, " (< epsilon = ", epsilon, ")");
  writeln("# of iterations: ", iteration);
}

/*
 *  Jacobi Method Example
 *
 *  This program computes the solution of a Laplace equation using Jacobi method.
 *
 */

// Configuration variables for program:
config var n = 5,                       // size of nxn grid
           epsilon = 0.00001,           // convergence tolerance
           verbose = false;             // printing control

def main() {
  const ProblemSpace = [1..n, 1..n],    // domain for interior grid points
        BigDomain = [0..n+1, 0..n+1];   // larger domain including boundary points

  var X, XNew: [BigDomain] real = 0.0;  // X contains approximate solution
                                        // XNew is work array used in Jacobi iteration
  X[n+1, 1..n] = 1.0;

  if (verbose) {
    writeln("Initial configuration:");
    writeln(X, "\n");
  }

  var iteration = 0,                    // iteration counter
      delta: real;                      // measure of convergence at each iteration

  do {
    // compute next approximation using Jacobi method and store in XNew
    forall (i,j) in ProblemSpace do
      XNew(i,j) = (X(i-1,j) + X(i+1,j) + X(i,j-1) + X(i,j+1)) / 4.0;

    // compute difference between next and current approximations
    delta = max reduce abs(XNew[ProblemSpace] - X[ProblemSpace]);

    // update X with next approximation
    X[ProblemSpace] = XNew[ProblemSpace];

    // advance iteration counter
    iteration += 1;

    if (verbose) {
      writeln("iteration: ", iteration);
      writeln(X);
      writeln("delta: ", delta, "\n");
    }
  } while (delta > epsilon);

  writeln("Jacobi computation complete.");
  writeln("Delta is ", delta, " (< epsilon = ", epsilon, ")");
  writeln("# of iterations: ", iteration);
}


Remove "no doc" pragmas from private symbols in VisualDebug module

/*
 * Copyright 2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*

   Support for the visualization tool "chplvis"

   This module provides access to and enables hooks to dump out
   task and communication information for post-run visualization
   of the tasks and communication.

*/
module VisualDebug
{

  use String;

  private extern proc chpl_now_time():real;

  //
  // Data Generation for the Visual Debug tool  (offline)
  //

  private extern proc chpl_vdebug_start ( rootname: c_string, time:real);

  private extern proc chpl_vdebug_stop ();

  private extern proc chpl_vdebug_tag ( tagname: c_string);

  private extern proc chpl_vdebug_pause ();

  private extern proc chpl_vdebug_nolog ();

/* Tree "coforall procedure .... calls one of the above routines */

pragma "no doc"
  enum vis_op {v_start, v_stop, v_tag, v_pause};

  private proc VDebugTree (what: vis_op, name: string, time: real, id: int = 0) {
      var child = id * 2 + 1;
      chpl_vdebug_nolog();
      cobegin {
         /* left */
         if child < numLocales then
             on Locales[child] do VDebugTree (what, name, time, child);
         /* right */
         if child+1 < numLocales then
             on Locales[child+1] do VDebugTree (what, name, time, child+1);
     }
         /* Do the op at the root  */
         select what {
            when vis_op.v_start    do chpl_vdebug_start (name.c_str(), time);
            when vis_op.v_stop     do chpl_vdebug_stop ();
            when vis_op.v_tag      do chpl_vdebug_tag (name.c_str());
            when vis_op.v_pause    do chpl_vdebug_pause ();
         }
  }

/*
  Start logging events for VisualDebug.  Open a new set of data
  files, one for each locale, for chplvis.  This routine should be
  called only once for each program.  It creates a directory with the
  rootname and creates the files in that directory.  The files are
  named with the rootname and "-n" is added where n is the locale
  number.

  :arg rootname:  Directory name and rootname for files.
*/
  proc startVdebug ( rootname : string ) {
    var now = chpl_now_time();
    //coforall l in Locales do
    //  on l do chpl_vdebug_start (rootname.c_str(), now);
    VDebugTree (vis_op.v_start, rootname, now);
  }

/*
  Add a tag to the data for chplvis to allow "view points" in the data.

  :arg tagname: name of the tag
*/
  proc tagVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 0);
    //chpl_vdebug_tag (tagname.c_str(), 0);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

/*
  Stop collecting VisualDebug data and close the data files.
*/
  proc stopVdebug () {
    // chpl_vdebug_stop();
    // coforall l in Locales[1..] do
    //  on l do chpl_vdebug_stop();
    VDebugTree (vis_op.v_stop, "", 0);
  }

/*
  Suspend collection of VisualDebug data.
*/
  proc pauseVdebug () {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 1);
    //chpl_vdebug_tag(tagname.c_str(), 1);
    VDebugTree (vis_op.v_pause, "", 0);
  }

/*
  Resume collection of VisualDebug data for chplvis after a pauseVdebug().
  This also generates a tag record visible by chplvis.

  :arg tagname: name of the tag
*/
  proc resumeVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_resume ();
    //chpl_vdebug_resume(tagname);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

}

/*
 * Copyright 2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*

   Support for the visualization tool "chplvis"

   This module provides access to and enables hooks to dump out
   task and communication information for post-run visualization
   of the tasks and communication.

*/
module VisualDebug
{

  use String;

  pragma "no doc"
  private extern proc chpl_now_time():real;

  //
  // Data Generation for the Visual Debug tool  (offline)
  //

  pragma "no doc"
  private extern proc chpl_vdebug_start ( rootname: c_string, time:real);

  pragma "no doc"
  private extern proc chpl_vdebug_stop ();

  pragma "no doc"
  private extern proc chpl_vdebug_tag ( tagname: c_string);

  pragma "no doc"
  private extern proc chpl_vdebug_pause ();

  pragma "no doc"
  private extern proc chpl_vdebug_nolog ();

/* Tree "coforall procedure .... calls one of the above routines */

pragma "no doc"
  enum vis_op {v_start, v_stop, v_tag, v_pause};

pragma "no doc"
  private proc VDebugTree (what: vis_op, name: string, time: real, id: int = 0) {
      var child = id * 2 + 1;
      chpl_vdebug_nolog();
      cobegin {
         /* left */
         if child < numLocales then
             on Locales[child] do VDebugTree (what, name, time, child);
         /* right */
         if child+1 < numLocales then
             on Locales[child+1] do VDebugTree (what, name, time, child+1);
     }
         /* Do the op at the root  */
         select what {
            when vis_op.v_start    do chpl_vdebug_start (name.c_str(), time);
            when vis_op.v_stop     do chpl_vdebug_stop ();
            when vis_op.v_tag      do chpl_vdebug_tag (name.c_str());
            when vis_op.v_pause    do chpl_vdebug_pause ();
         }
  }

/*
  Start logging events for VisualDebug.  Open a new set of data
  files, one for each locale, for chplvis.  This routine should be
  called only once for each program.  It creates a directory with the
  rootname and creates the files in that directory.  The files are
  named with the rootname and "-n" is added where n is the locale
  number.

  :arg rootname:  Directory name and rootname for files.
*/
  proc startVdebug ( rootname : string ) {
    var now = chpl_now_time();
    //coforall l in Locales do
    //  on l do chpl_vdebug_start (rootname.c_str(), now);
    VDebugTree (vis_op.v_start, rootname, now);
  }

/*
  Add a tag to the data for chplvis to allow "view points" in the data.

  :arg tagname: name of the tag
*/
  proc tagVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 0);
    //chpl_vdebug_tag (tagname.c_str(), 0);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

/*
  Stop collecting VisualDebug data and close the data files.
*/
  proc stopVdebug () {
    // chpl_vdebug_stop();
    // coforall l in Locales[1..] do
    //  on l do chpl_vdebug_stop();
    VDebugTree (vis_op.v_stop, "", 0);
  }

/*
  Suspend collection of VisualDebug data.
*/
  proc pauseVdebug () {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 1);
    //chpl_vdebug_tag(tagname.c_str(), 1);
    VDebugTree (vis_op.v_pause, "", 0);
  }

/*
  Resume collection of VisualDebug data for chplvis after a pauseVdebug().
  This also generates a tag record visible by chplvis.

  :arg tagname: name of the tag
*/
  proc resumeVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_resume ();
    //chpl_vdebug_resume(tagname);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

}