Update one last test to the new task intent syntax

config const n = 11;
var s0 = "kiss kiss";

var l0: atomic int;
var l1: atomic int;

writeln(s0);
begin with (ref s0) {
  l0.waitFor(1);
  s0 = "bang bang";
  l1.write(1);
}

on Locales[numLocales-1] {
  l0.write(1);
  l1.waitFor(1);
  writeln(s0);
}

config const n = 11;
var s0 = "kiss kiss";

var l0: atomic int;
var l1: atomic int;

writeln(s0);
begin ref(s0) {
  l0.waitFor(1);
  s0 = "bang bang";
  l1.write(1);
}

on Locales[numLocales-1] {
  l0.write(1);
  l1.waitFor(1);
  writeln(s0);
}


Ooops. Forgot to remove a bit of unused commented out code.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@7535 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// _chpl_base module

// Primitive functions and operators on bool
pragma "inline" def =(a: bool, b: bool) return b;
pragma "inline" def ~(a: bool) return __primitive("u~", a);
pragma "inline" def &(a: bool, b: bool) return __primitive("&", a, b);
pragma "inline" def |(a: bool, b: bool) return __primitive("|", a, b);
pragma "inline" def ^(a: bool, b: bool) return __primitive("^", a, b);
pragma "inline" def ==(a: bool, b: bool) return __primitive("==", a, b);
pragma "inline" def !=(a: bool, b: bool) return __primitive("!=", a, b);
pragma "inline" def <=(a: bool, b: bool) return __primitive("<=", a, b);
pragma "inline" def >=(a: bool, b: bool) return __primitive(">=", a, b);
pragma "inline" def <(a: bool, b: bool) return __primitive("<", a, b);
pragma "inline" def >(a: bool, b: bool) return __primitive(">", a, b);
pragma "inline" def !(a: bool) return __primitive("!", a);
pragma "inline" def &&(a: bool, b: bool) return __primitive("&&", a, b);
pragma "inline" def ||(a: bool, b: bool) return __primitive("||", a, b);

// Primitive functions and operators on ints
pragma "inline" def =(a: int(?w), b: int(w)) return b;
pragma "inline" def +(a: int(32)) return __primitive("u+", a);
pragma "inline" def -(a: int(32)) return __primitive("u-", a);
pragma "inline" def ~(a: int(32)) return __primitive("u~", a);
pragma "inline" def !(a: int(32)) return __primitive("!", a);
pragma "inline" def +(a: int(32), b: int(32)) return __primitive("+", a, b);
pragma "inline" def -(a: int(32), b: int(32)) return __primitive("-", a, b);
pragma "inline" def *(a: int(32), b: int(32)) return __primitive("*", a, b);
pragma "inline" def /(a: int(32), b: int(32)) return __primitive("/", a, b);
pragma "inline" def **(a: int(32), b: int(32)) return __primitive("**", a, b);
pragma "inline" def %(a: int(32), b: int(32)) return __primitive("%", a, b);
pragma "inline" def &(a: int(32), b: int(32)) return __primitive("&", a, b);
pragma "inline" def |(a: int(32), b: int(32)) return __primitive("|", a, b);
pragma "inline" def ^(a: int(32), b: int(32)) return __primitive("^", a, b);
pragma "inline" def ==(a: int(32), b: int(32)) return __primitive("==", a, b);
pragma "inline" def !=(a: int(32), b: int(32)) return __primitive("!=", a, b);
pragma "inline" def <=(a: int(32), b: int(32)) return __primitive("<=", a, b);
pragma "inline" def >=(a: int(32), b: int(32)) return __primitive(">=", a, b);
pragma "inline" def <(a: int(32), b: int(32)) return __primitive("<", a, b);
pragma "inline" def >(a: int(32), b: int(32)) return __primitive(">", a, b);
pragma "inline" def &&(a: int(32), b: int(32)) return __primitive("&&", a, b);
pragma "inline" def ||(a: int(32), b: int(32)) return __primitive("||", a, b);

pragma "inline" def +(a: int(64)) return __primitive("u+", a);
pragma "inline" def -(a: int(64)) return __primitive("u-", a);
pragma "inline" def ~(a: int(64)) return __primitive("u~", a);
pragma "inline" def !(a: int(64)) return __primitive("!", a);
pragma "inline" def +(a: int(64), b: int(64)) return __primitive("+", a, b);
pragma "inline" def -(a: int(64), b: int(64)) return __primitive("-", a, b);
pragma "inline" def *(a: int(64), b: int(64)) return __primitive("*", a, b);
pragma "inline" def /(a: int(64), b: int(64)) return __primitive("/", a, b);
pragma "inline" def **(a: int(64), b: int(64)) return __primitive("**", a, b);
pragma "inline" def %(a: int(64), b: int(64)) return __primitive("%", a, b);
pragma "inline" def &(a: int(64), b: int(64)) return __primitive("&", a, b);
pragma "inline" def |(a: int(64), b: int(64)) return __primitive("|", a, b);
pragma "inline" def ^(a: int(64), b: int(64)) return __primitive("^", a, b);
pragma "inline" def ==(a: int(64), b: int(64)) return __primitive("==", a, b);
pragma "inline" def !=(a: int(64), b: int(64)) return __primitive("!=", a, b);
pragma "inline" def <=(a: int(64), b: int(64)) return __primitive("<=", a, b);
pragma "inline" def >=(a: int(64), b: int(64)) return __primitive(">=", a, b);
pragma "inline" def <(a: int(64), b: int(64)) return __primitive("<", a, b);
pragma "inline" def >(a: int(64), b: int(64)) return __primitive(">", a, b);
pragma "inline" def &&(a: int(64), b: int(64)) return __primitive("&&", a, b);
pragma "inline" def ||(a: int(64), b: int(64)) return __primitive("||", a, b);

pragma "inline" def bits(type t) where t == int(8)  return 8;
pragma "inline" def bits(type t) where t == int(16) return 16;
pragma "inline" def bits(type t) where t == int(32) return 32;
pragma "inline" def bits(type t) where t == int(64) return 64;

// Primitive functions and operators on uints
pragma "inline" def =(a: uint(?w), b: uint(w)) return b;
pragma "inline" def +(a: uint(32)) return __primitive("u+", a);
pragma "inline" def -(a: uint(32)) return __primitive("u-", a);
pragma "inline" def +(a: uint(32), b: uint(32)) return __primitive("+", a, b);
pragma "inline" def -(a: uint(32), b: uint(32)) return __primitive("-", a, b);
pragma "inline" def *(a: uint(32), b: uint(32)) return __primitive("*", a, b);
pragma "inline" def /(a: uint(32), b: uint(32)) return __primitive("/", a, b);
pragma "inline" def **(a: uint(32), b: uint(32)) return __primitive("**", a, b);
pragma "inline" def %(a: uint(32), b: uint(32)) return __primitive("%", a, b);
pragma "inline" def ~(a: uint(32)) return __primitive("u~", a);
pragma "inline" def &(a: uint(32), b: uint(32)) return __primitive("&", a, b);
pragma "inline" def |(a: uint(32), b: uint(32)) return __primitive("|", a, b);
pragma "inline" def ^(a: uint(32), b: uint(32)) return __primitive("^", a, b);
pragma "inline" def ==(a: uint(32), b: uint(32)) return __primitive("==", a, b);
pragma "inline" def !=(a: uint(32), b: uint(32)) return __primitive("!=", a, b);
pragma "inline" def <=(a: uint(32), b: uint(32)) return __primitive("<=", a, b);
pragma "inline" def >=(a: uint(32), b: uint(32)) return __primitive(">=", a, b);
pragma "inline" def <(a: uint(32), b: uint(32)) return __primitive("<", a, b);
pragma "inline" def >(a: uint(32), b: uint(32)) return __primitive(">", a, b);
pragma "inline" def !(a: uint(32)) return __primitive("!", a);
pragma "inline" def &&(a: uint(32), b: uint(32)) return __primitive("&&", a, b);
pragma "inline" def ||(a: uint(32), b: uint(32)) return __primitive("||", a, b);

pragma "inline" def +(a: uint(64)) return __primitive("u+", a);
pragma "inline" def -(a: uint(64)) return __primitive("u-", a);
pragma "inline" def +(a: uint(64), b: uint(64)) return __primitive("+", a, b);
pragma "inline" def -(a: uint(64), b: uint(64)) return __primitive("-", a, b);
pragma "inline" def *(a: uint(64), b: uint(64)) return __primitive("*", a, b);
pragma "inline" def /(a: uint(64), b: uint(64)) return __primitive("/", a, b);
pragma "inline" def **(a: uint(64), b: uint(64)) return __primitive("**", a, b);
pragma "inline" def %(a: uint(64), b: uint(64)) return __primitive("%", a, b);
pragma "inline" def ~(a: uint(64)) return __primitive("u~", a);
pragma "inline" def &(a: uint(64), b: uint(64)) return __primitive("&", a, b);
pragma "inline" def |(a: uint(64), b: uint(64)) return __primitive("|", a, b);
pragma "inline" def ^(a: uint(64), b: uint(64)) return __primitive("^", a, b);
pragma "inline" def ==(a: uint(64), b: uint(64)) return __primitive("==", a, b);
pragma "inline" def !=(a: uint(64), b: uint(64)) return __primitive("!=", a, b);
pragma "inline" def <=(a: uint(64), b: uint(64)) return __primitive("<=", a, b);
pragma "inline" def >=(a: uint(64), b: uint(64)) return __primitive(">=", a, b);
pragma "inline" def <(a: uint(64), b: uint(64)) return __primitive("<", a, b);
pragma "inline" def >(a: uint(64), b: uint(64)) return __primitive(">", a, b);
pragma "inline" def !(a: uint(64)) return __primitive("!", a);
pragma "inline" def &&(a: uint(64), b: uint(64)) return __primitive("&&", a, b);
pragma "inline" def ||(a: uint(64), b: uint(64)) return __primitive("||", a, b);

pragma "inline" def <<(a: int(32), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: int(32), b: _anyint) return __primitive(">>", a, b);
pragma "inline" def <<(a: int(64), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: int(64), b: _anyint) return __primitive(">>", a, b);
pragma "inline" def <<(a: uint(32), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: uint(32), b: _anyint) return __primitive(">>", a, b);
pragma "inline" def <<(a: uint(64), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: uint(64), b: _anyint) return __primitive(">>", a, b);

pragma "inline" def bits(type t) where t == uint(8)  return 8;
pragma "inline" def bits(type t) where t == uint(16) return 16;
pragma "inline" def bits(type t) where t == uint(32) return 32;
pragma "inline" def bits(type t) where t == uint(64) return 64;

// Primitive functions and operators on floats
pragma "inline" def =(a: float(?w), b: float(w)) return b;
pragma "inline" def +(a: float(?w)) return __primitive("u+", a);
pragma "inline" def -(a: float(?w)) return __primitive("u-", a);
pragma "inline" def +(a: float(?w), b: float(w)) return __primitive("+", a, b);
pragma "inline" def -(a: float(?w), b: float(w)) return __primitive("-", a, b);
pragma "inline" def *(a: float(?w), b: float(w)) return __primitive("*", a, b);
pragma "inline" def /(a: float(?w), b: float(w)) return __primitive("/", a, b);
pragma "inline" def **(a: float(?w), b: float(w)) return __primitive("**", a, b);
pragma "inline" def ==(a: float(?w), b: float(w)) return __primitive("==", a, b);
pragma "inline" def !=(a: float(?w), b: float(w)) return __primitive("!=", a, b);
pragma "inline" def <=(a: float(?w), b: float(w)) return __primitive("<=", a, b);
pragma "inline" def >=(a: float(?w), b: float(w)) return __primitive(">=", a, b);
pragma "inline" def <(a: float(?w), b: float(w)) return __primitive("<", a, b);
pragma "inline" def >(a: float(?w), b: float(w)) return __primitive(">", a, b);
pragma "inline" def !(a: float(?w)) return __primitive("!", a);
pragma "inline" def &&(a: float(?w), b: float(w)) return __primitive("&&", a, b);
pragma "inline" def ||(a: float(?w), b: float(w)) return __primitive("||", a, b);
pragma "inline" def bits(type t) where t == float(32)  return 32;
pragma "inline" def bits(type t) where t == float(64)  return 64;
pragma "inline" def bits(type t) where t == float(128) return 128;

// Primitive functions and operators on string
pragma "inline" def =(a: string, b: string) return __primitive("string_copy", b);
pragma "inline" def ==(a: string, b: string) return __primitive("string_equal", a, b);
pragma "inline" def !=(a: string, b: string) return !(a == b);
pragma "inline" def +(a: string, b: string) return __primitive("string_concat", a, b);
pragma "inline" def length(a: string) return __primitive("string_length", a);
pragma "inline" def string.substring(i: int) return __primitive("string_index", this, i);

pragma "inline" def _string_contains(a: string, b: string)
  return __primitive("string_contains", a, b);

pragma "inline" def _tostring(x : bool, format : string)
  return __primitive("to_string", format, x);

pragma "inline" def _tostring(x : int, format : string)
  return __primitive("to_string", format, x);

pragma "inline" def _tostring(x : float, format : string)
  return __primitive("to_string", format, x);

pragma "inline" def ascii(a: string) return __primitive("ascii", a);

//
// Primitive funs for objects
//
pragma "inline" def ==(a: object, b: object) return __primitive("ptr_eq", a, b);
pragma "inline" def !=(a: object, b: object) return __primitive("ptr_neq", a, b);
pragma "inline" def =(a, b) return b;
pragma "inline" def _copy(a) return a;

//
// _init
//
pragma "inline" def _init(x : bool) return false;
pragma "inline" def _init(x : int(?w)) return 0:int(w);
pragma "inline" def _init(x : uint(?w)) return 0:uint(w);
pragma "inline" def _init(x : float(?w)) return 0:float(w);
pragma "inline" def _init(x : string) return "";
pragma "inline" def _init(x) return nil:x;

// identity functions (for reduction operations)
pragma "inline" def _prod_id( type t) return __primitive( "_prod_id", t);
pragma "inline" def _land_id( type t) return __primitive( "_land_id", t);
pragma "inline" def _lor_id( type t) return __primitive( "_lor_id", t);
pragma "inline" def _lxor_id( type t) return __primitive( "_lxor_id", t);
pragma "inline" def _band_id( type t) return __primitive( "_band_id", t);
pragma "inline" def _bor_id( type t) return __primitive( "_bor_id", t);
pragma "inline" def _bxor_id( type t) return __primitive( "_bxor_id", t);

// predefined functions
pragma "inline" def min(type t) return __primitive( "_min", t);
pragma "inline" def max(type t) return __primitive( "_max", t);

pragma "inline" def min(x, y) return if x < y then x else y;
pragma "inline" def max(x, y) return if x > y then x else y;

pragma "inline" def min(x, y, z...?k) return min(min(x, y), (...z));
pragma "inline" def max(x, y, z...?k) return max(max(x, y), (...z));

//
// More primitive funs
//

pragma "inline" def sleep( t: int) {
  __primitive( "sleep", t);
}

pragma "no codegen" pragma "rename _chpl_exit" def exit(status : int) {
  __primitive("exit");
}

pragma "no codegen" pragma "rename fflush" def fflush(fp: _file) : int {
  return __primitive("fflush", fp);
}

def init_elts(x, s, type t) {
  for i in 1..s {
    // var y : e.type;
    var y : t;
    x(i-1) = y;
  }
}

// dynamic data block class
pragma "data class"
class _ddata {
  type elt_type;
  var size : int;
  pragma "inline" def init() {
    __primitive("array_init", this, elt_type, size);
    init_elts(this, size, elt_type);
  }
  pragma "inline" def this(i : int) {
    return __primitive("array_get", this, i);
  }
  pragma "inline" def =this(i : int, val : elt_type) {
    __primitive("array_set", this, i, val);
  }
}

// synchronization primitives support
pragma "inline" def _init( cv: _mutex_p) return __primitive( "mutex_new");
pragma "inline" def =( a: _mutex_p, b: _mutex_p) return b;
pragma "inline" def _init( cv: _condvar_p) return __primitive( "condvar_new");
pragma "inline" def =( a: _condvar_p, b: _condvar_p) return b;

pragma "synchronization primitive"
pragma "sync var"
pragma "no default functions"
class _syncvar {
  type base_type;
  var  value: base_type;     // actual data
  var  is_full: bool;
  var  lock: _mutex_p;       // need to acquire before accessing this record
  var  cv_empty: _condvar_p; // wait for empty, signal this when empty
  var  cv_full: _condvar_p;  // wait for full, signal this when full

  def initialize() {
    is_full = false;
    lock = __primitive( "mutex_new");
    cv_empty = __primitive( "condvar_new");
    cv_full = __primitive( "condvar_new");
  }
}

pragma "synchronization primitive"
def _copy( sv:_syncvar) {
  return readFE(sv);
}

pragma "synchronization primitive"
def _init( sv:_syncvar) {
  return _syncvar( sv.value.type);
}

// The operations are:
//  = (i.e., write_wait_empty_leave_full)
//  writeFE - wait full, leave empty
//  writeXF - no wait, leave full
//  writeXE - no wait, leave empty
//  readFE - wait full, leave empty
//  readFF - wait full, leave full
//  readXF - no wait, leave full
//  readXX - no wait, leave F/E unchanged

// wait for empty, leaves full. This is the default write on sync vars.
pragma "synchronization primitive"
def =( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  if (sv.is_full) {
    __primitive( "sync_wait_empty", sv);
  }
  sv.value = value;
  sv.is_full = true;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return sv;
}

// wait full, leave empty.
pragma "synchronization primitive"
def writeFE( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  sv.value = value;
  sv.is_full = false;
  __primitive( "sync_signal_empty", sv);
  __primitive( "sync_unlock", sv);
  return sv;
}

// no wait, leave full.
pragma "synchronization primitive"
def writeXF( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  sv.value = value;
  sv.is_full = true;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
}

// no wait, leave empty. One use of this is with using sync var as a lock.
pragma "synchronization primitive"
def writeXE( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  sv.value = value;
  sv.is_full = false;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
}

// wait for full, leave empty. This is the default read on sync vars.
pragma "synchronization primitive"
def readFE( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  ret = sv.value;
  sv.is_full = false;
  __primitive( "sync_signal_empty", sv);
  __primitive( "sync_unlock", sv);
  return ret;
}

// wait for full, leave full.
pragma "synchronization primitive"
def readFF( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  ret = sv.value;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return ret;
}

// no wait, leave full.
pragma "synchronization primitive"
def readXF( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  ret = sv.value;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return ret;
}

// read value.  No state change or signals.
pragma "synchronization primitive"
def readXX( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  ret = sv.value;
  __primitive( "sync_unlock", sv);
  return ret;
}

pragma "synchronization primitive"
def isFull( sv:_syncvar) {
  var isfull: bool;
  __primitive( "sync_lock", sv);
  isfull = sv.is_full;
  __primitive( "sync_unlock", sv);
  return isfull;
}

// single variable support
pragma "synchronization primitive"
pragma "single var"
pragma "no default functions"
class _singlevar {
  type base_type;
  var  value: base_type;     // actual data
  var  is_full: bool;
  var  lock: _mutex_p;       // need to acquire before accessing this record
  var  cv_full: _condvar_p;  // wait for full

  def initialize() {
    is_full = false;
    lock = __primitive( "mutex_new");
    cv_full = __primitive( "condvar_new");
  }
}

pragma "synchronization primitive"
def _copy( sv:_singlevar) {
  return readFF( sv);
}

pragma "synchronization primitive"
def _init( sv:_singlevar) {
  return _singlevar( sv.value.type);
}

// Can only write once.  Otherwise, it is an error.
pragma "synchronization primitive"
def =( sv:_singlevar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  if (sv.is_full) {
    halt( "***Error: single var already defined***");
  }
  sv.value = value;
  sv.is_full = true;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return sv;
}

// Wait for full
pragma "synchronization primitive"
def readFF( sv:_singlevar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  ret = sv.value;
  __primitive( "sync_signal_full", sv); // in case others are waiting
  __primitive( "sync_unlock", sv);
  return ret;
}

//
// MOVE THESE, maybe to a memory module?
//
pragma "no codegen" def _chpl_memtest_printMemTable(threshold: int = 1000);
pragma "no codegen" def _chpl_memtest_printMemStat();
pragma "no codegen" def _chpl_memtest_resetMemStat();
pragma "no codegen" def _chpl_memtest_allocAndFree();
pragma "no codegen" def _chpl_memtest_freedMalloc();
pragma "no codegen" def _chpl_memtest_freedWithoutMalloc();
pragma "no codegen" def _chpl_memtest_reallocWithoutMalloc();
pragma "no codegen" def _chpl_memtest_reallocZeroSize();
pragma "no codegen" def _chpl_memtest_mallocOutOfMemory();
pragma "no codegen" def _chpl_memtest_reallocOutOfMemory();

pragma "no codegen" def startTrackingMem();

// _chpl_base module

// Primitive functions and operators on bool
pragma "inline" def =(a: bool, b: bool) return b;
pragma "inline" def ~(a: bool) return __primitive("u~", a);
pragma "inline" def &(a: bool, b: bool) return __primitive("&", a, b);
pragma "inline" def |(a: bool, b: bool) return __primitive("|", a, b);
pragma "inline" def ^(a: bool, b: bool) return __primitive("^", a, b);
pragma "inline" def ==(a: bool, b: bool) return __primitive("==", a, b);
pragma "inline" def !=(a: bool, b: bool) return __primitive("!=", a, b);
pragma "inline" def <=(a: bool, b: bool) return __primitive("<=", a, b);
pragma "inline" def >=(a: bool, b: bool) return __primitive(">=", a, b);
pragma "inline" def <(a: bool, b: bool) return __primitive("<", a, b);
pragma "inline" def >(a: bool, b: bool) return __primitive(">", a, b);
pragma "inline" def !(a: bool) return __primitive("!", a);
pragma "inline" def &&(a: bool, b: bool) return __primitive("&&", a, b);
pragma "inline" def ||(a: bool, b: bool) return __primitive("||", a, b);

// Primitive functions and operators on ints
pragma "inline" def =(a: int(?w), b: int(w)) return b;
pragma "inline" def +(a: int(32)) return __primitive("u+", a);
pragma "inline" def -(a: int(32)) return __primitive("u-", a);
pragma "inline" def ~(a: int(32)) return __primitive("u~", a);
pragma "inline" def !(a: int(32)) return __primitive("!", a);
pragma "inline" def +(a: int(32), b: int(32)) return __primitive("+", a, b);
pragma "inline" def -(a: int(32), b: int(32)) return __primitive("-", a, b);
pragma "inline" def *(a: int(32), b: int(32)) return __primitive("*", a, b);
pragma "inline" def /(a: int(32), b: int(32)) return __primitive("/", a, b);
pragma "inline" def **(a: int(32), b: int(32)) return __primitive("**", a, b);
pragma "inline" def %(a: int(32), b: int(32)) return __primitive("%", a, b);
pragma "inline" def &(a: int(32), b: int(32)) return __primitive("&", a, b);
pragma "inline" def |(a: int(32), b: int(32)) return __primitive("|", a, b);
pragma "inline" def ^(a: int(32), b: int(32)) return __primitive("^", a, b);
pragma "inline" def ==(a: int(32), b: int(32)) return __primitive("==", a, b);
pragma "inline" def !=(a: int(32), b: int(32)) return __primitive("!=", a, b);
pragma "inline" def <=(a: int(32), b: int(32)) return __primitive("<=", a, b);
pragma "inline" def >=(a: int(32), b: int(32)) return __primitive(">=", a, b);
pragma "inline" def <(a: int(32), b: int(32)) return __primitive("<", a, b);
pragma "inline" def >(a: int(32), b: int(32)) return __primitive(">", a, b);
pragma "inline" def &&(a: int(32), b: int(32)) return __primitive("&&", a, b);
pragma "inline" def ||(a: int(32), b: int(32)) return __primitive("||", a, b);

pragma "inline" def +(a: int(64)) return __primitive("u+", a);
pragma "inline" def -(a: int(64)) return __primitive("u-", a);
pragma "inline" def ~(a: int(64)) return __primitive("u~", a);
pragma "inline" def !(a: int(64)) return __primitive("!", a);
pragma "inline" def +(a: int(64), b: int(64)) return __primitive("+", a, b);
pragma "inline" def -(a: int(64), b: int(64)) return __primitive("-", a, b);
pragma "inline" def *(a: int(64), b: int(64)) return __primitive("*", a, b);
pragma "inline" def /(a: int(64), b: int(64)) return __primitive("/", a, b);
pragma "inline" def **(a: int(64), b: int(64)) return __primitive("**", a, b);
pragma "inline" def %(a: int(64), b: int(64)) return __primitive("%", a, b);
pragma "inline" def &(a: int(64), b: int(64)) return __primitive("&", a, b);
pragma "inline" def |(a: int(64), b: int(64)) return __primitive("|", a, b);
pragma "inline" def ^(a: int(64), b: int(64)) return __primitive("^", a, b);
pragma "inline" def ==(a: int(64), b: int(64)) return __primitive("==", a, b);
pragma "inline" def !=(a: int(64), b: int(64)) return __primitive("!=", a, b);
pragma "inline" def <=(a: int(64), b: int(64)) return __primitive("<=", a, b);
pragma "inline" def >=(a: int(64), b: int(64)) return __primitive(">=", a, b);
pragma "inline" def <(a: int(64), b: int(64)) return __primitive("<", a, b);
pragma "inline" def >(a: int(64), b: int(64)) return __primitive(">", a, b);
pragma "inline" def &&(a: int(64), b: int(64)) return __primitive("&&", a, b);
pragma "inline" def ||(a: int(64), b: int(64)) return __primitive("||", a, b);

pragma "inline" def bits(type t) where t == int(8)  return 8;
pragma "inline" def bits(type t) where t == int(16) return 16;
pragma "inline" def bits(type t) where t == int(32) return 32;
pragma "inline" def bits(type t) where t == int(64) return 64;

// Primitive functions and operators on uints
pragma "inline" def =(a: uint(?w), b: uint(w)) return b;
pragma "inline" def +(a: uint(32)) return __primitive("u+", a);
pragma "inline" def -(a: uint(32)) return __primitive("u-", a);
pragma "inline" def +(a: uint(32), b: uint(32)) return __primitive("+", a, b);
pragma "inline" def -(a: uint(32), b: uint(32)) return __primitive("-", a, b);
pragma "inline" def *(a: uint(32), b: uint(32)) return __primitive("*", a, b);
pragma "inline" def /(a: uint(32), b: uint(32)) return __primitive("/", a, b);
pragma "inline" def **(a: uint(32), b: uint(32)) return __primitive("**", a, b);
pragma "inline" def %(a: uint(32), b: uint(32)) return __primitive("%", a, b);
pragma "inline" def ~(a: uint(32)) return __primitive("u~", a);
pragma "inline" def &(a: uint(32), b: uint(32)) return __primitive("&", a, b);
pragma "inline" def |(a: uint(32), b: uint(32)) return __primitive("|", a, b);
pragma "inline" def ^(a: uint(32), b: uint(32)) return __primitive("^", a, b);
pragma "inline" def ==(a: uint(32), b: uint(32)) return __primitive("==", a, b);
pragma "inline" def !=(a: uint(32), b: uint(32)) return __primitive("!=", a, b);
pragma "inline" def <=(a: uint(32), b: uint(32)) return __primitive("<=", a, b);
pragma "inline" def >=(a: uint(32), b: uint(32)) return __primitive(">=", a, b);
pragma "inline" def <(a: uint(32), b: uint(32)) return __primitive("<", a, b);
pragma "inline" def >(a: uint(32), b: uint(32)) return __primitive(">", a, b);
pragma "inline" def !(a: uint(32)) return __primitive("!", a);
pragma "inline" def &&(a: uint(32), b: uint(32)) return __primitive("&&", a, b);
pragma "inline" def ||(a: uint(32), b: uint(32)) return __primitive("||", a, b);

pragma "inline" def +(a: uint(64)) return __primitive("u+", a);
pragma "inline" def -(a: uint(64)) return __primitive("u-", a);
pragma "inline" def +(a: uint(64), b: uint(64)) return __primitive("+", a, b);
pragma "inline" def -(a: uint(64), b: uint(64)) return __primitive("-", a, b);
pragma "inline" def *(a: uint(64), b: uint(64)) return __primitive("*", a, b);
pragma "inline" def /(a: uint(64), b: uint(64)) return __primitive("/", a, b);
pragma "inline" def **(a: uint(64), b: uint(64)) return __primitive("**", a, b);
pragma "inline" def %(a: uint(64), b: uint(64)) return __primitive("%", a, b);
pragma "inline" def ~(a: uint(64)) return __primitive("u~", a);
pragma "inline" def &(a: uint(64), b: uint(64)) return __primitive("&", a, b);
pragma "inline" def |(a: uint(64), b: uint(64)) return __primitive("|", a, b);
pragma "inline" def ^(a: uint(64), b: uint(64)) return __primitive("^", a, b);
pragma "inline" def ==(a: uint(64), b: uint(64)) return __primitive("==", a, b);
pragma "inline" def !=(a: uint(64), b: uint(64)) return __primitive("!=", a, b);
pragma "inline" def <=(a: uint(64), b: uint(64)) return __primitive("<=", a, b);
pragma "inline" def >=(a: uint(64), b: uint(64)) return __primitive(">=", a, b);
pragma "inline" def <(a: uint(64), b: uint(64)) return __primitive("<", a, b);
pragma "inline" def >(a: uint(64), b: uint(64)) return __primitive(">", a, b);
pragma "inline" def !(a: uint(64)) return __primitive("!", a);
pragma "inline" def &&(a: uint(64), b: uint(64)) return __primitive("&&", a, b);
pragma "inline" def ||(a: uint(64), b: uint(64)) return __primitive("||", a, b);

pragma "inline" def <<(a: int(32), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: int(32), b: _anyint) return __primitive(">>", a, b);
pragma "inline" def <<(a: int(64), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: int(64), b: _anyint) return __primitive(">>", a, b);
pragma "inline" def <<(a: uint(32), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: uint(32), b: _anyint) return __primitive(">>", a, b);
pragma "inline" def <<(a: uint(64), b: _anyint) return __primitive("<<", a, b);
pragma "inline" def >>(a: uint(64), b: _anyint) return __primitive(">>", a, b);

pragma "inline" def bits(type t) where t == uint(8)  return 8;
pragma "inline" def bits(type t) where t == uint(16) return 16;
pragma "inline" def bits(type t) where t == uint(32) return 32;
pragma "inline" def bits(type t) where t == uint(64) return 64;

// Primitive functions and operators on floats
pragma "inline" def =(a: float(?w), b: float(w)) return b;
pragma "inline" def +(a: float(?w)) return __primitive("u+", a);
pragma "inline" def -(a: float(?w)) return __primitive("u-", a);
pragma "inline" def +(a: float(?w), b: float(w)) return __primitive("+", a, b);
pragma "inline" def -(a: float(?w), b: float(w)) return __primitive("-", a, b);
pragma "inline" def *(a: float(?w), b: float(w)) return __primitive("*", a, b);
pragma "inline" def /(a: float(?w), b: float(w)) return __primitive("/", a, b);
pragma "inline" def **(a: float(?w), b: float(w)) return __primitive("**", a, b);
pragma "inline" def ==(a: float(?w), b: float(w)) return __primitive("==", a, b);
pragma "inline" def !=(a: float(?w), b: float(w)) return __primitive("!=", a, b);
pragma "inline" def <=(a: float(?w), b: float(w)) return __primitive("<=", a, b);
pragma "inline" def >=(a: float(?w), b: float(w)) return __primitive(">=", a, b);
pragma "inline" def <(a: float(?w), b: float(w)) return __primitive("<", a, b);
pragma "inline" def >(a: float(?w), b: float(w)) return __primitive(">", a, b);
pragma "inline" def !(a: float(?w)) return __primitive("!", a);
pragma "inline" def &&(a: float(?w), b: float(w)) return __primitive("&&", a, b);
pragma "inline" def ||(a: float(?w), b: float(w)) return __primitive("||", a, b);
pragma "inline" def bits(type t) where t == float(32)  return 32;
pragma "inline" def bits(type t) where t == float(64)  return 64;
pragma "inline" def bits(type t) where t == float(128) return 128;

// Primitive functions and operators on string
pragma "inline" def =(a: string, b: string) return __primitive("string_copy", b);
pragma "inline" def ==(a: string, b: string) return __primitive("string_equal", a, b);
pragma "inline" def !=(a: string, b: string) return !(a == b);
pragma "inline" def +(a: string, b: string) return __primitive("string_concat", a, b);
pragma "inline" def length(a: string) return __primitive("string_length", a);
pragma "inline" def string.substring(i: int) return __primitive("string_index", this, i);

pragma "inline" def _string_contains(a: string, b: string)
  return __primitive("string_contains", a, b);

pragma "inline" def _tostring(x : bool, format : string)
  return __primitive("to_string", format, x);

pragma "inline" def _tostring(x : int, format : string)
  return __primitive("to_string", format, x);

pragma "inline" def _tostring(x : float, format : string)
  return __primitive("to_string", format, x);

pragma "inline" def ascii(a: string) return __primitive("ascii", a);

//
// Primitive funs for objects
//
pragma "inline" def ==(a: object, b: object) return __primitive("ptr_eq", a, b);
pragma "inline" def !=(a: object, b: object) return __primitive("ptr_neq", a, b);
pragma "inline" def =(a, b) return b;
pragma "inline" def _copy(a) return a;

//
// _init
//
pragma "inline" def _init(x : bool) return false;
pragma "inline" def _init(x : int(?w)) return 0:int(w);
pragma "inline" def _init(x : uint(?w)) return 0:uint(w);
pragma "inline" def _init(x : float(?w)) return 0:float(w);
pragma "inline" def _init(x : string) return "";
pragma "inline" def _init(x) return nil:x;

// identity functions (for reduction operations)
pragma "inline" def _prod_id( type t) return __primitive( "_prod_id", t);
pragma "inline" def _land_id( type t) return __primitive( "_land_id", t);
pragma "inline" def _lor_id( type t) return __primitive( "_lor_id", t);
pragma "inline" def _lxor_id( type t) return __primitive( "_lxor_id", t);
pragma "inline" def _band_id( type t) return __primitive( "_band_id", t);
pragma "inline" def _bor_id( type t) return __primitive( "_bor_id", t);
pragma "inline" def _bxor_id( type t) return __primitive( "_bxor_id", t);

// predefined functions
pragma "inline" def min(type t) return __primitive( "_min", t);
pragma "inline" def max(type t) return __primitive( "_max", t);

pragma "inline" def min(x, y) return if x < y then x else y;
pragma "inline" def max(x, y) return if x > y then x else y;

pragma "inline" def min(x, y, z...?k) return min(min(x, y), (...z));
pragma "inline" def max(x, y, z...?k) return max(max(x, y), (...z));

//
// More primitive funs
//

pragma "inline" def sleep( t: int) {
  __primitive( "sleep", t);
}

pragma "no codegen" pragma "rename _chpl_exit" def exit(status : int) {
  __primitive("exit");
}

pragma "no codegen" pragma "rename fflush" def fflush(fp: _file) : int {
  return __primitive("fflush", fp);
}

def init_elts(x, s, type t) {
  for i in 1..s {
    // var y : e.type;
    var y : t;
    x(i-1) = y;
  }
}

// dynamic data block class
pragma "data class"
class _ddata {
  type elt_type;
  var size : int;
  // var _element : elt_type;
  pragma "inline" def init() {
    __primitive("array_init", this, elt_type, size);
    init_elts(this, size, elt_type);
  }
  pragma "inline" def this(i : int) {
    return __primitive("array_get", this, i);
  }
  pragma "inline" def =this(i : int, val : elt_type) {
    __primitive("array_set", this, i, val);
  }
}

// synchronization primitives support
pragma "inline" def _init( cv: _mutex_p) return __primitive( "mutex_new");
pragma "inline" def =( a: _mutex_p, b: _mutex_p) return b;
pragma "inline" def _init( cv: _condvar_p) return __primitive( "condvar_new");
pragma "inline" def =( a: _condvar_p, b: _condvar_p) return b;

pragma "synchronization primitive"
pragma "sync var"
pragma "no default functions"
class _syncvar {
  type base_type;
  var  value: base_type;     // actual data
  var  is_full: bool;
  var  lock: _mutex_p;       // need to acquire before accessing this record
  var  cv_empty: _condvar_p; // wait for empty, signal this when empty
  var  cv_full: _condvar_p;  // wait for full, signal this when full

  def initialize() {
    is_full = false;
    lock = __primitive( "mutex_new");
    cv_empty = __primitive( "condvar_new");
    cv_full = __primitive( "condvar_new");
  }
}

pragma "synchronization primitive"
def _copy( sv:_syncvar) {
  return readFE(sv);
}

pragma "synchronization primitive"
def _init( sv:_syncvar) {
  return _syncvar( sv.value.type);
}

// The operations are:
//  = (i.e., write_wait_empty_leave_full)
//  writeFE - wait full, leave empty
//  writeXF - no wait, leave full
//  writeXE - no wait, leave empty
//  readFE - wait full, leave empty
//  readFF - wait full, leave full
//  readXF - no wait, leave full
//  readXX - no wait, leave F/E unchanged

// wait for empty, leaves full. This is the default write on sync vars.
pragma "synchronization primitive"
def =( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  if (sv.is_full) {
    __primitive( "sync_wait_empty", sv);
  }
  sv.value = value;
  sv.is_full = true;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return sv;
}

// wait full, leave empty.
pragma "synchronization primitive"
def writeFE( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  sv.value = value;
  sv.is_full = false;
  __primitive( "sync_signal_empty", sv);
  __primitive( "sync_unlock", sv);
  return sv;
}

// no wait, leave full.
pragma "synchronization primitive"
def writeXF( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  sv.value = value;
  sv.is_full = true;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
}

// no wait, leave empty. One use of this is with using sync var as a lock.
pragma "synchronization primitive"
def writeXE( sv:_syncvar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  sv.value = value;
  sv.is_full = false;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
}

// wait for full, leave empty. This is the default read on sync vars.
pragma "synchronization primitive"
def readFE( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  ret = sv.value;
  sv.is_full = false;
  __primitive( "sync_signal_empty", sv);
  __primitive( "sync_unlock", sv);
  return ret;
}

// wait for full, leave full.
pragma "synchronization primitive"
def readFF( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  ret = sv.value;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return ret;
}

// no wait, leave full.
pragma "synchronization primitive"
def readXF( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  ret = sv.value;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return ret;
}

// read value.  No state change or signals.
pragma "synchronization primitive"
def readXX( sv:_syncvar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  ret = sv.value;
  __primitive( "sync_unlock", sv);
  return ret;
}

pragma "synchronization primitive"
def isFull( sv:_syncvar) {
  var isfull: bool;
  __primitive( "sync_lock", sv);
  isfull = sv.is_full;
  __primitive( "sync_unlock", sv);
  return isfull;
}

// single variable support
pragma "synchronization primitive"
pragma "single var"
pragma "no default functions"
class _singlevar {
  type base_type;
  var  value: base_type;     // actual data
  var  is_full: bool;
  var  lock: _mutex_p;       // need to acquire before accessing this record
  var  cv_full: _condvar_p;  // wait for full

  def initialize() {
    is_full = false;
    lock = __primitive( "mutex_new");
    cv_full = __primitive( "condvar_new");
  }
}

pragma "synchronization primitive"
def _copy( sv:_singlevar) {
  return readFF( sv);
}

pragma "synchronization primitive"
def _init( sv:_singlevar) {
  return _singlevar( sv.value.type);
}

// Can only write once.  Otherwise, it is an error.
pragma "synchronization primitive"
def =( sv:_singlevar, value:sv.base_type) {
  __primitive( "sync_lock", sv);
  if (sv.is_full) {
    halt( "***Error: single var already defined***");
  }
  sv.value = value;
  sv.is_full = true;
  __primitive( "sync_signal_full", sv);
  __primitive( "sync_unlock", sv);
  return sv;
}

// Wait for full
pragma "synchronization primitive"
def readFF( sv:_singlevar) {
  var ret: sv.base_type;
  __primitive( "sync_lock", sv);
  if (!sv.is_full) {
    __primitive( "sync_wait_full", sv);
  }
  ret = sv.value;
  __primitive( "sync_signal_full", sv); // in case others are waiting
  __primitive( "sync_unlock", sv);
  return ret;
}

//
// MOVE THESE, maybe to a memory module?
//
pragma "no codegen" def _chpl_memtest_printMemTable(threshold: int = 1000);
pragma "no codegen" def _chpl_memtest_printMemStat();
pragma "no codegen" def _chpl_memtest_resetMemStat();
pragma "no codegen" def _chpl_memtest_allocAndFree();
pragma "no codegen" def _chpl_memtest_freedMalloc();
pragma "no codegen" def _chpl_memtest_freedWithoutMalloc();
pragma "no codegen" def _chpl_memtest_reallocWithoutMalloc();
pragma "no codegen" def _chpl_memtest_reallocZeroSize();
pragma "no codegen" def _chpl_memtest_mallocOutOfMemory();
pragma "no codegen" def _chpl_memtest_reallocOutOfMemory();

pragma "no codegen" def startTrackingMem();


Fixed a glitch I introduced in my test in r20699.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20712 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use DimensionalDist2D;
use ReplicatedDim;
use BlockCycDim;
use Memory, Time, Random, UtilMath;

/////////// configuration ///////////

// debugging/reporting flags
config const reproducible = boundsChecking, // auto off upon --fast
             verbose = false, // currently unused
             show_dgemms = false;
config param dgemm_asserts = false,
             do_dgemms = true,
             dummycomps_insteadof_dgemms = false;

type indexType = int,
     elemType = real(64);

// grid dimensions
config const tl1:indexType = tlFromNLocs(1),
             tl2:indexType = tlFromNLocs(2);
var tla: [0..#tl1, 0..#tl2] locale;
var tld: bool;  // whether our targetLocales are all distinct

setupTargetLocales();

// automatically compute an acceptable (reqN) or desirable (factorN) size
config const reqN = false,
             factorN = 20;

// problem and block sizes
config const blkSize = if reqN || factorN > 0 then 64 else 8,
             blk = 1, // can add any multiple of blkSize
             n = if reqN then computeRequiredN()
                 else if factorN then computeFactorN()
                 else 63;

// report it
writeln("n ", n, "  blkSize ", blkSize, "  blk ", blk,
        "  locales ", tl1, "*", tl2,
        if dgemm_asserts then "  dgemm_asserts are ON" else "");

/////////// domains and arrays ///////////

// non-distributed index space for Ab
const MatVectSpace = {1..n, 1..n+1};

// block-cyclic starting indices
const st1=1, st2=1;

// 1-d descriptors for Dimensional
const
  bdim1 = new BlockCyclicDim(lowIdx=st1, blockSize=blkSize, numLocales=tl1, name="D1"),
  rdim1 = new ReplicatedDim(tl1),
  bdim2 = new BlockCyclicDim(lowIdx=st2, blockSize=blkSize, numLocales=tl2, name="D2"),
  rdim2 = new ReplicatedDim(tl2);

const dimdist = new DimensionalDist2D(tla, bdim1, bdim2, "dim");

// the distributed domain for Ab
const AbD: domain(2, indexType)
// dmapped BlockCyclic(startIdx=(st1,st2), blocksize=(blkSize,blkSize), targetLocales=tla)
// dmapped DimensionalDist2D(tla, bdim1, bdim2, "dim")
   dmapped new dmap(dimdist)
  = MatVectSpace;

// temporaries
var Rest: domain(2, indexType) dmapped new dmap(dimdist);
var RestByBlkSize: domain(2, indexType, true) dmapped new dmap(dimdist);

// Ab: the matrix A and vector b
var Ab: [if do_dgemms then AbD else 1..1] elemType; // small if !do_dgemms

// the domains for replication
const
  replAD = {1..n, 1..blkSize} dmapped
    DimensionalDist2D(tla, bdim1, rdim2, "distBR"),
  replBD = {1..blkSize, 1..n+1} dmapped
    DimensionalDist2D(tla, rdim1, bdim2, "distRB");

// the arrays for replication
var replA: [replAD] elemType,
    replB: [replBD] elemType;

/////////// for the reference implementation ///////////

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;
config const verify = false;
// can be too much memory
//var Abref: [MatVectSpace] elemType;
var refsuccess = true;

/////////// run it ///////////

const startTime = getCurrentTime();

schurComplement(blk);

const execTime = getCurrentTime() - startTime;  // store the elapsed time
write("DONE");
if reproducible then writeln(); else writeln("  time = ", execTime);

/////////////////////////////////////////////////////////////////////////////

proc schurComplement(blk) {

  //showCurrTime("schurComplement  blk ", blk);

//  const AD   = AbD[blk+blkSize.., blk..#blkSize];
//  const BD   = AbD[blk..#blkSize, blk+blkSize..];

  const AD_dim2 = MatVectSpace.dim(2)(blk..#blkSize);
  const BD_dim1 = MatVectSpace.dim(1)(blk..#blkSize);

  const AbSlice1 = Ab[1..n, AD_dim2],
        AbSlice2 = Ab[BD_dim1, 1..n+1];
  //showCurrTime("AbSlices");

  forall (ab, ra) in zip(AbSlice1, replA) do
    local
      ra = ab;

  replicateA(blk);

  forall (ab, rb) in zip(AbSlice2, replB) do
    local
      rb = ab;

  replicateB(blk);

  const RestLocal = MatVectSpace[blk+blkSize.., blk+blkSize..];
  Rest = RestLocal;
  RestByBlkSize = RestLocal by (blkSize, blkSize);
  //showCurrTime("replication; Rest");

  if Rest.numIndices == 0 {
    writeln("schurComplement: nothing to do   Rest = ", Rest);
    return;
  }

// Referring to 'Rest' in dgdriver() seems faster than passing these constants.
//  const RestD1cp = Rest.dim(1),
//        RestD2cp = Rest.dim(2),
//        Rest1end = RestD1cp.last,
//        Rest2end = RestD2cp.last,
//        lastFullRow = Rest1end - blkSize + 1,
//        lastFullCol = Rest2end - blkSize + 1;

  reportBlocks((Rest.dim(1) by blkSize).length, (Rest.dim(2) by blkSize).length);

  forall (row,col) in RestByBlkSize {
    // we have some additional stuff for debugging/tuning

    if show_dgemms then
      writeln((row, col), " on ", here.id);

    if dummycomps_insteadof_dgemms then local {
      const r1 = Rest.dim(1)(row..#blkSize), r2 = Rest.dim(2)(col..#blkSize);
    }

    if do_dgemms then
      dgdriver(row, col);

  } // forall row,col

  //showCurrTime("dgemms");
}

proc dgdriver(row, col) {
  local {

    const
          rng1 = Rest.dim(1)(row..#blkSize),
          rng2 = Rest.dim(2)(col..#blkSize);
//          rng1 = if row <= lastFullRow then row..#blkSize else row..Rest1end,
//          rng2 = if col <= lastFullCol then col..#blkSize else col..Rest2end;

    dgemm(replA.localSlice(rng1, ..),
          replB.localSlice(.., rng2),
          Ab.localSlice(rng1, rng2));
  }

// The original loop:
//      for a in rng1 do
//        for w in 1..blkSize do
//          for b in rng2 do
//            Ab[a,b] -= replA[a,w] * replB[w,b];

}

proc dgemm(LreplA, LreplB, LAb) {
  if dgemm_asserts {
    // This is because these dimensions have the same user indices
    // and are handled by the same distribution 1-d descriptors.
    // So they generate the same (pair-wise) storage indices.
    assert(LAb.domain.dim(1) == LreplA.domain.dim(1));
    assert(LAb.domain.dim(2) == LreplB.domain.dim(2));
    // These dimensions are replicated and their user indices are 1..blkSize.
    assert(LreplA.domain.dim(2) == (1..blkSize));
    assert(LreplB.domain.dim(1) == (1..blkSize));
  }
//  write("replA ", LreplA.domain, "  ");
//  write("replB ", LreplB.domain, "  ");
//  write("Ab    ", LAb.domain, "  ");
//  writeln();

  const r1 = LAb.domain.dim(1),
        r2 = LAb.domain.dim(2),
        rB = 1..blkSize;

  // interchange the j and k loops
  // plus hoist LAb[i,k]
  for i in r1 do
    for k in r2 do {
      var acc: elemType = 0;
      for j in rB do
        acc -= LreplA[i,j] * LreplB[j,k];
      LAb[i,k] += acc;
    }
}

/////////////////////////////////////////////////////////////////////////////

proc replicateA(abIx) {
    const fromLocId2 = targetLocalesIndexForAbIndex(2, abIx);
    coforall lid1 in 0..#tl1 do
      on tla(lid1, fromLocId2) do
        coforall lid2 in 0..#tl2 do
          if lid2 != fromLocId2 then
            replA._value.localAdescs[lid1,lid2].myStorageArr =
              replA._value.localAdescs[lid1,fromLocId2].myStorageArr;
}

proc replicateB(abIx) {
    const fromLocId1 = targetLocalesIndexForAbIndex(1, abIx);
    coforall lid2 in 0..#tl2 do
      on tla(fromLocId1, lid2) do
        coforall lid1 in 0..#tl1 do
          if lid1 != fromLocId1 then
            replB._value.localAdescs[lid1,lid2].myStorageArr =
              replB._value.localAdescs[fromLocId1,lid2].myStorageArr;
}

proc targetLocalesIndexForAbIndex(param dim, abIx)
  return (divceilpos(abIx, blkSize) - 1) % (if dim == 1 then tl1 else tl2);

/////////////////////////////////////////////////////////////////////////////

proc setupTargetLocales() {
//  writeln("setting up for ", tl1, "*", tl2, " locales");
  tld = numLocales >= tla.numElements;
  if tld {
    if numLocales > tla.numElements then
      writeln("UNUSED LOCALES ", numLocales - tla.numElements);
    for (l,i) in zip(tla,0..) do l = Locales[i];
  } else {
writeln("insufficient locales");
halt();
    writeln("oversubscribing Locales(0)");
    tla = Locales(0);
  }
}

// interpreting numLocales
proc tlFromNLocs(dim) {
  var tl1, tl2: indexType;
  select numLocales {
    when  1 do (tl1,tl2) = (1,1);
    when  2 do (tl1,tl2) = (1,2);
    when  3 do (tl1,tl2) = (1,3);
    when  4 do (tl1,tl2) = (2,2);
    when  8 do (tl1,tl2) = (2,4);
    when  9 do (tl1,tl2) = (3,3);
    when 16 do (tl1,tl2) = (4,4);
    when 18 do (tl1,tl2) = (3,6);
    when 25 do (tl1,tl2) = (5,5);
    when 32 do (tl1,tl2) = (4,8);
    when 50 do (tl1,tl2) = (5,10);
    when 64 do (tl1,tl2) = (8,8);
    when 72 do (tl1,tl2) = (6,12);
//    when  do (tl1,tl2) = ();
    otherwise { tl1 = sqrt(numLocales):indexType; tl2 = tl1; }
  }
  select dim {
    when 1 do return tl1;
    when 2 do return tl2;
    otherwise halt("invalid dim ", dim);
  }
}

proc computeRequiredN() {
  // HPL requirement: 8*n^2 > (total memory on the system) / 2.
  // We compute total memory only over tla's locales, and seek
  //    n > sqrt(totalMem / 16)
  // We also want 'Rest' to fit perfectly in a multiple of tl1*tl2 blocks.
  // So we make nRest a multiple of (tl1*blkSize) and (tl2*blkSize),
  // where Ab's longer dimension (n+1) == (nRest + blkSize).

  const totalMem = + reduce tla.physicalMemory(unit = MemUnits.Bytes);
  const nUnsquaredReqReal = sqrt(totalMem / 16);
  const nUnsquaredReq = floor(nUnsquaredReqReal) : int + 1; // n must be at least this
  const nRestRaw = floor(nUnsquaredReq):int + 1 - blkSize;
  const (nResult, multiple) = computeNfromNRestRaw(nRestRaw, true);

//writeln("n ", nResult, "  nPrev ", nResult - multiple,
//        "  nUnsquaredReqReal ", nUnsquaredReqReal, "  blkSize ", blkSize,
//        "  nlocs ", tla.numElements);

  // verify we computed it well
  assert(nResult > nUnsquaredReq);
  assert(nResult - multiple <= nUnsquaredReq);

  return nResult;
}

proc computeNfromNRest(nRest) {
  return nRest - 1 + blkSize;
}
proc computeNfromNRestRaw(nRestRaw, param needMultiple = false) {
  const multiple = blkSize * (if tl2 == tl1 || tl2 == 2*tl1 then tl2 else tl1*tl2);
  const nRest = divceil(nRestRaw, multiple) * multiple;
  const nResult = computeNfromNRest(nRest);

//writeln("nRestRaw ", nRestRaw, "  multiple ", multiple, "  nRest ", nRest);
  return if needMultiple then (nResult, multiple) else nResult;
}

proc computeFactorN() {
  // nRest = factor * (tlEach * blkSize),
  // if tlEach==tl1==tl2. More generally,
  // nRest ~= factor * sqrt(tl1*tl2) * blkSize,
  // making nRest a multiple of tl1*tl2*blkSize.
  // nRest = (n+1) - blkSize.

  const nRestRaw = ceil(factorN * sqrt(tl1*tl2) * blkSize) : int;
  return computeNfromNRestRaw(nRestRaw);
}

proc reportBlocks(len1, len2) {
  const pfx  = "",
        rem1 = len1 % tl1,
        rem2 = len2 % tl2,
        remnote = if rem1 == 0 && rem2 == 0 then "" else
          "  rem " + rem1:string + "*" + rem2:string;

  writeln(pfx, len1, "*", len2, " blocks",
          "  factor ", (len1 / tl1) * (len2 / tl2),
          " = ", len1 / tl1, "*", len2 / tl2, remnote);
}

use DimensionalDist2D;
use ReplicatedDim;
use BlockCycDim;
use Memory, Time, Random;

/////////// configuration ///////////

// debugging/reporting flags
config const reproducible = boundsChecking, // auto off upon --fast
             verbose = false, // currently unused
             show_dgemms = false;
config param dgemm_asserts = false,
             do_dgemms = true,
             dummycomps_insteadof_dgemms = false;

type indexType = int,
     elemType = real(64);

// grid dimensions
config const tl1:indexType = tlFromNLocs(1),
             tl2:indexType = tlFromNLocs(2);
var tla: [0..#tl1, 0..#tl2] locale;
var tld: bool;  // whether our targetLocales are all distinct

setupTargetLocales();

// automatically compute an acceptable (reqN) or desirable (factorN) size
config const reqN = false,
             factorN = 20;

// problem and block sizes
config const blkSize = if reqN || factorN > 0 then 64 else 8,
             blk = 1, // can add any multiple of blkSize
             n = if reqN then computeRequiredN()
                 else if factorN then computeFactorN()
                 else 63;

// report it
writeln("n ", n, "  blkSize ", blkSize, "  blk ", blk,
        "  locales ", tl1, "*", tl2,
        if dgemm_asserts then "  dgemm_asserts are ON" else "");

/////////// domains and arrays ///////////

// non-distributed index space for Ab
const MatVectSpace = {1..n, 1..n+1};

// block-cyclic starting indices
const st1=1, st2=1;

// 1-d descriptors for Dimensional
const
  bdim1 = new BlockCyclicDim(lowIdx=st1, blockSize=blkSize, numLocales=tl1, name="D1"),
  rdim1 = new ReplicatedDim(tl1),
  bdim2 = new BlockCyclicDim(lowIdx=st2, blockSize=blkSize, numLocales=tl2, name="D2"),
  rdim2 = new ReplicatedDim(tl2);

const dimdist = new DimensionalDist2D(tla, bdim1, bdim2, "dim");

// the distributed domain for Ab
const AbD: domain(2, indexType)
// dmapped BlockCyclic(startIdx=(st1,st2), blocksize=(blkSize,blkSize), targetLocales=tla)
// dmapped DimensionalDist2D(tla, bdim1, bdim2, "dim")
   dmapped new dmap(dimdist)
  = MatVectSpace;

// temporaries
var Rest: domain(2, indexType) dmapped new dmap(dimdist);
var RestByBlkSize: domain(2, indexType, true) dmapped new dmap(dimdist);

// Ab: the matrix A and vector b
var Ab: [if do_dgemms then AbD else 1..1] elemType; // small if !do_dgemms

// the domains for replication
const
  replAD = {1..n, 1..blkSize} dmapped
    DimensionalDist2D(tla, bdim1, rdim2, "distBR"),
  replBD = {1..blkSize, 1..n+1} dmapped
    DimensionalDist2D(tla, rdim1, bdim2, "distRB");

// the arrays for replication
var replA: [replAD] elemType,
    replB: [replBD] elemType;

/////////// for the reference implementation ///////////

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;
config const verify = false;
// can be too much memory
//var Abref: [MatVectSpace] elemType;
var refsuccess = true;

/////////// run it ///////////

const startTime = getCurrentTime();

schurComplement(blk);

const execTime = getCurrentTime() - startTime;  // store the elapsed time
write("DONE");
if reproducible then writeln(); else writeln("  time = ", execTime);

/////////////////////////////////////////////////////////////////////////////

proc schurComplement(blk) {

  //showCurrTime("schurComplement  blk ", blk);

//  const AD   = AbD[blk+blkSize.., blk..#blkSize];
//  const BD   = AbD[blk..#blkSize, blk+blkSize..];

  const AD_dim2 = MatVectSpace.dim(2)(blk..#blkSize);
  const BD_dim1 = MatVectSpace.dim(1)(blk..#blkSize);

  const AbSlice1 = Ab[1..n, AD_dim2],
        AbSlice2 = Ab[BD_dim1, 1..n+1];
  //showCurrTime("AbSlices");

  forall (ab, ra) in zip(AbSlice1, replA) do
    local
      ra = ab;

  replicateA(blk);

  forall (ab, rb) in zip(AbSlice2, replB) do
    local
      rb = ab;

  replicateB(blk);

  const RestLocal = MatVectSpace[blk+blkSize.., blk+blkSize..];
  Rest = RestLocal;
  RestByBlkSize = RestLocal by (blkSize, blkSize);
  //showCurrTime("replication; Rest");

  if Rest.numIndices == 0 {
    writeln("schurComplement: nothing to do   Rest = ", Rest);
    return;
  }

// Referring to 'Rest' in dgdriver() seems faster than passing these constants.
//  const RestD1cp = Rest.dim(1),
//        RestD2cp = Rest.dim(2),
//        Rest1end = RestD1cp.last,
//        Rest2end = RestD2cp.last,
//        lastFullRow = Rest1end - blkSize + 1,
//        lastFullCol = Rest2end - blkSize + 1;

  reportBlocks((Rest.dim(1) by blkSize).length, (Rest.dim(2) by blkSize).length);

  forall (row,col) in RestByBlkSize {
    // we have some additional stuff for debugging/tuning

    if show_dgemms then
      writeln((row, col), " on ", here.id);

    if dummycomps_insteadof_dgemms then local {
      const r1 = Rest.dim(1)(row..#blkSize), r2 = Rest.dim(2)(col..#blkSize);
    }

    if do_dgemms then
      dgdriver(row, col);

  } // forall row,col

  //showCurrTime("dgemms");
}

proc dgdriver(row, col) {
  local {

    const
          rng1 = Rest.dim(1)(row..#blkSize),
          rng2 = Rest.dim(2)(col..#blkSize);
//          rng1 = if row <= lastFullRow then row..#blkSize else row..Rest1end,
//          rng2 = if col <= lastFullCol then col..#blkSize else col..Rest2end;

    dgemm(replA.localSlice(rng1, ..),
          replB.localSlice(.., rng2),
          Ab.localSlice(rng1, rng2));
  }

// The original loop:
//      for a in rng1 do
//        for w in 1..blkSize do
//          for b in rng2 do
//            Ab[a,b] -= replA[a,w] * replB[w,b];

}

proc dgemm(LreplA, LreplB, LAb) {
  if dgemm_asserts {
    // This is because these dimensions have the same user indices
    // and are handled by the same distribution 1-d descriptors.
    // So they generate the same (pair-wise) storage indices.
    assert(LAb.domain.dim(1) == LreplA.domain.dim(1));
    assert(LAb.domain.dim(2) == LreplB.domain.dim(2));
    // These dimensions are replicated and their user indices are 1..blkSize.
    assert(LreplA.domain.dim(2) == (1..blkSize));
    assert(LreplB.domain.dim(1) == (1..blkSize));
  }
//  write("replA ", LreplA.domain, "  ");
//  write("replB ", LreplB.domain, "  ");
//  write("Ab    ", LAb.domain, "  ");
//  writeln();

  const r1 = LAb.domain.dim(1),
        r2 = LAb.domain.dim(2),
        rB = 1..blkSize;

  // interchange the j and k loops
  // plus hoist LAb[i,k]
  for i in r1 do
    for k in r2 do {
      var acc: elemType = 0;
      for j in rB do
        acc -= LreplA[i,j] * LreplB[j,k];
      LAb[i,k] += acc;
    }
}

/////////////////////////////////////////////////////////////////////////////

proc replicateA(abIx) {
    const fromLocId2 = targetLocalesIndexForAbIndex(2, abIx);
    coforall lid1 in 0..#tl1 do
      on tla(lid1, fromLocId2) do
        coforall lid2 in 0..#tl2 do
          if lid2 != fromLocId2 then
            replA._value.localAdescs[lid1,lid2].myStorageArr =
              replA._value.localAdescs[lid1,fromLocId2].myStorageArr;
}

proc replicateB(abIx) {
    const fromLocId1 = targetLocalesIndexForAbIndex(1, abIx);
    coforall lid2 in 0..#tl2 do
      on tla(fromLocId1, lid2) do
        coforall lid1 in 0..#tl1 do
          if lid1 != fromLocId1 then
            replB._value.localAdescs[lid1,lid2].myStorageArr =
              replB._value.localAdescs[fromLocId1,lid2].myStorageArr;
}

proc targetLocalesIndexForAbIndex(param dim, abIx)
  return (divceilpos(abIx, blkSize) - 1) % (if dim == 1 then tl1 else tl2);

/////////////////////////////////////////////////////////////////////////////

proc setupTargetLocales() {
//  writeln("setting up for ", tl1, "*", tl2, " locales");
  tld = numLocales >= tla.numElements;
  if tld {
    if numLocales > tla.numElements then
      writeln("UNUSED LOCALES ", numLocales - tla.numElements);
    for (l,i) in zip(tla,0..) do l = Locales[i];
  } else {
writeln("insufficient locales");
halt();
    writeln("oversubscribing Locales(0)");
    tla = Locales(0);
  }
}

// interpreting numLocales
proc tlFromNLocs(dim) {
  var tl1, tl2: indexType;
  select numLocales {
    when  1 do (tl1,tl2) = (1,1);
    when  2 do (tl1,tl2) = (1,2);
    when  3 do (tl1,tl2) = (1,3);
    when  4 do (tl1,tl2) = (2,2);
    when  8 do (tl1,tl2) = (2,4);
    when  9 do (tl1,tl2) = (3,3);
    when 16 do (tl1,tl2) = (4,4);
    when 18 do (tl1,tl2) = (3,6);
    when 25 do (tl1,tl2) = (5,5);
    when 32 do (tl1,tl2) = (4,8);
    when 50 do (tl1,tl2) = (5,10);
    when 64 do (tl1,tl2) = (8,8);
    when 72 do (tl1,tl2) = (6,12);
//    when  do (tl1,tl2) = ();
    otherwise { tl1 = sqrt(numLocales):indexType; tl2 = tl1; }
  }
  select dim {
    when 1 do return tl1;
    when 2 do return tl2;
    otherwise halt("invalid dim ", dim);
  }
}

proc computeRequiredN() {
  // HPL requirement: 8*n^2 > (total memory on the system) / 2.
  // We compute total memory only over tla's locales, and seek
  //    n > sqrt(totalMem / 16)
  // We also want 'Rest' to fit perfectly in a multiple of tl1*tl2 blocks.
  // So we make nRest a multiple of (tl1*blkSize) and (tl2*blkSize),
  // where Ab's longer dimension (n+1) == (nRest + blkSize).

  const totalMem = + reduce tla.physicalMemory(unit = MemUnits.Bytes);
  const nUnsquaredReqReal = sqrt(totalMem / 16);
  const nUnsquaredReq = floor(nUnsquaredReqReal) : int + 1; // n must be at least this
  const nRestRaw = floor(nUnsquaredReq):int + 1 - blkSize;
  const (nResult, multiple) = computeNfromNRestRaw(nRestRaw, true);

//writeln("n ", nResult, "  nPrev ", nResult - multiple,
//        "  nUnsquaredReqReal ", nUnsquaredReqReal, "  blkSize ", blkSize,
//        "  nlocs ", tla.numElements);

  // verify we computed it well
  assert(nResult > nUnsquaredReq);
  assert(nResult - multiple <= nUnsquaredReq);

  return nResult;
}

proc computeNfromNRest(nRest) {
  return nRest - 1 + blkSize;
}
proc computeNfromNRestRaw(nRestRaw, param needMultiple = false) {
  const multiple = blkSize * (if tl2 == tl1 || tl2 == 2*tl1 then tl2 else tl1*tl2);
  const nRest = divceil(nRestRaw, multiple) * multiple;
  const nResult = computeNfromNRest(nRest);

//writeln("nRestRaw ", nRestRaw, "  multiple ", multiple, "  nRest ", nRest);
  return if needMultiple then (nResult, multiple) else nResult;
}

proc computeFactorN() {
  // nRest = factor * (tlEach * blkSize),
  // if tlEach==tl1==tl2. More generally,
  // nRest ~= factor * sqrt(tl1*tl2) * blkSize,
  // making nRest a multiple of tl1*tl2*blkSize.
  // nRest = (n+1) - blkSize.

  const nRestRaw = ceil(factorN * sqrt(tl1*tl2) * blkSize) : int;
  return computeNfromNRestRaw(nRestRaw);
}

proc reportBlocks(len1, len2) {
  const pfx  = "",
        rem1 = len1 % tl1,
        rem2 = len2 % tl2,
        remnote = if rem1 == 0 && rem2 == 0 then "" else
          "  rem " + rem1:string + "*" + rem2:string;

  writeln(pfx, len1, "*", len2, " blocks",
          "  factor ", (len1 / tl1) * (len2 / tl2),
          " = ", len1 / tl1, "*", len2 / tl2, remnote);
}