Replaced format() with writef()

Replaced format() with writef()

/*
   NAS Parallel Benchmarks
   "Embarrassingly Parallel"

   This is a Chapel implementation of the embarrassingly parallel benchmark.
   It uses the Chapel pseudo-random number generator, which is modeled after
   the NAS linear congruential generator. Due to the different structure of
   the Chapel language, this implementation has a noticeably different structure
   than the Fortan reference version. However, attempts were made to keep it
   aligned with the reference implementation with respect to the work that is
   done and the results that are reported. For example, while this version uses
   Chapel's "reduce" statement, the iterator that is reduced on still generates
   a "batch" of 2^16 random numbers before quantifying the gaussian pairs in
   the set.

   This 'elegant' version attempts to express the EP benchmark as succintly
   and readably as possible. This means that it does not perform quite as
   well as the base Chapel implementation (about 2x slower), but because this
   implementation relies more on Chapel constructs such as "reduce", future
   implementations of the Chapel compiler and runtime could possibly make
   greater improvements in this version's performance.

   Written by Brandon Holt and Margaret Cahir, October 2011.
*/

use Random, Time, BlockDist, ReplicatedDist;

enum classVals {S, W, A, B, C, D};

const Class: domain(classVals);

const probSizes: [Class] int  = ( 24, 25, 28, 30, 32, 36 );
const refSx:     [Class] real =				(-3.247834652034740e+03,
							 -2.863319731645753e+03,
							 -4.295875165629892e+03,
							  4.033815542441498e+04,
							  4.764367927995374e+04,
							  1.982481200946593e+05),

      refSy:     [Class] real =				(-6.958407078382297e+03,
							 -6.320053679109499e+03,
							 -1.580732573678431e+04,
							 -2.660669192809235e+04,
							 -8.084072988043731e+04,
							 -1.020596636361769e+05);
config const probClass    = classVals.S;
config const printTimings = false;

const m = probSizes[probClass];

param mk = 16,      //batch size
      nq = 10,      //number of concentric
      nk = 2 ** mk; //number per batch

type RingsTuple = nq*real;

const mm = m - mk,  //log2(numBatches)
      nn = 2:int(64) ** mm; //number of batches

const epsilon = 1.0e-8,
      seed = 271828183:int(64);

const WorkSpace = {0:int(64)..#nn},
      WorkDist  = WorkSpace dmapped Block(WorkSpace);

var totalTime: Timer;

writef(" NAS Parallel Benchmarks 2.4 -- EP Benchmark\n");
writef(" Number of random numbers generated:  %15i\n", 2**(m+1));
writef(" Number of locales:                   %15i\n", numLocales);
writef(" Number of tasks per locale:          %15i\n", dataParTasksPerLocale);

/* Call the random number generator functions and initialize
   the x-array to reduce the effects of paging on the timers.
   Also, call all mathematical functions that are used. Make
   sure these initializations cannot be eliminated as dead code. */

totalTime.start();

proc gaussPairsBatch(k: int(64), numPairs: int) {
	var x: [1..numPairs*2] real;
	var q: RingsTuple;
	var t1, t2, tx, ty, x1, x2: real;

	var startTime, randTime, gaussTime: real;

	var rs = new RandomStream(seed=seed, parSafe=false);
	//Find starting seed for this batch
	rs.skipToNth(k * 2*nk + 1);

	// Compute uniform pseudorandom numbers.
	for e in x do e = rs.getNext();

	/* Compute Gaussian deviates by acceptance-rejection method and
	   tally counts in concentric square annuli.  This loop is not
	   vectorizable. */
	var sx, sy = 0.0;
	for i in 1..numPairs {
		x1 = x[i*2-1]*2.0-1.0;
		x2 = x[i*2  ]*2.0-1.0;
		t1 = (x1)**2 + (x2)**2;
		if t1 < 1.0 {
			t2 = sqrt(-2.0 * log(t1) / t1);
			tx = x1 * t2;
			ty = x2 * t2;
			var l = max(abs(tx), abs(ty)):int+1;
			q[l] += 1;
			sx += tx;
			sy += ty;
		}
	}

	return (q, sx, sy);
}

//do each batch of work (in parallel) and accumulate results
var (total_q, total_sx, total_sy) =
		+ reduce [k in WorkDist] gaussPairsBatch(k, nk);

//calculate total gaussian count (should equal prob size)
var gc = + reduce [i in 1..nq] total_q[i];

totalTime.stop();
var tm = totalTime.elapsed();

//verify
var verified = abs((total_sx - refSx[probClass])/total_sx) <= epsilon &&
	       abs((total_sy - refSy[probClass])/total_sy) <= epsilon;

var Mops = 2:int(64)**(m+1)/tm/1e6;

// Print Results
writeln();
writeln("EP Benchmark Results:");
if printTimings then writeln("CPU Time = ", tm);
writeln("N = 2^", m);
writeln("No. Gaussian Pairs = ", gc);
writef("Sums = %25.12er, %25.12er\n", total_sx, total_sy);
writeln("Counts:");
for i in 1..nq do
  writef("%3i %13i\n", i, total_q[i]);
writeln();

//print_results()
writeln(" EP Benchmark Completed.");
writeln(" Class           =                        ", probClass);
writef(" Size            = %24i\n", 2:int(64)**(m+1));
writef(" Iterations      = %24i\n", 0);
if printTimings then
  writef(" Time in seconds = %24r\n", tm);
writef(" Total tasks     = %24i\n", numLocales*dataParTasksPerLocale);
writef("    %# Locales    = %24i\n", numLocales);
writef("    Tasks/Locale = %24i\n", dataParTasksPerLocale);
if printTimings {
  writef(" Mop/s total     = %24r\n", Mops);
  writef(" Mop/s/process   = %24r\n", Mops/numLocales);
}
writeln(" Operation type  = Random numbers generated");
writeln(" Verification    = ", if verified then
						   "              SUCCESSFUL" else
						   "                  FAILED");

/*
   NAS Parallel Benchmarks
   "Embarrassingly Parallel"

   This is a Chapel implementation of the embarrassingly parallel benchmark.
   It uses the Chapel pseudo-random number generator, which is modeled after
   the NAS linear congruential generator. Due to the different structure of
   the Chapel language, this implementation has a noticeably different structure
   than the Fortan reference version. However, attempts were made to keep it
   aligned with the reference implementation with respect to the work that is
   done and the results that are reported. For example, while this version uses
   Chapel's "reduce" statement, the iterator that is reduced on still generates
   a "batch" of 2^16 random numbers before quantifying the gaussian pairs in
   the set.

   This 'elegant' version attempts to express the EP benchmark as succintly
   and readably as possible. This means that it does not perform quite as
   well as the base Chapel implementation (about 2x slower), but because this
   implementation relies more on Chapel constructs such as "reduce", future
   implementations of the Chapel compiler and runtime could possibly make
   greater improvements in this version's performance.

   Written by Brandon Holt and Margaret Cahir, October 2011.
*/

use Random, Time, BlockDist, ReplicatedDist;

enum classVals {S, W, A, B, C, D};

const Class: domain(classVals);

const probSizes: [Class] int  = ( 24, 25, 28, 30, 32, 36 );
const refSx:     [Class] real =				(-3.247834652034740e+03,
							 -2.863319731645753e+03,
							 -4.295875165629892e+03,
							  4.033815542441498e+04,
							  4.764367927995374e+04,
							  1.982481200946593e+05),

      refSy:     [Class] real =				(-6.958407078382297e+03,
							 -6.320053679109499e+03,
							 -1.580732573678431e+04,
							 -2.660669192809235e+04,
							 -8.084072988043731e+04,
							 -1.020596636361769e+05);
config const probClass    = classVals.S;
config const printTimings = false;

const m = probSizes[probClass];

param mk = 16,      //batch size
      nq = 10,      //number of concentric
      nk = 2 ** mk; //number per batch

type RingsTuple = nq*real;

const mm = m - mk,  //log2(numBatches)
      nn = 2:int(64) ** mm; //number of batches

const epsilon = 1.0e-8,
      seed = 271828183:int(64);

const WorkSpace = {0:int(64)..#nn},
      WorkDist  = WorkSpace dmapped Block(WorkSpace);

var totalTime: Timer;

writeln(" NAS Parallel Benchmarks 2.4 -- EP Benchmark");
writeln(" Number of random numbers generated:  ", format("%15ld",2:int(64)**(m+1)));
writeln(" Number of locales:                   ", format("%15d",numLocales));
writeln(" Number of tasks per locale:          ", format("%15d",dataParTasksPerLocale));

/* Call the random number generator functions and initialize
   the x-array to reduce the effects of paging on the timers.
   Also, call all mathematical functions that are used. Make
   sure these initializations cannot be eliminated as dead code. */

totalTime.start();

proc gaussPairsBatch(k: int(64), numPairs: int) {
	var x: [1..numPairs*2] real;
	var q: RingsTuple;
	var t1, t2, tx, ty, x1, x2: real;

	var startTime, randTime, gaussTime: real;

	var rs = new RandomStream(seed=seed, parSafe=false);
	//Find starting seed for this batch
	rs.skipToNth(k * 2*nk + 1);

	// Compute uniform pseudorandom numbers.
	for e in x do e = rs.getNext();

	/* Compute Gaussian deviates by acceptance-rejection method and
	   tally counts in concentric square annuli.  This loop is not
	   vectorizable. */
	var sx, sy = 0.0;
	for i in 1..numPairs {
		x1 = x[i*2-1]*2.0-1.0;
		x2 = x[i*2  ]*2.0-1.0;
		t1 = (x1)**2 + (x2)**2;
		if t1 < 1.0 {
			t2 = sqrt(-2.0 * log(t1) / t1);
			tx = x1 * t2;
			ty = x2 * t2;
			var l = max(abs(tx), abs(ty)):int+1;
			q[l] += 1;
			sx += tx;
			sy += ty;
		}
	}

	return (q, sx, sy);
}

//do each batch of work (in parallel) and accumulate results
var (total_q, total_sx, total_sy) =
		+ reduce [k in WorkDist] gaussPairsBatch(k, nk);

//calculate total gaussian count (should equal prob size)
var gc = + reduce [i in 1..nq] total_q[i];

totalTime.stop();
var tm = totalTime.elapsed();

//verify
var verified = abs((total_sx - refSx[probClass])/total_sx) <= epsilon &&
	       abs((total_sy - refSy[probClass])/total_sy) <= epsilon;

var Mops = 2:int(64)**(m+1)/tm/1e6;

// Print Results
writeln();
writeln("EP Benchmark Results:");
if printTimings then writeln("CPU Time = ", tm);
writeln("N = 2^", m);
writeln("No. Gaussian Pairs = ", gc);
//writeln("Sums = ", format("%25.15e",total_sx), ", ", format("%25.15e",total_sy));
writeln("Sums = ", format("%25.12e",total_sx), ", ", format("%25.12e",total_sy));
writeln("Counts:");
for i in 1..nq do {
	writeln (format("%3d",i)," ",format("%13ld",total_q[i]:int(64)));
}
writeln();

//print_results()
writeln(" EP Benchmark Completed.");
writeln(" Class           =                        ", probClass);
writeln(" Size            = ", format("%24ld", 2:int(64)**(m+1) ));
writeln(" Iterations      = ", format("%24d", 0));
if printTimings then
writeln(" Time in seconds = ", format("%24g", tm));
writeln(" Total tasks     = ", format("%24d", numLocales*dataParTasksPerLocale));
writeln("    # Locales    = ", format("%24d", numLocales));
writeln("    Tasks/Locale = ", format("%24d", dataParTasksPerLocale));
if printTimings then  {
writeln(" Mop/s total     = ", format("%24g", Mops));
writeln(" Mop/s/process   = ", format("%24g", Mops/numLocales));
}
writeln(" Operation type  = Random numbers generated");
writeln(" Verification    = ", if verified then
						   "              SUCCESSFUL" else
						   "                  FAILED");


Superficial change to test.

Superficial change to test.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@6776 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

fun sum(is: int...?k) {
  var result: int;
  for param j in 1..k do
    result += is(j);
  return result;
}

config const n = 5;

var A : [1..n, 1..n] float = sum[n*(0..n-1), 0..n-1];

writeln(A);

fun crossSum(is: int...?k) {
  var sum: int;
  for param j in 1..k do
    sum += is(j);
  return sum;
}

config const n = 5;

var A : [1..n, 1..n] float = crossSum[n*(0..n-1), 0..n-1];

writeln(A);


Overloaded fillRandom for 1D arrays of complex numbers.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@7768 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use Time;

class RandomStream {

  const seed = 2.0*(getCurrentTime(unit=microseconds))+1.0;
  const arand = 2.0*(getCurrentTime() + getCurrentTime(unit=milliseconds)) + 1.0;

  var numGenerated = 0;

  def nextrandlc(x : float, a: float = arand) {
    const
      r23   = 0.5**23,
      t23   = 2.0**23,
      r46   = 0.5**46,
      t46   = 2.0**46;
    var t1 = r23 * a;
    var a1 = floor(t1);
    var a2 = a - t23 * a1;
    t1 = r23 * x;
    var x1 = floor(t1);
    var x2 = x - t23 * x1;
    t1 = a1 * x2 + a2 * x1;
    var t2 = floor(r23 * t1);
    var z  = t1 - t23 * t2;
    var t3 = t23 * z + a2 * x2;
    var t4 = floor(r46 * t3);
    var x3 = t3 - t46 * t4;
    return (x3, r46 * x3);
  }

  def initrandlc(in n : int) : float {
    var i : int, t : float, g : float;
    var x : float = seed;
    t = arand;
    while n != 0 do {
      i = n / 2;
      if 2 * i != n then
        (x, g) = nextrandlc(x, t);
      (t, g) = nextrandlc(t, t);
      n = i;
    }
    return x;
  }

  def fillRandom(x:[?D] float) {

    if (D.rank > 1) {
      writeln ("Not yet implemented for 2D or higher arrays.");
      [i in D] x(i) = -1.0;
    }
    else {
      var randlc_last_x: float = initrandlc(numGenerated+1);
      for i in D {
        (randlc_last_x,x(i)) = nextrandlc(randlc_last_x);
        numGenerated += 1;
      }
    }
  }

  def fillRandom(x:[?D] complex) {

    if (D.rank > 1) {
      writeln ("Not yet implemented for 2D or higher arrays.");
      [i in D] x(i) = -1.0;
    }
    else {
      var randlc_last_x: float = initrandlc(numGenerated+1);
      for i in D {
        (randlc_last_x,x(i).real) = nextrandlc(randlc_last_x);
        (randlc_last_x,x(i).imag) = nextrandlc(randlc_last_x);
        numGenerated += 2;
      }
    }
  }
}

def fillRandom(x:[], seed: float = (2.0*getCurrentTime(unit=microseconds)+1.0),
               arand: float = (2.0*(getCurrentTime()
                                   + getCurrentTime(unit=milliseconds))+1.0)) {

  var randNums = RandomStream(seed,arand);

  randNums.fillRandom(x);
}

use Time;

class RandomStream {

  const seed = 2.0*(getCurrentTime(unit=microseconds))+1.0;
  const arand = 2.0*(getCurrentTime() + getCurrentTime(unit=milliseconds)) + 1.0;

  var numGenerated = 0;

  def nextrandlc(x : float, a: float = arand) {
    const
      r23   = 0.5**23,
      t23   = 2.0**23,
      r46   = 0.5**46,
      t46   = 2.0**46;
    var t1 = r23 * a;
    var a1 = floor(t1);
    var a2 = a - t23 * a1;
    t1 = r23 * x;
    var x1 = floor(t1);
    var x2 = x - t23 * x1;
    t1 = a1 * x2 + a2 * x1;
    var t2 = floor(r23 * t1);
    var z  = t1 - t23 * t2;
    var t3 = t23 * z + a2 * x2;
    var t4 = floor(r46 * t3);
    var x3 = t3 - t46 * t4;
    return (x3, r46 * x3);
  }

  def initrandlc(in n : int) : float {
    var i : int, t : float, g : float;
    var x : float = seed;
    t = arand;
    while n != 0 do {
      i = n / 2;
      if 2 * i != n then
        (x, g) = nextrandlc(x, t);
      (t, g) = nextrandlc(t, t);
      n = i;
    }
    return x;
  }

  def fillRandom(x:[?D] float) {

    if (D.rank > 1) {
      writeln ("Not yet implemented for 2D or higher arrays.");
      [i in D] x(i) = -1.0;
    }
    else {
      var randlc_last_x: float = initrandlc(numGenerated+1);
      for i in D {
        (randlc_last_x,x(i)) = nextrandlc(randlc_last_x);
        numGenerated += 1;
      }
    }
  }
}

def fillRandom(x:[] float, seed: float = (2.0*getCurrentTime(unit=microseconds)+1.0),
    arand: float = (2.0*(getCurrentTime() + getCurrentTime(unit=milliseconds))+1.0)) {

  var randNums = RandomStream(seed,arand);

  randNums.fillRandom(x);
}


Use the writeXE0, now that it is available, instead of writeXF followed by
readFE.  This also fixes a regression in which line numbers from this file
were explicitly mentioned in error messages.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@12436 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// class file
//
//  chapel-level implementations of read, write, writeln
//  chapel-level implementations of assert, halt

const stdin  = file("stdin", "r", "/dev", _get_stdin());
const stdout = file("stdout", "w", "/dev", _get_stdout());
const stderr = file("stderr", "w", "/dev", _get_stderr());

class file: Writer {
  var filename : string = "";
  var mode : string = "r";
  var path : string = ".";
  var _fp : _file;
  var _lock : sync uint(64);    // for serializing output

  def open() {
    if this == stdin || this == stdout || this == stderr then
      halt("***Error: It is not necessary to open \"", filename, "\"***");

    var fullFilename = path + "/" + filename;
    _fp = _fopen(fullFilename, mode);

    if _fp == _get_nullfile() then
      halt("***Error: Unable to open \"", fullFilename, "\": ", _get_errno(), "***");
  }

  def _checkFileStateChangeLegality(state) {
    if (isOpen) {
      halt("Cannot change ", state, " of file ", path, "/", filename,
           " while it is open");
    }
  }

  def filename var : string {
    if setter then
      _checkFileStateChangeLegality("filename");
    return filename;
  }

  def path var : string {
    if setter then
      _checkFileStateChangeLegality("path");
    return path;
  }

  def mode var {
    if setter then
      _checkFileStateChangeLegality("mode");
    return mode;
  }

  def isOpen: bool {
    var openStatus: bool = false;
    if (_fp != _get_nullfile()) {
      openStatus = true;
    }
    return openStatus;
  }

  def close() {
    if (this == stdin || this == stdout || this == stderr) {
      halt("***Error: You may not close \"", filename, "\"***");
    }
    if (_fp == _get_nullfile()) {
      var fullFilename = path + "/" + filename;
      halt("***Error: Trying to close \"", fullFilename,
           "\" which isn't open***");
    }
    var returnVal: int = _fclose(_fp);
    if (returnVal < 0) {
      var fullFilename = path + "/" + filename;
      halt("***Error: The close of \"", fullFilename, "\" failed: ",
           _get_errno(), "***");
    }
    _fp = _get_nullfile();
  }
}

def file.writeThis(f: Writer) {
  f.write("(filename = ",this.filename);
  f.write(", path = ",this.path);
  f.write(", mode = ",this.mode);
  f.write(")");
}

def file.flush() {
  _fflush(_fp);
}

def _fopenError(f: file, isRead: bool) {
  var fullFilename:string = f.path + "/" + f.filename;
  if (isRead) {
    halt("***Error: You must open \"", fullFilename,
         "\" before trying to read from it***");
  } else {
    halt("***Error: You must open \"", fullFilename,
         "\" before trying to write to it***");
  }
}

def _fmodeError(f: file, isRead: bool) {
  var fullFilename: string = f.path + "/" + f.filename;
  var modestring = if (isRead) then "reading" else "writing";
  halt("***Error: ", fullFilename, " not open for ", modestring, "***");
}

def _fprintfError() {
  halt("***Error: Write failed: ", _get_errno(), "***");
}

def file.readln() {
  if !isOpen then
    _fopenError(this, isRead=true);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout list ...?n) {
  read((...list));
  __primitive("_readToEndOfLine",_fp);
}

// Manually duplicating readln methods for each type.
// This is done because the compiler doesn't yet
// support overloading between generic and type
// arguments.

def file.readln(inout val: int) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: uint) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: real) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: complex) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: string) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: bool) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(type t) {
  var val: t;
  this.readln(val);
  return val;
}

def file.read(inout first, inout rest ...?n) {
  read(first);
  for param i in 1..n do
    read(rest(i));
}

def file.read(inout val: int) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_int32", _fp);
}

def file.read(inout val: uint) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_uint32", _fp);
}

def file.read(inout val: real) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_real64", _fp);
}

def file.read(inout val: complex) {
  var realPart: real;
  var imagPart: real;
  var imagI: string;
  var matchingCharWasRead: int;
  var isNeg: bool;

  read(realPart);
  matchingCharWasRead = _readLitChar(_fp, "+", true);
  if (matchingCharWasRead != 1) {
    matchingCharWasRead = _readLitChar(_fp, "-", true);
    if (matchingCharWasRead != 1) {
      halt("***Error: Incorrect format for complex numbers***");
    }
    isNeg = true;
  }

  read(imagPart);
  matchingCharWasRead = _readLitChar(_fp, "i", false);
  if (matchingCharWasRead != 1) {
    halt("***Error: Incorrect format for complex numbers***");
  }

  val.re = realPart;
  if (isNeg) {
    val.im = -imagPart;
  } else {
    val.im = imagPart;
  }
}

def file.read(inout val: string) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_string", _fp);
}

def file.read(inout val: bool) {
  var s: string;
  this.read(s);
  if (s == "true") {
    val = true;
  } else if (s == "false") {
    val = false;
  } else {
    halt("read of bool value that is neither true nor false");
  }
}

def file.read(type t) {
  var val: t;
  this.read(val);
  return val;
}

def string.writeThis(f: Writer) {
  f.writeIt(this);
}

def file.writeIt(s: string) {
  if !isOpen then
    _fopenError(this, isRead = false);
  if mode != "w" then
    _fmodeError(this, isRead = false);
  var returnVal = fprintf(_fp, "%s", s);
  if returnVal < 0 then
    _fprintfError();
}

class StringClass: Writer {
  var s: string;
  def writeIt(s: string) { this.s += s; }
}

pragma "ref this"
def string.write(args ...?n) {
  var sc = StringClass(this);
  sc.write((...args));
  this = sc.s;
}

def file.lockWrite() {
  var me: uint(64) = __primitive("thread_id");
  if isFull(_lock) then
    if readXX(_lock) == me then
      return false;
  _lock = me;
  return true;
}

def file.unlockWrite() {
  writeXE0(_lock);
}

class Writer {
  def writeIt(s: string);
  def lockWrite() return false;
  def unlockWrite();
  def write(args ...?n) {
    var need_release: bool;
    //    on locale(0) do  // ahh!!! new instance to write; want to use array
      need_release = lockWrite();
    for param i in 1..n do
      args(i).writeThis(this);
    if need_release then
      //      on locale(0) do  // ahh!!! new instance to write; want to use array
        unlockWrite();
  }
  def writeln(args ...?n) {
    write((...args), "\n");
  }
  def writeln() {
    write("\n");
  }
}

def write(args ...?n) {
  stdout.write((...args));
}

def writeln(args ...?n) {
  stdout.writeln((...args));
}

def writeln() {
  stdout.writeln();
}

def read(inout args ...?n) {
  stdin.read((...args));
}

def readln(inout args ...?n) {
  stdin.readln((...args));
}

def readln() {
  stdin.readln();
}

//def read(type t)
//  return stdin.read(t);

def _tuple2string(t) {
  var s: string;
  for param i in 1..t.size do
    s.write(t(i));
  return s;
}

def assert(test: bool) {
  if !test then
    _handleRuntimeError("assert failed");
}

def assert(test: bool, args ...?numArgs) {
  if !test then
    _handleRuntimeError("assert failed - "+_tuple2string(args));
}

def halt() {
  _handleRuntimeError("halt reached");
}

def halt(args ...?numArgs) {
  _handleRuntimeError("halt reached - "+_tuple2string(args));
}

def _debugWrite(args: string ...?n) {
  for param i in 1..n do
    if fprintf(_get_stdout(), "%s", args(i)) < 0 then
      _fprintfError();
  _fflush(_get_stdout());
}

def _debugWriteln(args: string ...?n) {
  _debugWrite((...args), "\n");
}

def _debugWriteln() {
  _debugWrite("\n");
}

def _ddata.writeThis(f: Writer) {
  for i in 0..size-2 do
    f.write(this(i), " ");
  f.write(this(size-1));
}

// class file
//
//  chapel-level implementations of read, write, writeln
//  chapel-level implementations of assert, halt

const stdin  = file("stdin", "r", "/dev", _get_stdin());
const stdout = file("stdout", "w", "/dev", _get_stdout());
const stderr = file("stderr", "w", "/dev", _get_stderr());

class file: Writer {
  var filename : string = "";
  var mode : string = "r";
  var path : string = ".";
  var _fp : _file;
  var _lock : sync uint(64);    // for serializing output

  def open() {
    if this == stdin || this == stdout || this == stderr then
      halt("***Error: It is not necessary to open \"", filename, "\"***");

    var fullFilename = path + "/" + filename;
    _fp = _fopen(fullFilename, mode);

    if _fp == _get_nullfile() then
      halt("***Error: Unable to open \"", fullFilename, "\": ", _get_errno(), "***");
  }

  def _checkFileStateChangeLegality(state) {
    if (isOpen) {
      halt("Cannot change ", state, " of file ", path, "/", filename,
           " while it is open");
    }
  }

  def filename var : string {
    if setter then
      _checkFileStateChangeLegality("filename");
    return filename;
  }

  def path var : string {
    if setter then
      _checkFileStateChangeLegality("path");
    return path;
  }

  def mode var {
    if setter then
      _checkFileStateChangeLegality("mode");
    return mode;
  }

  def isOpen: bool {
    var openStatus: bool = false;
    if (_fp != _get_nullfile()) {
      openStatus = true;
    }
    return openStatus;
  }

  def close() {
    if (this == stdin || this == stdout || this == stderr) {
      halt("***Error: You may not close \"", filename, "\"***");
    }
    if (_fp == _get_nullfile()) {
      var fullFilename = path + "/" + filename;
      halt("***Error: Trying to close \"", fullFilename,
           "\" which isn't open***");
    }
    var returnVal: int = _fclose(_fp);
    if (returnVal < 0) {
      var fullFilename = path + "/" + filename;
      halt("***Error: The close of \"", fullFilename, "\" failed: ",
           _get_errno(), "***");
    }
    _fp = _get_nullfile();
  }
}

def file.writeThis(f: Writer) {
  f.write("(filename = ",this.filename);
  f.write(", path = ",this.path);
  f.write(", mode = ",this.mode);
  f.write(")");
}

def file.flush() {
  _fflush(_fp);
}

def _fopenError(f: file, isRead: bool) {
  var fullFilename:string = f.path + "/" + f.filename;
  if (isRead) {
    halt("***Error: You must open \"", fullFilename,
         "\" before trying to read from it***");
  } else {
    halt("***Error: You must open \"", fullFilename,
         "\" before trying to write to it***");
  }
}

def _fmodeError(f: file, isRead: bool) {
  var fullFilename: string = f.path + "/" + f.filename;
  var modestring = if (isRead) then "reading" else "writing";
  halt("***Error: ", fullFilename, " not open for ", modestring, "***");
}

def _fprintfError() {
  halt("***Error: Write failed: ", _get_errno(), "***");
}

def file.readln() {
  if !isOpen then
    _fopenError(this, isRead=true);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout list ...?n) {
  read((...list));
  __primitive("_readToEndOfLine",_fp);
}

// Manually duplicating readln methods for each type.
// This is done because the compiler doesn't yet
// support overloading between generic and type
// arguments.

def file.readln(inout val: int) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: uint) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: real) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: complex) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: string) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(inout val: bool) {
  read(val);
  __primitive("_readToEndOfLine",_fp);
}

def file.readln(type t) {
  var val: t;
  this.readln(val);
  return val;
}

def file.read(inout first, inout rest ...?n) {
  read(first);
  for param i in 1..n do
    read(rest(i));
}

def file.read(inout val: int) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_int32", _fp);
}

def file.read(inout val: uint) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_uint32", _fp);
}

def file.read(inout val: real) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_real64", _fp);
}

def file.read(inout val: complex) {
  var realPart: real;
  var imagPart: real;
  var imagI: string;
  var matchingCharWasRead: int;
  var isNeg: bool;

  read(realPart);
  matchingCharWasRead = _readLitChar(_fp, "+", true);
  if (matchingCharWasRead != 1) {
    matchingCharWasRead = _readLitChar(_fp, "-", true);
    if (matchingCharWasRead != 1) {
      halt("***Error: Incorrect format for complex numbers***");
    }
    isNeg = true;
  }

  read(imagPart);
  matchingCharWasRead = _readLitChar(_fp, "i", false);
  if (matchingCharWasRead != 1) {
    halt("***Error: Incorrect format for complex numbers***");
  }

  val.re = realPart;
  if (isNeg) {
    val.im = -imagPart;
  } else {
    val.im = imagPart;
  }
}

def file.read(inout val: string) {
  if !isOpen then
    _fopenError(this, isRead=true);
  val = __primitive("_fscan_string", _fp);
}

def file.read(inout val: bool) {
  var s: string;
  this.read(s);
  if (s == "true") {
    val = true;
  } else if (s == "false") {
    val = false;
  } else {
    halt("read of bool value that is neither true nor false");
  }
}

def file.read(type t) {
  var val: t;
  this.read(val);
  return val;
}

def string.writeThis(f: Writer) {
  f.writeIt(this);
}

def file.writeIt(s: string) {
  if !isOpen then
    _fopenError(this, isRead = false);
  if mode != "w" then
    _fmodeError(this, isRead = false);
  var returnVal = fprintf(_fp, "%s", s);
  if returnVal < 0 then
    _fprintfError();
}

class StringClass: Writer {
  var s: string;
  def writeIt(s: string) { this.s += s; }
}

pragma "ref this"
def string.write(args ...?n) {
  var sc = StringClass(this);
  sc.write((...args));
  this = sc.s;
}

def file.lockWrite() {
  var me: uint(64) = __primitive("thread_id");
  if isFull(_lock) then
    if readXX(_lock) == me then
      return false;
  _lock = me;
  return true;
}

def file.unlockWrite() {
  // Perhaps it's not necessary to reset to zero?
  // Alternatively, perhaps the purge() generic function should somehow be called?
  //writeFE(_lock, 0); // this can also be 'writeXE' since _lock is full
  writeXF(_lock, 0:uint(64));
  readFE(_lock);
}

class Writer {
  def writeIt(s: string);
  def lockWrite() return false;
  def unlockWrite();
  def write(args ...?n) {
    var need_release: bool;
    //    on locale(0) do  // ahh!!! new instance to write; want to use array
      need_release = lockWrite();
    for param i in 1..n do
      args(i).writeThis(this);
    if need_release then
      //      on locale(0) do  // ahh!!! new instance to write; want to use array
        unlockWrite();
  }
  def writeln(args ...?n) {
    write((...args), "\n");
  }
  def writeln() {
    write("\n");
  }
}

def write(args ...?n) {
  stdout.write((...args));
}

def writeln(args ...?n) {
  stdout.writeln((...args));
}

def writeln() {
  stdout.writeln();
}

def read(inout args ...?n) {
  stdin.read((...args));
}

def readln(inout args ...?n) {
  stdin.readln((...args));
}

def readln() {
  stdin.readln();
}

//def read(type t)
//  return stdin.read(t);

def _tuple2string(t) {
  var s: string;
  for param i in 1..t.size do
    s.write(t(i));
  return s;
}

def assert(test: bool) {
  if !test then
    _handleRuntimeError("assert failed");
}

def assert(test: bool, args ...?numArgs) {
  if !test then
    _handleRuntimeError("assert failed - "+_tuple2string(args));
}

def halt() {
  _handleRuntimeError("halt reached");
}

def halt(args ...?numArgs) {
  _handleRuntimeError("halt reached - "+_tuple2string(args));
}

def _debugWrite(args: string ...?n) {
  for param i in 1..n do
    if fprintf(_get_stdout(), "%s", args(i)) < 0 then
      _fprintfError();
  _fflush(_get_stdout());
}

def _debugWriteln(args: string ...?n) {
  _debugWrite((...args), "\n");
}

def _debugWriteln() {
  _debugWrite("\n");
}

def _ddata.writeThis(f: Writer) {
  for i in 0..size-2 do
    f.write(this(i), " ");
  f.write(this(size-1));
}


Add the "dsi" prefix changes to BlockDist. In revision 16597, I missed this
file when explicitly specifying the files to commit on the svn command line.

Thanks Steve for catching it.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@16598 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Block Distribution
//
//      Block       BlockDom     BlockArr
//
//   LocBlock    LocBlockDom  LocBlockArr
//

//
// Limitations
//
// 1. Changes to Block.tasksPerLocale are not made in privatized
//    copies of this distribution.
//

config param debugBlockDist = false; // internal development flag (debugging)
config param sanityCheckDistribution = false; // ditto; should promote to compiler flag

////////////////////////////////////////////////////////////////////////////////
// Block Distribution Class
//
class Block : BaseDist {
  param rank: int;
  type idxType = int;

  const boundingBox: domain(rank, idxType);
  const targetLocDom: domain(rank);
  const targetLocs: [targetLocDom] locale;
  const locDist: [targetLocDom] LocBlock(rank, idxType);

  var tasksPerLocale: int; // tasks per locale for forall iteration

  var pid: int = -1; // privatized object id

  def Block(param rank: int,
            type idxType = int,
            bbox: domain(rank, idxType),
            targetLocales: [] locale = thisRealm.Locales,
            tasksPerLocale = 0) {
    boundingBox = bbox;
    if rank != 1 && targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank do
        ranges(i) = 0..#factors(i);
      targetLocDom = [(...ranges)];
      targetLocs = reshape(targetLocales, targetLocDom);
    } else {
      if targetLocales.rank != rank then
	compilerError("targetLocales rank must equal distribution rank or one");

      var ranges: rank*range;
      for param i in 1..rank do
	ranges(i) = 0..#targetLocales.domain.dim(i).length;

      targetLocDom = [(...ranges)];
      targetLocs = targetLocales;
    }

    if debugBlockDist {
      writeln(targetLocDom);
      writeln(targetLocs);
    }

    const boundingBoxDims = boundingBox.dims();
    const targetLocDomDims = targetLocDom.dims();
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) =  new LocBlock(rank, idxType, locid, boundingBoxDims,
                                       targetLocDomDims);

    if tasksPerLocale == 0 then
      this.tasksPerLocale = min reduce targetLocs.numCores;
    else
      this.tasksPerLocale = tasksPerLocale;

    if debugBlockDist then
      for loc in locDist do writeln(loc);
  }

  // copy constructor for privatization
  def Block(param rank: int, type idxType, other: Block(rank, idxType), privateData) {
    boundingBox = [(...privateData(1))];
    targetLocDom = [(...privateData(2))];
    tasksPerLocale = privateData(3);
    for i in targetLocDom {
      targetLocs(i) = other.targetLocs(i);
      locDist(i) = other.locDist(i);
    }
  }

  def dsiClone() {
    return new Block(rank, idxType, boundingBox, targetLocs, tasksPerLocale);
  }

  def dsiDestroyDistClass() {
    coforall ld in locDist do {
      on ld do
        delete ld;
    }
  }
}

// dsi
def Block.dsiSupportsPrivatization() param return true;

// dsi
def Block.getPrivatizeData() {
  return (boundingBox.dims(), targetLocDom.dims(), tasksPerLocale);
}

// dsi
def Block.privatize(privatizeData) {
  return new Block(rank=rank, idxType=idxType, this, privatizeData);
}

// dsi
def Block.reprivatize(other) {
  boundingBox = other.boundingBox;
  targetLocDom = other.targetLocDom;
  targetLocs = other.targetLocs;
  locDist = other.locDist;
  tasksPerLocale = other.tasksPerLocale;
}

//
// create a new arithmetic domain over this distribution
//
def Block.dsiNewArithmeticDom(param rank: int, type idxType,
                           param stridable: bool) {
  if idxType != this.idxType then
    compilerError("Block domain index type does not match distribution's");
  if rank != this.rank then
    compilerError("Block domain rank does not match distribution's");

  var dom = new BlockDom(rank=rank, idxType=idxType, dist=this, stridable=stridable);
  dom.setup();
  return dom;
}

//
// output distribution
//
def Block.writeThis(x:Writer) {
  x.writeln("Block");
  x.writeln("-------");
  x.writeln("distributes: ", boundingBox);
  x.writeln("across locales: ", targetLocs);
  x.writeln("indexed via: ", targetLocDom);
  x.writeln("resulting in: ");
  for locid in targetLocDom do
    x.writeln("  [", locid, "] ", locDist(locid));
}

//
// convert an index into a locale value
//
// dsiIndexLocale
//  note - we only need the second one
def Block.ind2loc(ind: idxType) where rank == 1 {
  return targetLocs(ind2locInd(ind));
}

def Block.ind2loc(ind: rank*idxType) {
  return targetLocs(ind2locInd(ind));
}

//
// compute what chunk of inds is owned by a given locale -- assumes
// it's being called on the locale in question
//
def Block.getChunk(inds, locid) {
  // use domain slicing to get the intersection between what the
  // locale owns and the domain's index set
  //
  // TODO: Should this be able to be written as myChunk[inds] ???
  //
  // TODO: Does using David's detupling trick work here?
  //
  const chunk = locDist(locid).myChunk((...inds.getIndices()));
  if sanityCheckDistribution then
    if chunk.numIndices > 0 {
      if ind2locInd(chunk.low) != locid then
        writeln("[", here.id, "] ", chunk.low, " is in my chunk but maps to ",
                ind2locInd(chunk.low));
      if ind2locInd(chunk.high) != locid then
        writeln("[", here.id, "] ", chunk.high, " is in my chunk but maps to ",
                ind2locInd(chunk.high));
    }
  return chunk;
}

//
// determine which locale owns a particular index
//
// TODO: I jotted down a note during the code review asking whether
// targetLocs.numElements and boundingbox.numIndices should be
// captured locally, or captured in the default dom/array implementation
// or inlined.  Not sure what that point was anymore, though.  Maybe
// someone else can help me remember it (since it was probably someone
// else's suggestion).
//
def Block.ind2locInd(ind: idxType) where rank == 1 {
  const ind0 = ind - boundingBox.low;
  const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
  return max(0, min(locInd:int, (targetLocDom.dim(1).length-1):int));
}

def Block.ind2locInd(ind: rank*idxType) where rank == 1 {
  return ind2locInd(ind(1));
}

def Block.ind2locInd(ind: rank*idxType) where rank != 1 {
  var locInd: rank*int;
  for param i in 1..rank {
    const ind0 = ind(i) - boundingBox.low(i);
    const dimLocInd = (ind0 * targetLocDom.dim(i).length:idxType) / boundingBox.dim(i).length;
    locInd(i) = max(0, min(dimLocInd:int, (targetLocDom.dim(i).length-1):int));
  }
  return locInd;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Distribution Class
//
class LocBlock {
  param rank: int;
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(rank, idxType);

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock(param rank: int,
               type idxType,
               locid,   // the locale index from the target domain
               boundingBox: rank*range(idxType),
               targetLocBox: rank*range) {
    if rank == 1 {
      const lo = boundingBox(1).low;
      const hi = boundingBox(1).high;
      const numelems = hi - lo + 1;
      const numlocs = targetLocBox(1).length;
      const (blo, bhi) = _computeBlock(min(idxType), numelems, lo,
                                       max(idxType), numlocs, locid);
      myChunk = [blo..bhi];
    } else {
      var tuple: rank*range(idxType);
      for param i in 1..rank {
        const lo = boundingBox(i).low;
        const hi = boundingBox(i).high;
        const numelems = hi - lo + 1;
        const numlocs = targetLocBox(i).length;
        const (blo, bhi) = _computeBlock(min(idxType), numelems, lo,
                                         max(idxType), numlocs, locid(i));
        tuple(i) = blo..bhi;
      }
      myChunk = [(...tuple)];
    }
  }
}

def LocBlock.writeThis(x:Writer) {
  var localeid: int;
  on this {
    localeid = here.uid;
  }
  x.write("locale ", localeid, " owns chunk: ", myChunk);
}

////////////////////////////////////////////////////////////////////////////////
// Block Domain Class
//
class BlockDom: BaseArithmeticDom {
  param rank: int;
  type idxType;
  param stridable: bool;
  //
  // LEFT LINK: a pointer to the parent distribution
  //
  const dist: Block(rank, idxType);

  //
  // DOWN LINK: an array of local domain class descriptors -- set up in
  // setup() below
  //
  var locDoms: [dist.targetLocDom] LocBlockDom(rank, idxType, stridable);

  //
  // a domain describing the complete domain
  //
  const whole: domain(rank=rank, idxType=idxType, stridable=stridable);

  var pid: int = -1; // privatized object id

  def getBaseDist() return dist;
}

def BlockDom.dsiDims() return whole.dims();

def BlockDom.dsiDim(d: int) return whole.dim(d);

// stopgap to avoid accessing locDoms field (and returning an array)
def BlockDom.getLocDom(localeIdx) return locDoms(localeIdx);

def BlockDom.these() {
  for i in whole do
    yield i;
}

def BlockDom.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dist.tasksPerLocale;
  const precomputedWholeLow = whole.low;
  coforall locDom in locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;

    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }
    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).low, locBlock(1).length,
                                      locBlock(1).low, locBlock(1).high,
                                      numTasks, taskid);
        tuple(1) = lo..hi;
        yield tuple;
      }
    }
  }
}

//
// TODO: Abstract the addition of low into a function?
// Note relationship between this operation and the
// order/position functions -- any chance for creating similar
// support? (esp. given how frequent this seems likely to be?)
//
// TODO: Is there some clever way to invoke the leader/follower
// iterator on the local blocks in here such that the per-core
// parallelism is expressed at that level?  Seems like a nice
// natural composition and might help with my fears about how
// stencil communication will be done on a per-locale basis.
//
def BlockDom.these(param tag: iterator, follower) where tag == iterator.follower {
  var t: rank*range(idxType, stridable=stridable);
  for param i in 1..rank {
    var stride = whole.dim(i).stride: idxType;
    var low = stride * follower(i).low;
    var high = stride * follower(i).high;
    t(i) = (low..high by stride:int) + whole.dim(i).low;
  }
  for i in [(...t)] {
    yield i;
  }
}

def BlockDom.dsiStrideBy(str: int) {
  var alias = new BlockDom(rank=rank, idxType=idxType, stridable=true, dist=dist);
  var t: rank*range(eltType=idxType, stridable=true);
  for i in 1..rank {
    t(i) = this.dsiDim(i) by str;
  }
  alias.dsiSetIndices(t);
  return alias;
}

def BlockDom.dsiStrideBy(str: rank*int) {
  var alias = new BlockDom(rank=rank, idxType=idxType, stridable=true, dist=dist);
  var t: rank*range(eltType=idxType, stridable=true);
  for i in 1..rank {
    t(i) = this.dim(i) by str(i);
  }
  alias.setIndices(t);
  return alias;
}

//
// output domain
//
def BlockDom.dsiSerialWrite(x:Writer) {
  x.write(whole);
}

//
// how to allocate a new array over this domain
//
def BlockDom.dsiBuildArray(type eltType) {
  var arr = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this);
  arr.setup();
  return arr;
}

def BlockDom.dsiNumIndices return whole.numIndices;
def BlockDom.dsiLow return whole.low;
def BlockDom.dsiHigh return whole.high;

//
// INTERFACE NOTES: Could we make dsiSetIndices() for an arithmetic
// domain take a domain rather than something else?
//
def BlockDom.dsiSetIndices(x: domain) {
  if x.rank != rank then
    compilerError("rank mismatch in domain assignment");
  if x._value.idxType != idxType then
    compilerError("index type mismatch in domain assignment");
  whole = x;
  setup();
}

def BlockDom.dsiSetIndices(x) {
  if x.size != rank then
    compilerError("rank mismatch in domain assignment");
  if x(1).eltType != idxType then
    compilerError("index type mismatch in domain assignment");
  //
  // TODO: This seems weird:
  //
  whole.setIndices(x);
  setup();
}

def BlockDom.dsiGetIndices() {
  return whole.getIndices();
}

// remove all instances of getDist
def BlockDom.getDist(): Block(idxType) {
  return dist;
}

def BlockDom.dsiSlice(param stridable: bool, ranges) {
  var d = new BlockDom(rank=rank, idxType=idxType, dist=dist, stridable=stridable||this.stridable);
  d.dsiSetIndices(whole((...ranges)).getIndices());
  return d;
}

// dsiLocalSlice
def BlockDom.localSlice(param stridable: bool, ranges) {
  return whole((...ranges));
}

def BlockDom.setup() {
  if locDoms(dist.targetLocDom.low) == nil {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocBlockDom(rank, idxType, stridable, this,
                                             dist.getChunk(whole, localeIdx));
    }
  } else {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    }
  }
  if debugBlockDist then
    for loc in dist.targetLocDom do writeln(loc, " owns ", locDoms(loc));
}

def BlockDom.dsiSupportsPrivatization() param return true;

def BlockDom.dsiGetPrivatizeData() return (dist.pid, whole.dims());

def BlockDom.dsiPrivatize(privatizeData) {
  var distpid = privatizeData(1);
  var thisdist = dist;
  var privdist = __primitive("chpl_getPrivatizedClass", thisdist, distpid);
  var c = new BlockDom(rank=rank, idxType=idxType, stridable=stridable, dist=privdist);
  for i in c.dist.targetLocDom do
    c.locDoms(i) = locDoms(i);
  c.whole = [(...privatizeData(2))];
  return c;
}

def BlockDom.dsiGetReprivatizeData() return whole.dims();

def BlockDom.dsiReprivatize(other, reprivatizeData) {
  for i in dist.targetLocDom do
    locDoms(i) = other.locDoms(i);
  whole = [(...reprivatizeData)];
}

def BlockDom.dsiMember(i) {
  return whole.member(i);
}

def BlockDom.dsiIndexOrder(i) {
  return whole.order(i);
}

def BlockDom.dsiPosition(i) {
  return whole.position(i);
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Domain Class
//
class LocBlockDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // UP LINK: a reference to the parent global domain class
  //
  const wholeDom: BlockDom(rank, idxType, stridable);

  //
  // a local domain describing the indices owned by this locale
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(rank, idxType, stridable);
}

//
// output local domain piece
//
def LocBlockDom.writeThis(x:Writer) {
  x.write(myBlock);
}

//
// Added as a performance stopgap to avoid returning a domain
//
def LocBlockDom.member(i) return myBlock.member(i);

////////////////////////////////////////////////////////////////////////////////
// Block Array Class
//
class BlockArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: the global domain descriptor for this array
  //
  var dom: BlockDom(rank, idxType, stridable);

  //
  // DOWN LINK: an array of local array classes
  //
  var locArr: [dom.dist.targetLocDom] LocBlockArr(eltType, rank, idxType, stridable);

  //
  // optimized reference to a local LocBlockArr instance (or nil)
  //
  var myLocArr: LocBlockArr(eltType, rank, idxType, stridable);

  var pid: int = -1; // privatized object id
}

def BlockArr.dsiGetBaseDom() return dom;

def BlockArr.setup() {
  var thisid = this.locale.uid;
  coforall localeIdx in dom.dist.targetLocDom {
    on dom.dist.targetLocs(localeIdx) {
      const locDom = dom.getLocDom(localeIdx);
      locArr(localeIdx) = new LocBlockArr(eltType, rank, idxType, stridable, locDom);
      if thisid == here.uid then
        myLocArr = locArr(localeIdx);
    }
  }
}

def BlockArr.dsiSupportsPrivatization() param return true;

def BlockArr.dsiGetPrivatizeData() return dom.pid;

def BlockArr.dsiPrivatize(privatizeData) {
  var dompid = privatizeData;
  var thisdom = dom;
  var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
  var c = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
  for localeIdx in c.dom.dist.targetLocDom {
    c.locArr(localeIdx) = locArr(localeIdx);
    if c.locArr(localeIdx).locale.uid == here.uid then
      c.myLocArr = c.locArr(localeIdx);
  }
  return c;
}

//
// the global accessor for the array
//
// TODO: Do we need a global bounds check here or in ind2locind?
//
def BlockArr.dsiAccess(i: idxType) var where rank == 1 {
  if myLocArr then local {
    if myLocArr.locDom.member(i) then
      return myLocArr.this(i);
  }
  return locArr(dom.dist.ind2locInd(i))(i);
}

def BlockArr.dsiAccess(i: rank*idxType) var {
  if rank == 1 {
    return dsiAccess(i(1));
  } else {
    if myLocArr then local {
      if myLocArr.locDom.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }
}

def BlockArr.these() var {
  for i in dom do
    yield dsiAccess(i);
}

//
// TODO: Rewrite this to reuse more of the global domain iterator
// logic?  (e.g., can we forward the forall to the global domain
// somehow?
//
def BlockArr.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dom.dist.tasksPerLocale;
  const precomputedWholeLow = dom.whole.low;
  coforall locDom in dom.locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;
    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }

    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).low, locBlock(1).length,
                                      locBlock(1).low, locBlock(1).high,
                                      numTasks, taskid);

        tuple(1) = lo..hi;
        yield tuple;

      }
    }
  }
}

def BlockArr.dsiSupportsAlignedFollower() param return true;

def BlockArr.these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
  var followThis: rank*range(eltType=idxType, stridable=stridable);
  var lowIdx: rank*idxType;

  for param i in 1..rank {
    var stride = dom.whole.dim(i).stride;
    var low = follower(i).low * stride;
    var high = follower(i).high * stride;
    followThis(i) = (low..high by stride) + dom.whole.dim(i).low;
    lowIdx(i) = followThis(i).low;
  }
  const followThisDom = [(...followThis)];

  //
  // TODO: The following is a buggy hack that will only work when we're
  // distributing across the entire Locales array.  I still think the
  // locArr/locDoms arrays should be associative over locale values.
  //
  var arrSection = locArr(dom.dist.ind2locInd(lowIdx));
  if aligned {
    //
    // if arrSection is not local and we're aligned, it means that
    // followThisDom is empty; make arrSection local so that we can
    // use the local block below
    //
    if arrSection.locale.uid != here.uid then
      arrSection = myLocArr;
    local {
      for e in arrSection.myElems(followThisDom) do
        yield e;
    }
  } else {
    //
    // we don't own all the elements we're following
    //
    def accessHelper(i) var {
      local {
        if myLocArr.locDom.member(i) then
          return myLocArr.this(i);
      }
      return dsiAccess(i);
    }
    for i in followThisDom {
      yield accessHelper(i);
    }
  }
}

//
// output array
//
def BlockArr.dsiSerialWrite(f: Writer) {
  if dom.dsiNumIndices == 0 then return;
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dsiDim(dim)._low;
  label next while true {
    f.write(dsiAccess(i));
    if i(rank) <= (dom.dsiDim(rank)._high - dom.dsiDim(rank)._stride:idxType) {
      f.write(" ");
      i(rank) += dom.dsiDim(rank)._stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dsiDim(dim)._high - dom.dsiDim(dim)._stride:idxType) {
          i(dim) += dom.dsiDim(dim)._stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dsiDim(dim2)._low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

def BlockArr.dsiCheckSlice(ranges) {
  for param i in 1..rank do
    if !dom.dsiDim(i).boundsCheck(ranges(i)) then {
      writeln(dom.dsiDim(i), " ", ranges(i), " ", dom.dsiDim(i).boundsCheck(ranges(i)));
      halt("array slice out of bounds in dimension ", i, ": ", ranges(i));
    }
}

def BlockArr.dsiSlice(d: BlockDom) {
  var alias = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, dom=d, pid=pid);
  for i in dom.dist.targetLocDom {
    on dom.dist.targetLocs(i) {
      alias.locArr[i] = new LocBlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, locDom=d.locDoms[i]);
      alias.locArr[i].myElems => locArr[i].myElems[alias.locArr[i].locDom.myBlock];
    }
  }

  return alias;
}

def BlockArr.localSlice(ranges) {
  var low: rank*idxType;
  for param i in 1..rank {
    low(i) = ranges(i).low;
  }
  var A => locArr(dom.dist.ind2locInd(low)).myElems((...ranges));
  return A;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Array Class
//
class LocBlockArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlockDom(rank, idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;
}

//
// the accessor for the local array -- assumes the index is local
//
def LocBlockArr.this(i) var {
  return myElems(i);
}

//
// output local array piece
//
def LocBlockArr.writeThis(x: Writer) {
  // note on this fails; see writeThisUsingOn.chpl
  x.write(myElems);
}

//
// helper function for blocking index ranges
//
// 12/8/09 note: move this for sharing with forall over default
//               arrays, etc.
//
def _computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def intCeilXDivByY(x, y) return ((x + (y-1)) / y);

  const blo =
    if blocknum == 0 then waylo
    else lo + intCeilXDivByY(numelems * blocknum:lo.type, numblocks:lo.type);
  const bhi =
    if blocknum == numblocks - 1 then wayhi
    else lo + intCeilXDivByY(numelems * (blocknum+1):lo.type, numblocks:lo.type) - 1;

  return (blo, bhi);
}

//
// naive routine for dividing numLocales into rank factors
//
// move somewhere and share.
def _factor(param rank: int, value) {
  var factors: rank*int;
  for param i in 1..rank do
    factors(i) = 1;
  if value >= 1 {
    var iv = value;
    var factor = 1;
    while iv > 1 {
      for i in 2..iv {
        if iv % i == 0 {
          var j = 1;
          for i in 2..rank {
            if factors(i) < factors(j) then
              j = i;
          }
          factors(j) *= i;
          iv = iv / i;
          break;
        }
      }
    }
  }
  for i in 1..rank do
    for j in i+1..rank do
      if factors(i) < factors(j) then
        factors(i) <=> factors(j);
  return factors;
}

//
// Block Distribution
//
//      Block       BlockDom     BlockArr
//
//   LocBlock    LocBlockDom  LocBlockArr
//

//
// Limitations
//
// 1. Changes to Block.tasksPerLocale are not made in privatized
//    copies of this distribution.
//

config param debugBlockDist = false; // internal development flag (debugging)
config param sanityCheckDistribution = false; // ditto; should promote to compiler flag

////////////////////////////////////////////////////////////////////////////////
// Block Distribution Class
//
class Block : BaseDist {
  param rank: int;
  type idxType = int;

  const boundingBox: domain(rank, idxType);
  const targetLocDom: domain(rank);
  const targetLocs: [targetLocDom] locale;
  const locDist: [targetLocDom] LocBlock(rank, idxType);

  var tasksPerLocale: int; // tasks per locale for forall iteration

  var pid: int = -1; // privatized object id

  def Block(param rank: int,
            type idxType = int,
            bbox: domain(rank, idxType),
            targetLocales: [] locale = thisRealm.Locales,
            tasksPerLocale = 0) {
    boundingBox = bbox;
    if rank != 1 && targetLocales.rank == 1 {
      const factors = _factor(rank, targetLocales.numElements);
      var ranges: rank*range;
      for param i in 1..rank do
        ranges(i) = 0..#factors(i);
      targetLocDom = [(...ranges)];
      targetLocs = reshape(targetLocales, targetLocDom);
    } else {
      if targetLocales.rank != rank then
	compilerError("targetLocales rank must equal distribution rank or one");

      var ranges: rank*range;
      for param i in 1..rank do
	ranges(i) = 0..#targetLocales.domain.dim(i).length;

      targetLocDom = [(...ranges)];
      targetLocs = targetLocales;
    }

    if debugBlockDist {
      writeln(targetLocDom);
      writeln(targetLocs);
    }

    const boundingBoxDims = boundingBox.dims();
    const targetLocDomDims = targetLocDom.dims();
    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) =  new LocBlock(rank, idxType, locid, boundingBoxDims,
                                       targetLocDomDims);

    if tasksPerLocale == 0 then
      this.tasksPerLocale = min reduce targetLocs.numCores;
    else
      this.tasksPerLocale = tasksPerLocale;

    if debugBlockDist then
      for loc in locDist do writeln(loc);
  }

  // copy constructor for privatization
  def Block(param rank: int, type idxType, other: Block(rank, idxType), privateData) {
    boundingBox = [(...privateData(1))];
    targetLocDom = [(...privateData(2))];
    tasksPerLocale = privateData(3);
    for i in targetLocDom {
      targetLocs(i) = other.targetLocs(i);
      locDist(i) = other.locDist(i);
    }
  }

  def dsiClone() {
    return new Block(rank, idxType, boundingBox, targetLocs, tasksPerLocale);
  }

  def dsiDestroyDistClass() {
    coforall ld in locDist do {
      on ld do
        delete ld;
    }
  }
}

// dsi
def Block.supportsPrivatization() param return true;

// dsi
def Block.getPrivatizeData() {
  return (boundingBox.dims(), targetLocDom.dims(), tasksPerLocale);
}

// dsi
def Block.privatize(privatizeData) {
  return new Block(rank=rank, idxType=idxType, this, privatizeData);
}

// dsi
def Block.reprivatize(other) {
  boundingBox = other.boundingBox;
  targetLocDom = other.targetLocDom;
  targetLocs = other.targetLocs;
  locDist = other.locDist;
  tasksPerLocale = other.tasksPerLocale;
}

//
// create a new arithmetic domain over this distribution
//
def Block.dsiNewArithmeticDom(param rank: int, type idxType,
                           param stridable: bool) {
  if idxType != this.idxType then
    compilerError("Block domain index type does not match distribution's");
  if rank != this.rank then
    compilerError("Block domain rank does not match distribution's");

  var dom = new BlockDom(rank=rank, idxType=idxType, dist=this, stridable=stridable);
  dom.setup();
  return dom;
}

//
// output distribution
//
// dsiWrite
def Block.writeThis(x:Writer) {
  x.writeln("Block");
  x.writeln("-------");
  x.writeln("distributes: ", boundingBox);
  x.writeln("across locales: ", targetLocs);
  x.writeln("indexed via: ", targetLocDom);
  x.writeln("resulting in: ");
  for locid in targetLocDom do
    x.writeln("  [", locid, "] ", locDist(locid));
}

//
// convert an index into a locale value
//
// dsiIndexLocale
//  note - we only need the second one
def Block.ind2loc(ind: idxType) where rank == 1 {
  return targetLocs(ind2locInd(ind));
}

def Block.ind2loc(ind: rank*idxType) {
  return targetLocs(ind2locInd(ind));
}

//
// compute what chunk of inds is owned by a given locale -- assumes
// it's being called on the locale in question
//
def Block.getChunk(inds, locid) {
  // use domain slicing to get the intersection between what the
  // locale owns and the domain's index set
  //
  // TODO: Should this be able to be written as myChunk[inds] ???
  //
  // TODO: Does using David's detupling trick work here?
  //
  const chunk = locDist(locid).myChunk((...inds.getIndices()));
  if sanityCheckDistribution then
    if chunk.numIndices > 0 {
      if ind2locInd(chunk.low) != locid then
        writeln("[", here.id, "] ", chunk.low, " is in my chunk but maps to ",
                ind2locInd(chunk.low));
      if ind2locInd(chunk.high) != locid then
        writeln("[", here.id, "] ", chunk.high, " is in my chunk but maps to ",
                ind2locInd(chunk.high));
    }
  return chunk;
}

//
// determine which locale owns a particular index
//
// TODO: I jotted down a note during the code review asking whether
// targetLocs.numElements and boundingbox.numIndices should be
// captured locally, or captured in the default dom/array implementation
// or inlined.  Not sure what that point was anymore, though.  Maybe
// someone else can help me remember it (since it was probably someone
// else's suggestion).
//
def Block.ind2locInd(ind: idxType) where rank == 1 {
  const ind0 = ind - boundingBox.low;
  const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
  return max(0, min(locInd:int, (targetLocDom.dim(1).length-1):int));
}

def Block.ind2locInd(ind: rank*idxType) where rank == 1 {
  return ind2locInd(ind(1));
}

def Block.ind2locInd(ind: rank*idxType) where rank != 1 {
  var locInd: rank*int;
  for param i in 1..rank {
    const ind0 = ind(i) - boundingBox.low(i);
    const dimLocInd = (ind0 * targetLocDom.dim(i).length:idxType) / boundingBox.dim(i).length;
    locInd(i) = max(0, min(dimLocInd:int, (targetLocDom.dim(i).length-1):int));
  }
  return locInd;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Distribution Class
//
class LocBlock {
  param rank: int;
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(rank, idxType);

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock(param rank: int,
               type idxType,
               locid,   // the locale index from the target domain
               boundingBox: rank*range(idxType),
               targetLocBox: rank*range) {
    if rank == 1 {
      const lo = boundingBox(1).low;
      const hi = boundingBox(1).high;
      const numelems = hi - lo + 1;
      const numlocs = targetLocBox(1).length;
      const (blo, bhi) = _computeBlock(min(idxType), numelems, lo,
                                       max(idxType), numlocs, locid);
      myChunk = [blo..bhi];
    } else {
      var tuple: rank*range(idxType);
      for param i in 1..rank {
        const lo = boundingBox(i).low;
        const hi = boundingBox(i).high;
        const numelems = hi - lo + 1;
        const numlocs = targetLocBox(i).length;
        const (blo, bhi) = _computeBlock(min(idxType), numelems, lo,
                                         max(idxType), numlocs, locid(i));
        tuple(i) = blo..bhi;
      }
      myChunk = [(...tuple)];
    }
  }
}

def LocBlock.writeThis(x:Writer) {
  var localeid: int;
  on this {
    localeid = here.uid;
  }
  x.write("locale ", localeid, " owns chunk: ", myChunk);
}

////////////////////////////////////////////////////////////////////////////////
// Block Domain Class
//
class BlockDom: BaseArithmeticDom {
  param rank: int;
  type idxType;
  param stridable: bool;
  //
  // LEFT LINK: a pointer to the parent distribution
  //
  const dist: Block(rank, idxType);

  //
  // DOWN LINK: an array of local domain class descriptors -- set up in
  // setup() below
  //
  var locDoms: [dist.targetLocDom] LocBlockDom(rank, idxType, stridable);

  //
  // a domain describing the complete domain
  //
  const whole: domain(rank=rank, idxType=idxType, stridable=stridable);

  var pid: int = -1; // privatized object id

  def getBaseDist() return dist;
}

def BlockDom.dims() return whole.dims();

def BlockDom.dim(d: int) return whole.dim(d);

// stopgap to avoid accessing locDoms field (and returning an array)
def BlockDom.getLocDom(localeIdx) return locDoms(localeIdx);

def BlockDom.these() {
  for i in whole do
    yield i;
}

def BlockDom.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dist.tasksPerLocale;
  const precomputedWholeLow = whole.low;
  coforall locDom in locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;

    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }
    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).low, locBlock(1).length,
                                      locBlock(1).low, locBlock(1).high,
                                      numTasks, taskid);
        tuple(1) = lo..hi;
        yield tuple;
      }
    }
  }
}

//
// TODO: Abstract the addition of low into a function?
// Note relationship between this operation and the
// order/position functions -- any chance for creating similar
// support? (esp. given how frequent this seems likely to be?)
//
// TODO: Is there some clever way to invoke the leader/follower
// iterator on the local blocks in here such that the per-core
// parallelism is expressed at that level?  Seems like a nice
// natural composition and might help with my fears about how
// stencil communication will be done on a per-locale basis.
//
def BlockDom.these(param tag: iterator, follower) where tag == iterator.follower {
  var t: rank*range(idxType, stridable=stridable);
  for param i in 1..rank {
    var stride = whole.dim(i).stride: idxType;
    var low = stride * follower(i).low;
    var high = stride * follower(i).high;
    t(i) = (low..high by stride:int) + whole.dim(i).low;
  }
  for i in [(...t)] {
    yield i;
  }
}

def BlockDom.strideBy(str: int) {
  var alias = new BlockDom(rank=rank, idxType=idxType, stridable=true, dist=dist);
  var t: rank*range(eltType=idxType, stridable=true);
  for i in 1..rank {
    t(i) = this.dim(i) by str;
  }
  alias.setIndices(t);
  return alias;
}

def BlockDom.strideBy(str: rank*int) {
  var alias = new BlockDom(rank=rank, idxType=idxType, stridable=true, dist=dist);
  var t: rank*range(eltType=idxType, stridable=true);
  for i in 1..rank {
    t(i) = this.dim(i) by str(i);
  }
  alias.setIndices(t);
  return alias;
}

//
// output domain
//
def BlockDom.writeThis(x:Writer) {
  x.write(whole);
}

//
// how to allocate a new array over this domain
//
def BlockDom.buildArray(type eltType) {
  var arr = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=this);
  arr.setup();
  return arr;
}

def BlockDom.numIndices return whole.numIndices;
def BlockDom.low return whole.low;
def BlockDom.high return whole.high;

//
// INTERFACE NOTES: Could we make setIndices() for an arithmetic
// domain take a domain rather than something else?
//
def BlockDom.setIndices(x: domain) {
  if x.rank != rank then
    compilerError("rank mismatch in domain assignment");
  if x._value.idxType != idxType then
    compilerError("index type mismatch in domain assignment");
  whole = x;
  setup();
}

def BlockDom.setIndices(x) {
  if x.size != rank then
    compilerError("rank mismatch in domain assignment");
  if x(1).eltType != idxType then
    compilerError("index type mismatch in domain assignment");
  //
  // TODO: This seems weird:
  //
  whole.setIndices(x);
  setup();
}

def BlockDom.getIndices() {
  return whole.getIndices();
}

// remove all instances of getDist
def BlockDom.getDist(): Block(idxType) {
  return dist;
}

def BlockDom.slice(param stridable: bool, ranges) {
  var d = new BlockDom(rank=rank, idxType=idxType, dist=dist, stridable=stridable||this.stridable);
  d.setIndices(whole((...ranges)).getIndices());
  return d;
}

// dsiLocalSlice
def BlockDom.localSlice(param stridable: bool, ranges) {
  return whole((...ranges));
}

def BlockDom.setup() {
  if locDoms(dist.targetLocDom.low) == nil {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx) = new LocBlockDom(rank, idxType, stridable, this,
                                             dist.getChunk(whole, localeIdx));
    }
  } else {
    coforall localeIdx in dist.targetLocDom do {
      on dist.targetLocs(localeIdx) do
        locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    }
  }
  if debugBlockDist then
    for loc in dist.targetLocDom do writeln(loc, " owns ", locDoms(loc));
}

// dsi
def BlockDom.supportsPrivatization() param return true;

// dsi
def BlockDom.getPrivatizeData() return (dist.pid, whole.dims());

// dsi
def BlockDom.privatize(privatizeData) {
  var distpid = privatizeData(1);
  var thisdist = dist;
  var privdist = __primitive("chpl_getPrivatizedClass", thisdist, distpid);
  var c = new BlockDom(rank=rank, idxType=idxType, stridable=stridable, dist=privdist);
  for i in c.dist.targetLocDom do
    c.locDoms(i) = locDoms(i);
  c.whole = [(...privatizeData(2))];
  return c;
}

// dsi
def BlockDom.getReprivatizeData() return whole.dims();

// dsi
def BlockDom.reprivatize(other, reprivatizeData) {
  for i in dist.targetLocDom do
    locDoms(i) = other.locDoms(i);
  whole = [(...reprivatizeData)];
}

def BlockDom.member(i) {
  return whole.member(i);
}

def BlockDom.order(i) {
  return whole.order(i);
}

def BlockDom.position(i) {
  return whole.position(i);
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Domain Class
//
class LocBlockDom {
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // UP LINK: a reference to the parent global domain class
  //
  const wholeDom: BlockDom(rank, idxType, stridable);

  //
  // a local domain describing the indices owned by this locale
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(rank, idxType, stridable);
}

//
// output local domain piece
//
def LocBlockDom.writeThis(x:Writer) {
  x.write(myBlock);
}

//
// Added as a performance stopgap to avoid returning a domain
//
def LocBlockDom.member(i) return myBlock.member(i);

////////////////////////////////////////////////////////////////////////////////
// Block Array Class
//
class BlockArr: BaseArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: the global domain descriptor for this array
  //
  var dom: BlockDom(rank, idxType, stridable);

  //
  // DOWN LINK: an array of local array classes
  //
  var locArr: [dom.dist.targetLocDom] LocBlockArr(eltType, rank, idxType, stridable);

  //
  // optimized reference to a local LocBlockArr instance (or nil)
  //
  var myLocArr: LocBlockArr(eltType, rank, idxType, stridable);

  var pid: int = -1; // privatized object id
}

def BlockArr.getBaseDom() return dom;

def BlockArr.setup() {
  var thisid = this.locale.uid;
  coforall localeIdx in dom.dist.targetLocDom {
    on dom.dist.targetLocs(localeIdx) {
      const locDom = dom.getLocDom(localeIdx);
      locArr(localeIdx) = new LocBlockArr(eltType, rank, idxType, stridable, locDom);
      if thisid == here.uid then
        myLocArr = locArr(localeIdx);
    }
  }
}

// dsi
def BlockArr.supportsPrivatization() param return true;

// dsi
def BlockArr.getPrivatizeData() return dom.pid;

// dsi
def BlockArr.privatize(privatizeData) {
  var dompid = privatizeData;
  var thisdom = dom;
  var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
  var c = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=stridable, dom=privdom);
  for localeIdx in c.dom.dist.targetLocDom {
    c.locArr(localeIdx) = locArr(localeIdx);
    if c.locArr(localeIdx).locale.uid == here.uid then
      c.myLocArr = c.locArr(localeIdx);
  }
  return c;
}

//
// the global accessor for the array
//
// TODO: Do we need a global bounds check here or in ind2locind?
//
def BlockArr.this(i: idxType) var where rank == 1 {
  if myLocArr then local {
    if myLocArr.locDom.member(i) then
      return myLocArr.this(i);
  }
  return locArr(dom.dist.ind2locInd(i))(i);
}

def BlockArr.this(i: rank*idxType) var {
  if rank == 1 {
    return this(i(1));
  } else {
    if myLocArr then local {
      if myLocArr.locDom.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }
}

def BlockArr.these() var {
  for i in dom do
    yield this(i);
}

//
// TODO: Rewrite this to reuse more of the global domain iterator
// logic?  (e.g., can we forward the forall to the global domain
// somehow?
//
def BlockArr.these(param tag: iterator) where tag == iterator.leader {
  const precomputedNumTasks = dom.dist.tasksPerLocale;
  const precomputedWholeLow = dom.whole.low;
  coforall locDom in dom.locDoms do on locDom {
    var tmpBlock = locDom.myBlock - precomputedWholeLow;
    const numTasks = precomputedNumTasks;
    var locBlock: rank*range(idxType);
    for param i in 1..tmpBlock.rank {
      locBlock(i) = (tmpBlock.dim(i).low/tmpBlock.dim(i).stride:idxType)..#(tmpBlock.dim(i).length);
    }

    if (numTasks == 1) {
      yield locBlock;
    } else {
      coforall taskid in 0..#numTasks {
        var tuple: rank*range(idxType) = locBlock;
        const (lo,hi) = _computeBlock(locBlock(1).low, locBlock(1).length,
                                      locBlock(1).low, locBlock(1).high,
                                      numTasks, taskid);

        tuple(1) = lo..hi;
        yield tuple;

      }
    }
  }
}

// dsi
def BlockArr.supportsAlignedFollower() param return true;

def BlockArr.these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
  var followThis: rank*range(eltType=idxType, stridable=stridable);
  var lowIdx: rank*idxType;

  for param i in 1..rank {
    var stride = dom.whole.dim(i).stride;
    var low = follower(i).low * stride;
    var high = follower(i).high * stride;
    followThis(i) = (low..high by stride) + dom.whole.dim(i).low;
    lowIdx(i) = followThis(i).low;
  }
  const followThisDom = [(...followThis)];

  //
  // TODO: The following is a buggy hack that will only work when we're
  // distributing across the entire Locales array.  I still think the
  // locArr/locDoms arrays should be associative over locale values.
  //
  var arrSection = locArr(dom.dist.ind2locInd(lowIdx));
  if aligned {
    //
    // if arrSection is not local and we're aligned, it means that
    // followThisDom is empty; make arrSection local so that we can
    // use the local block below
    //
    if arrSection.locale.uid != here.uid then
      arrSection = myLocArr;
    local {
      for e in arrSection.myElems(followThisDom) do
        yield e;
    }
  } else {
    //
    // we don't own all the elements we're following
    //
    def accessHelper(i) var {
      local {
        if myLocArr.locDom.member(i) then
          return myLocArr.this(i);
      }
      return this(i);
    }
    for i in followThisDom {
      yield accessHelper(i);
    }
  }
}

//
// output array
//
def BlockArr.writeThis(f: Writer) {
  if dom.numIndices == 0 then return;
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dim(dim)._low;
  label next while true {
    f.write(this(i));
    if i(rank) <= (dom.dim(rank)._high - dom.dim(rank)._stride:idxType) {
      f.write(" ");
      i(rank) += dom.dim(rank)._stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dim(dim)._high - dom.dim(dim)._stride:idxType) {
          i(dim) += dom.dim(dim)._stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dim(dim2)._low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

def BlockArr.checkSlice(ranges) {
  for param i in 1..rank do
    if !dom.dim(i).boundsCheck(ranges(i)) then {
      writeln(dom.dim(i), " ", ranges(i), " ", dom.dim(i).boundsCheck(ranges(i)));
      halt("array slice out of bounds in dimension ", i, ": ", ranges(i));
    }
}

def BlockArr.slice(d: BlockDom) {
  var alias = new BlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, dom=d, pid=pid);
  for i in dom.dist.targetLocDom {
    on dom.dist.targetLocs(i) {
      alias.locArr[i] = new LocBlockArr(eltType=eltType, rank=rank, idxType=idxType, stridable=d.stridable, locDom=d.locDoms[i]);
      alias.locArr[i].myElems => locArr[i].myElems[alias.locArr[i].locDom.myBlock];
    }
  }

  return alias;
}

def BlockArr.localSlice(ranges) {
  var low: rank*idxType;
  for param i in 1..rank {
    low(i) = ranges(i).low;
  }
  var A => locArr(dom.dist.ind2locInd(low)).myElems((...ranges));
  return A;
}

////////////////////////////////////////////////////////////////////////////////
// Block Local Array Class
//
class LocBlockArr {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;

  //
  // LEFT LINK: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlockDom(rank, idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;
}

//
// the accessor for the local array -- assumes the index is local
//
def LocBlockArr.this(i) var {
  return myElems(i);
}

//
// output local array piece
//
def LocBlockArr.writeThis(x: Writer) {
  // note on this fails; see writeThisUsingOn.chpl
  x.write(myElems);
}

//
// helper function for blocking index ranges
//
// 12/8/09 note: move this for sharing with forall over default
//               arrays, etc.
//
def _computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def intCeilXDivByY(x, y) return ((x + (y-1)) / y);

  const blo =
    if blocknum == 0 then waylo
    else lo + intCeilXDivByY(numelems * blocknum:lo.type, numblocks:lo.type);
  const bhi =
    if blocknum == numblocks - 1 then wayhi
    else lo + intCeilXDivByY(numelems * (blocknum+1):lo.type, numblocks:lo.type) - 1;

  return (blo, bhi);
}

//
// naive routine for dividing numLocales into rank factors
//
// move somewhere and share.
def _factor(param rank: int, value) {
  var factors: rank*int;
  for param i in 1..rank do
    factors(i) = 1;
  if value >= 1 {
    var iv = value;
    var factor = 1;
    while iv > 1 {
      for i in 2..iv {
        if iv % i == 0 {
          var j = 1;
          for i in 2..rank {
            if factors(i) < factors(j) then
              j = i;
          }
          factors(j) *= i;
          iv = iv / i;
          break;
        }
      }
    }
  }
  for i in 1..rank do
    for j in i+1..rank do
      if factors(i) < factors(j) then
        factors(i) <=> factors(j);
  return factors;
}