Change arithmetic to rectangular in a couple places.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@18605 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
 * This example is written to demonstrate rectangular array features in
 * Chapel in a tutorial-like fashion.
 *
 */

//
// Arrays in Chapel are specified using a square-bracketed expression
// that specifies the array's index set, followed by the array's
// element type.  Rectangular arrays are those whose indices are
// integers or tuples of integers, supporting standard
// multidimensional, rectilinear array support.
//

config const n = 5;

//
// Declare an array of five 64-bit real values:
//
var A: [1..n] real;

//
// Like other variable types in Chapel, arrays are initialized so that
// each element stores its default value.  So our array of real values
// above will default to an array of the value 0.0.
//

writeln("Initially, A is: ", A);

//
// Arrays can be accessed using scalar index values of the appropriate
// type, using either parenthesis or square brackets:
//

A[1] = 1.1;
A(2) = 2.2;

writeln("After assigning two elements, A is: ", A);

//
// Arrays can also be accessed using ranges to refer to subsets of
// array elements, or sub-arrays, using a technique called slicing:
//

A[2..4] = 3.3;

writeln("After assigning its interior values, A is: ", A);
writeln();

//
// Chapel uses square brackets to represent tensor products in Chapel
// and parenthesis to represent zipper products.  By convention we
// square brackets for array slicing even when the tensor product is
// not required.  For example, the expression above could have just
// as correctly been written A(2..4):
//

writeln("A(2..4) is: ", A(2..4), "\n");

//
// Arrays can be multidimensional as well:
//

var B: [1..n, 1..n] real;

forall (i,j) in [1..n, 1..n] do
  B(i,j) = i + j/10.0;

writeln("Initially, B is:\n", B, "\n");

//
// An array's elements can be iterated over in a for or forall loop,
// which causes the index variable to store references to an array's
// elements:
//

forall b in B do
  b += 1;

writeln("After incrementing B's elements, B is:\n", B, "\n");

//
// An array's index set is referred to as a domain -- a first-class
// language concept that stores the set of indices used to access the
// array.  The arrays above are declared with the anonymous domains
// [1..n] and [1..n, 1..n].  An array's domain can be accessed using
// the .domain method:
//

forall (i,j) in B.domain do
  B(i,j) -= 1;

writeln("After decrementing B's elements, B is:\n", B, "\n");

//
// Domains can also be queried when arrays are passed to functions
// in order to refer to the domain via a new identifier within the
// context of that function:
//

printArr(B);

proc printArr(X: [?D] real) {
  writeln("within printArr, D is: ", D, "\n");
  forall (i,j) in D do
    X(i,j) = -X(i,j);
  writeln("after negating X within printArr, X is:\n", X, "\n");
}

//
// Arrays are passed to functions by reference by default, so the
// modifications to X in function printArr are reflected on B as
// well:
//

writeln("After calling printArr, B is:\n", B, "\n");

//
// Domains can also be declared and named.  This has the advantages
// of allowing them to be reused for multiple arrays, for associating
// names with different index spaces, and for amortizing the overhead
// of storing distributed arrays across multiple similar array
// variables.
//
// The following domain declaration defines a 2D arithemetic domain
// called ProbSpace that's the same size and shape as B was above.
//

var ProbSpace: domain(2) = [1..n, 1..n];

//
// We then use that domain to declare some arrays...
//

var C, D, E: [ProbSpace] bool;

// ...and to iterate over their index spaces...

for (i,j) in ProbSpace do
  C(i,j) = (i+j) % 3 == 0;

writeln("After initializing C, its value is:\n", C, "\n");

//
// When indexing over a multidimensional domain, the indices
// can be captured into a single tuple variable rather than
// destructuring the tuple into its integer components.
// Similarly, multidimensional array accesses can be expressed using
// tuple indices rather than multiple integer arguments.  In the
// following example, the index variable ij stores a 2-tuple of
// integers (2*int in Chapel).  This is a really inefficient way
// to assign the diagonal values "true" -- note the use of tuple
// indexing to tease the individual components out of the 2-tuple.
//

for ij in ProbSpace do
  D(ij) = ij(1) == ij(2);

writeln("After initializing D, its value is:\n", D, "\n");

//
// Array expressions of similar size and shape support whole-array
// assignment.
//

E = C;

writeln("After assigning C to E, E's value is:\n", E, "\n");

//
// Whole array assignment also allows scalar values to be promoted
// and assigned to every element of an array
//

B = 0.0;

writeln("After being reset, B is:\n", B, "\n");

//
// An array need not be indexed using the domain used to declare it,
// though doing so presents the compiler with opportunities to
// optimize bounds checks away.  In the following loop, there is
// no known relation between B and ProbSpace, so bounds checks are
// harder to prove away (requires symbolic analysis of the definitions
// of the two domains and the invariance of their bounds)
//

for (i,j) in ProbSpace do
  B(i,j) = i + j/10.0;

writeln("B has been re-initialized to:\n", B, "\n");

//
// Whole-array assignment can also be used for arrays or sub-arrays
// whose index spaces differ:
//

var F, G: [ProbSpace] real;

F[2..n-1, 2..n-1] = B[1..n-2, 3..n];

writeln("After assigning a slice of B to F, F's value is:\n", F, "\n");

//
// Arrays can also be sliced using unbounded ranges in which either
// the low or high bounds are omitted.  In this case, they will be
// inherited from the array's bounds.
//

G[2.., ..] = B[..n-1, ..];

writeln("After assigning a slice of B to G, G's value is:\n", G, "\n");

//
// Array slicing also supports rank-change semantics when sliced using
// a scalar value rather than a range.  In the following assignment,
// recall that A was our initial 1-dimensional array.
//

A = B[n/2, ..];

writeln("After being assigned a slice of B, A is:\n", A, "\n");

//
// As mentioned earlier, parenthesized promoted arguments in Chapel
// represent zipper products.  This means that B(1..n, 1..n) ought
// to represent the main diagonal of the B array.  However, these
// semantics are not yet implemented, so today it incorrectly yields
// the tensor product of the ranges -- in this case B in its entirety.
//

writeln("Incorrectly, B(1..n, 1..n) is:\n", B(1..n, 1..n), "\n");

//
// Domains can also be sliced.  However, rather than having indexing
// semantics, as with arrays, domain indexing results in intersection
// semantics.  Put another way, domains can be indexed out of bounds:
//

writeln("ProbSpace[0..n+1, 3..] is: ", ProbSpace[0..n+1, 3..], "\n");

// Ranges also support slicing in this way, though we don't
// demonstrate that here.

//
// ...while arrays cannot.  The following would result in an out-of-bounds
// error:
//

// writeln("B[0..n+1, 3..] is:\n", B[0..n+1, 3..], "\n");

//
// Domain variables and expressions can also be used to specify an
// array slice rather than using lists of ranges.  For example:
//

const ProbSpaceSlice = ProbSpace[0..n+1, 3..];

writeln("B[ProbSpaceSlice] is:\n", B[ProbSpaceSlice], "\n");

//
// Forall loops over domains and arrays can be written using the
// syntax "[<ind> in <Dom>] ..." which is shorthand for "forall ind
// in Dom do ..."
//

const offset = (1,1); // a 2-tuple offset

[ij in ProbSpace[2..n-1, 2..n-1]] F(ij) = B(ij + offset);

writeln("After assigning F a shifted slice of B, it is:\n", F, "\n");

[b in B] b = -b;

writeln("After negating B, it is:\n", B, "\n");

//
// Note that this shorthand resembles the array type definition in
// a variable declaration.  Due to this similarity, and for
// convenience, we allow the indices in an array type specifier to be
// named in order to support in-place initialization of the array.
// For example:
//

const Z: [ProbSpace] complex = [(i,j) in ProbSpace] i + j*1.0i;

writeln("Z is initialized to be:\n", Z, "\n");

//
// This is simply shorthand for:
//   const Z: [ProbSpace] complex = [i,j in ProbSpace] i + j*1.0i;
// or more generally:
//   [const|var] <ident>: [<ind> in <dom>] <elemtype> = <expr>;
// is equivalent to:
//   [const|var] <ident>: [<dom>] <elemtype> = [<ind> in <dom>] <expr>;
//

//
// Another advantage to declaring named domain variables is that their
// index sets can be reassigned.  This results in a logical
// re-allocation of the array variable in question, preserving array
// values for indices that existed in both the old and new domain
// values:
//

var VarDom = [1..n];

var VarArr: [VarDom] real = [i in VarDom] i;

writeln("Initially, VarArr = ", VarArr, "\n");

//
// Now, if we reassign VarDom, VarArr will be reallocated with the
// old values preserved and the new values initialized to the element
// type's default value.
//

VarDom = [1..2*n];

writeln("After doubling VarDom, VarArr = ", VarArr, "\n");

//
// As mentioned before, this reallocation preserves values according
// to index, so if we extend the lower bound of the domain, the
// non-zero values will still logically be associated with indices
// 1..n:
//

VarDom = [-n+1..2*n];

writeln("After lowering VarDom's lower bound, VarArr = ", VarArr, "\n");

//
// If the domain shrinks, values will be thrown away
//

VarDom = [2..n-1];

writeln("After shrinking VarDom, VarArr = ", VarArr, "\n");

//
// One trick to reallocate an array without preserving any values is
// to assign its domain a degenerate domain and then assign it the
// new value:
//

VarDom = [1..0];

writeln("VarArr should now be empty: ", VarArr, "\n");

VarDom = [1..n];

writeln("VarArr should now be reset: ", VarArr, "\n");

//
// Note that querying an array's domain via the .domain method or
// the function argument query syntax does not result in a domain
// expression that can be reassigned.  In particular, we cannot do:
//
//   VarArr.domain = [1..2*n];
//
// nor:
//
//   proc foo(X: [?D]) {  D = [1..2*n];  }
//
// Only a domain variable or a domain function argument can be
// reassigned to reallocate arrays.  This is to avoid confusion
// since assigning one domain variable can cause a number of
// arrays to be reallocated.  It also implies that arrays declared
// using an anonymous domain cannot be reallocated.  So for our
// original array declarations A and B, we have no way of reallocating
// them.  Arrays with constant domains provide the compiler with
// optimization benefits, so this supports a common case efficiently.
//
// A.domain = [1..2*n];
//

//
// As some of our examples have shown, arrays in Chapel can have
// arbitrary element types -- numeric values, classes, or records.
// Arrays can also support array elements, and initial support for
// this is implemented in our compiler.  For example:
//

var Y: [ProbSpace] [1..3] real;

forall ((i,j), k) in [ProbSpace, 1..3] do
  Y(i,j)(k) = i*10 + j + k/10.0;

writeln("Y is:\n", Y);

//
// Our current implementation does not yet support arrays of arrays
// where the inner array size is a function of the outer -- they must
// all be of uniform size.  In particular, it is our intention to
// support things like:
//
//   const Triangle: [1..row] real = [row in 1..n] [col in 1..row] row + col/10.0;
//
// and:
//
//   var HierArr: [lvl in 1..n] [1..2**lvl, 1..2**lvl] real;
//
// If such capabilities would be of use to you, please let us know and
// we will adjust their priority accordingly.
//

/*
 * This example is written to demonstrate arithmetic array features in
 * Chapel in a tutorial-like fashion.
 *
 */

//
// Arrays in Chapel are specified using a square-bracketed expression
// that specifies the array's index set, followed by the array's
// element type.  Arithmetic arrays are those whose indices are
// integers or tuples of integers, supporting standard
// multidimensional, rectilinear array support.
//

config const n = 5;

//
// Declare an array of five 64-bit real values:
//
var A: [1..n] real;

//
// Like other variable types in Chapel, arrays are initialized so that
// each element stores its default value.  So our array of real values
// above will default to an array of the value 0.0.
//

writeln("Initially, A is: ", A);

//
// Arrays can be accessed using scalar index values of the appropriate
// type, using either parenthesis or square brackets:
//

A[1] = 1.1;
A(2) = 2.2;

writeln("After assigning two elements, A is: ", A);

//
// Arrays can also be accessed using ranges to refer to subsets of
// array elements, or sub-arrays, using a technique called slicing:
//

A[2..4] = 3.3;

writeln("After assigning its interior values, A is: ", A);
writeln();

//
// Chapel uses square brackets to represent tensor products in Chapel
// and parenthesis to represent zipper products.  By convention we
// square brackets for array slicing even when the tensor product is
// not required.  For example, the expression above could have just
// as correctly been written A(2..4):
//

writeln("A(2..4) is: ", A(2..4), "\n");

//
// Arrays can be multidimensional as well:
//

var B: [1..n, 1..n] real;

forall (i,j) in [1..n, 1..n] do
  B(i,j) = i + j/10.0;

writeln("Initially, B is:\n", B, "\n");

//
// An array's elements can be iterated over in a for or forall loop,
// which causes the index variable to store references to an array's
// elements:
//

forall b in B do
  b += 1;

writeln("After incrementing B's elements, B is:\n", B, "\n");

//
// An array's index set is referred to as a domain -- a first-class
// language concept that stores the set of indices used to access the
// array.  The arrays above are declared with the anonymous domains
// [1..n] and [1..n, 1..n].  An array's domain can be accessed using
// the .domain method:
//

forall (i,j) in B.domain do
  B(i,j) -= 1;

writeln("After decrementing B's elements, B is:\n", B, "\n");

//
// Domains can also be queried when arrays are passed to functions
// in order to refer to the domain via a new identifier within the
// context of that function:
//

printArr(B);

proc printArr(X: [?D] real) {
  writeln("within printArr, D is: ", D, "\n");
  forall (i,j) in D do
    X(i,j) = -X(i,j);
  writeln("after negating X within printArr, X is:\n", X, "\n");
}

//
// Arrays are passed to functions by reference by default, so the
// modifications to X in function printArr are reflected on B as
// well:
//

writeln("After calling printArr, B is:\n", B, "\n");

//
// Domains can also be declared and named.  This has the advantages
// of allowing them to be reused for multiple arrays, for associating
// names with different index spaces, and for amortizing the overhead
// of storing distributed arrays across multiple similar array
// variables.
//
// The following domain declaration defines a 2D arithemetic domain
// called ProbSpace that's the same size and shape as B was above.
//

var ProbSpace: domain(2) = [1..n, 1..n];

//
// We then use that domain to declare some arrays...
//

var C, D, E: [ProbSpace] bool;

// ...and to iterate over their index spaces...

for (i,j) in ProbSpace do
  C(i,j) = (i+j) % 3 == 0;

writeln("After initializing C, its value is:\n", C, "\n");

//
// When indexing over a multidimensional domain, the indices
// can be captured into a single tuple variable rather than
// destructuring the tuple into its integer components.
// Similarly, multidimensional array accesses can be expressed using
// tuple indices rather than multiple integer arguments.  In the
// following example, the index variable ij stores a 2-tuple of
// integers (2*int in Chapel).  This is a really inefficient way
// to assign the diagonal values "true" -- note the use of tuple
// indexing to tease the individual components out of the 2-tuple.
//

for ij in ProbSpace do
  D(ij) = ij(1) == ij(2);

writeln("After initializing D, its value is:\n", D, "\n");

//
// Array expressions of similar size and shape support whole-array
// assignment.
//

E = C;

writeln("After assigning C to E, E's value is:\n", E, "\n");

//
// Whole array assignment also allows scalar values to be promoted
// and assigned to every element of an array
//

B = 0.0;

writeln("After being reset, B is:\n", B, "\n");

//
// An array need not be indexed using the domain used to declare it,
// though doing so presents the compiler with opportunities to
// optimize bounds checks away.  In the following loop, there is
// no known relation between B and ProbSpace, so bounds checks are
// harder to prove away (requires symbolic analysis of the definitions
// of the two domains and the invariance of their bounds)
//

for (i,j) in ProbSpace do
  B(i,j) = i + j/10.0;

writeln("B has been re-initialized to:\n", B, "\n");

//
// Whole-array assignment can also be used for arrays or sub-arrays
// whose index spaces differ:
//

var F, G: [ProbSpace] real;

F[2..n-1, 2..n-1] = B[1..n-2, 3..n];

writeln("After assigning a slice of B to F, F's value is:\n", F, "\n");

//
// Arrays can also be sliced using unbounded ranges in which either
// the low or high bounds are omitted.  In this case, they will be
// inherited from the array's bounds.
//

G[2.., ..] = B[..n-1, ..];

writeln("After assigning a slice of B to G, G's value is:\n", G, "\n");

//
// Array slicing also supports rank-change semantics when sliced using
// a scalar value rather than a range.  In the following assignment,
// recall that A was our initial 1-dimensional array.
//

A = B[n/2, ..];

writeln("After being assigned a slice of B, A is:\n", A, "\n");

//
// As mentioned earlier, parenthesized promoted arguments in Chapel
// represent zipper products.  This means that B(1..n, 1..n) ought
// to represent the main diagonal of the B array.  However, these
// semantics are not yet implemented, so today it incorrectly yields
// the tensor product of the ranges -- in this case B in its entirety.
//

writeln("Incorrectly, B(1..n, 1..n) is:\n", B(1..n, 1..n), "\n");

//
// Domains can also be sliced.  However, rather than having indexing
// semantics, as with arrays, domain indexing results in intersection
// semantics.  Put another way, domains can be indexed out of bounds:
//

writeln("ProbSpace[0..n+1, 3..] is: ", ProbSpace[0..n+1, 3..], "\n");

// Ranges also support slicing in this way, though we don't
// demonstrate that here.

//
// ...while arrays cannot.  The following would result in an out-of-bounds
// error:
//

// writeln("B[0..n+1, 3..] is:\n", B[0..n+1, 3..], "\n");

//
// Domain variables and expressions can also be used to specify an
// array slice rather than using lists of ranges.  For example:
//

const ProbSpaceSlice = ProbSpace[0..n+1, 3..];

writeln("B[ProbSpaceSlice] is:\n", B[ProbSpaceSlice], "\n");

//
// Forall loops over domains and arrays can be written using the
// syntax "[<ind> in <Dom>] ..." which is shorthand for "forall ind
// in Dom do ..."
//

const offset = (1,1); // a 2-tuple offset

[ij in ProbSpace[2..n-1, 2..n-1]] F(ij) = B(ij + offset);

writeln("After assigning F a shifted slice of B, it is:\n", F, "\n");

[b in B] b = -b;

writeln("After negating B, it is:\n", B, "\n");

//
// Note that this shorthand resembles the array type definition in
// a variable declaration.  Due to this similarity, and for
// convenience, we allow the indices in an array type specifier to be
// named in order to support in-place initialization of the array.
// For example:
//

const Z: [ProbSpace] complex = [(i,j) in ProbSpace] i + j*1.0i;

writeln("Z is initialized to be:\n", Z, "\n");

//
// This is simply shorthand for:
//   const Z: [ProbSpace] complex = [i,j in ProbSpace] i + j*1.0i;
// or more generally:
//   [const|var] <ident>: [<ind> in <dom>] <elemtype> = <expr>;
// is equivalent to:
//   [const|var] <ident>: [<dom>] <elemtype> = [<ind> in <dom>] <expr>;
//

//
// Another advantage to declaring named domain variables is that their
// index sets can be reassigned.  This results in a logical
// re-allocation of the array variable in question, preserving array
// values for indices that existed in both the old and new domain
// values:
//

var VarDom = [1..n];

var VarArr: [VarDom] real = [i in VarDom] i;

writeln("Initially, VarArr = ", VarArr, "\n");

//
// Now, if we reassign VarDom, VarArr will be reallocated with the
// old values preserved and the new values initialized to the element
// type's default value.
//

VarDom = [1..2*n];

writeln("After doubling VarDom, VarArr = ", VarArr, "\n");

//
// As mentioned before, this reallocation preserves values according
// to index, so if we extend the lower bound of the domain, the
// non-zero values will still logically be associated with indices
// 1..n:
//

VarDom = [-n+1..2*n];

writeln("After lowering VarDom's lower bound, VarArr = ", VarArr, "\n");

//
// If the domain shrinks, values will be thrown away
//

VarDom = [2..n-1];

writeln("After shrinking VarDom, VarArr = ", VarArr, "\n");

//
// One trick to reallocate an array without preserving any values is
// to assign its domain a degenerate domain and then assign it the
// new value:
//

VarDom = [1..0];

writeln("VarArr should now be empty: ", VarArr, "\n");

VarDom = [1..n];

writeln("VarArr should now be reset: ", VarArr, "\n");

//
// Note that querying an array's domain via the .domain method or
// the function argument query syntax does not result in a domain
// expression that can be reassigned.  In particular, we cannot do:
//
//   VarArr.domain = [1..2*n];
//
// nor:
//
//   proc foo(X: [?D]) {  D = [1..2*n];  }
//
// Only a domain variable or a domain function argument can be
// reassigned to reallocate arrays.  This is to avoid confusion
// since assigning one domain variable can cause a number of
// arrays to be reallocated.  It also implies that arrays declared
// using an anonymous domain cannot be reallocated.  So for our
// original array declarations A and B, we have no way of reallocating
// them.  Arrays with constant domains provide the compiler with
// optimization benefits, so this supports a common case efficiently.
//
// A.domain = [1..2*n];
//

//
// As some of our examples have shown, arrays in Chapel can have
// arbitrary element types -- numeric values, classes, or records.
// Arrays can also support array elements, and initial support for
// this is implemented in our compiler.  For example:
//

var Y: [ProbSpace] [1..3] real;

forall ((i,j), k) in [ProbSpace, 1..3] do
  Y(i,j)(k) = i*10 + j + k/10.0;

writeln("Y is:\n", Y);

//
// Our current implementation does not yet support arrays of arrays
// where the inner array size is a function of the outer -- they must
// all be of uniform size.  In particular, it is our intention to
// support things like:
//
//   const Triangle: [1..row] real = [row in 1..n] [col in 1..row] row + col/10.0;
//
// and:
//
//   var HierArr: [lvl in 1..n] [1..2**lvl, 1..2**lvl] real;
//
// If such capabilities would be of use to you, please let us know and
// we will adjust their priority accordingly.
//


Improve my distributed array class

Improve my distributed array class

In the my previous commit message, I stated that most array references in this
test should be local. Unfortunately, I didn't notice that although the array
references were local, it took at least two remote references to find the
location to locally reference.

I added one copy of the array data structure per locale, with the references
used to calculate the correct array location on that locale. This resulted in
a huge speedup, though it's still unbearably slow.

Also, changed an array copy used to swap arrays in between iterations into
a use of the swap operator.

At the tiny array size of 50, with GASNET_SPAWNFN=l, and 2 locales, the
previously committed version took 85 seconds. Now, it takes 2.4 seconds.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13547 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config const size = 10;
config const epsilon = 0.0001;

class DistribArrayNode {
  type arrType;
  var size: int;
  var arr: [0..size-1] arrType;
}

class DistribArray {
  type arrType;
  const arrSize: int;
  const localSize:int = arrSize / numLocales;
  var internalArr: [0..numLocales-1] DistribArrayNode(arrType);
  var isLocalize = false;

  def initialize() {
    if isLocalize then return;

    for i in 0..numLocales-1 {
      on Locales(i) {
        if (i != numLocales-1) {
          internalArr(i) = new DistribArrayNode(arrType, localSize);
        } else {
          const numElts = arrSize - localSize*(numLocales-1);
          internalArr(i) = new DistribArrayNode(arrType, numElts);
        }
      }
    }
  }

  def localize() {
    var localArrays: [0..numLocales-1] DistribArray(arrType);
    for loc in [0..numLocales-1] {
      on Locales(loc) {
        var x = new DistribArray(arrType, arrSize, localSize, isLocalize=true);
        [i in 0..numLocales-1] x.internalArr(i) = internalArr(i);
        localArrays(loc) = x;
      }
    }
    return localArrays;
  }

  def element(indexNum: int) var {
    const localeNum = min(indexNum / localSize, numLocales-1);
    const localIndex = indexNum - (localeNum * localSize);
    return internalArr(localeNum).arr(localIndex);
  }

  def getLocales() {
    // One could imagine not distributing it across all locales
    [i in 0..numLocales-1] yield i;
  }

  def getLocalIndices() {
    // Get the indices that live on this locale
    var myID = localeID();
    var size = if myID == numLocales-1
                 then arrSize-localSize*(numLocales-1)
                 else localSize;

    for i in 0..size-1 {
      yield myID*localSize + i;
    }
  }

  def writeThis(w: Writer) {
    on Locales(0) {
      for i in 0..arrSize-1 {
        w.writeln(element(i));
      }
    }
  }

  def copy(b: DistribArray(arrType)) {
    if (b.arrSize != arrSize) then
      halt("Bad sizes in DistribArray.copy");

    forall loc in getLocales() {
      on Locales(loc) {
        // hope that most indices match locale between the arrays
        // With this DistribArray class, they are.
        forall i in getLocalIndices() {
          this.element(i) = b.element(i);
        }
      }
    }
  }
}

def main {
  var delta: real;
  var localAs => (new DistribArray(real, size)).localize();
  var localBs => (new DistribArray(real, size)).localize();
  localAs(0).element(0) = 1.0;

  do {
    delta = 0.0;
    coforall loc in localAs(0).getLocales() {
      on Locales(loc) {
        var localDelta: real;
        var A = localAs(loc);
        var B = localBs(loc);
        for i in A.getLocalIndices() {
          if (i == 0) {
            // Pin the low end at 1.0
            B.element(i) = 1.0;
          } else if (i == size-1) {
            B.element(i) = A.element(i-1)/2.0;
          } else {
            B.element(i) = (A.element(i-1) + A.element(i+1))/2.0;
          }
          localDelta = max(localDelta, abs(B.element(i) - A.element(i)));
        }
        delta = max(delta, localDelta);
      }
    }
    localAs <=> localBs;
  } while (delta > epsilon);

  write(localAs(0));
}

config const size = 10;
config const epsilon = 0.0001;

class DistribArrayNode {
  type arrType;
  var size: int;
  var arr: [0..size-1] arrType;
}

class DistribArray {
  type arrType;
  const arrSize: int;
  const localSize:int = arrSize / numLocales;
  var internalArr: [0..numLocales-1] DistribArrayNode(arrType);

  def initialize() {
    for i in 0..numLocales-1 {
      on Locales(i) {
        if (i != numLocales-1) {
          internalArr(i) = new DistribArrayNode(arrType, localSize);
        } else {
          const numElts = arrSize - localSize*(numLocales-1);
          internalArr(i) = new DistribArrayNode(arrType, numElts);
        }
      }
    }
  }

  def element(indexNum: int) var {
    const localeNum = min(indexNum / localSize, numLocales-1);
    const localIndex = indexNum - (localeNum * localSize);
    return internalArr(localeNum).arr(localIndex);
  }

  def getLocales() {
    // One could imagine not distributing it across all locales
    [i in 0..numLocales-1] yield i;
  }

  def getLocalIndices() {
    // Get the indices that live on this locale
    var myID = localeID();
    var size = if myID == numLocales-1
                 then arrSize-localSize*(numLocales-1)
                 else localSize;

    for i in 0..size-1 {
      yield myID*localSize + i;
    }
  }

  def writeThis(w: Writer) {
    on Locales(0) {
      for i in 0..arrSize-1 {
        w.writeln(element(i));
      }
    }
  }

  def copy(b: DistribArray(arrType)) {
    if (b.arrSize != arrSize) then
      halt("Bad sizes in DistribArray.copy");

    forall loc in getLocales() {
      on Locales(loc) {
        // hope that most indices match locale between the arrays
        // With this DistribArray class, they are.
        forall i in getLocalIndices() {
          this.element(i) = b.element(i);
        }
      }
    }
  }
}

def main {
  var A = new DistribArray(real, size);
  var B = new DistribArray(real, size);
  var delta: real;

  A.element(0) = 1.0;

  do {
    delta = 0.0;
    coforall loc in A.getLocales() {
      on Locales(loc) {
        var localDelta: real;
        for i in A.getLocalIndices() {
          if (i == 0) {
            // Pin the low end at 1.0
            B.element(i) = 1.0;
          } else if (i == size-1) {
            B.element(i) = A.element(i-1)/2.0;
          } else {
            B.element(i) = (A.element(i-1) + A.element(i+1))/2.0;
          }
          localDelta = max(localDelta, abs(B.element(i) - A.element(i)));
        }
        delta = max(delta, localDelta);
      }
    }

    A.copy(B);
  } while (delta > epsilon);

  write(A);
}


Take Ben's version of intsort.mtml.chpl

Take Ben's version of intsort.mtml.chpl

I'd run dos2unix on this file which resulted in all sorts of
merge conflicts.  Will do dos2unix separately, merge, and then
re-implement my format() changes.

That said, git says something about converting CRLF so maybe
it's going to try and fix this up for me...?

<<<<<<< HEAD
/*
   NPB Integer Sort
   Chapel parallel version: multi-task, multi-locale
     based on bucket sort (same as NPB reference version)

   Originally written by Margaret Cahir (Cray, Inc.)

   NOTES on the Block version:
     Limitations:
       - works with 32-bit integers only
       - tested on 1, 2 and 4 locales, Classes S, A, B and C (not D)

   NOTES on the BlockCyclic version:
     Still need to
     - adjust to solve for Class D (means adjusting for int64)
     - can gen keys be made a little cleaner? (see note)
     - can a backwards index be made to work? see note in fullVerify
  	var mype: int = i / nkeys_per_pe;  //  can this be i >> log2nkeys_per_pe?
     - loop2: can we use scan in line below

   Modifications [sungeun@cray.com]: 12/2011

   - Combined Block and BlockCyclic versions

*/

// Problem size parameters
enum  classVals {S=0, W=1, A=2, B=3, C=4};
const Classes: domain(classVals);
const ClassNkeys:  [Classes] int = ( 16, 20, 23, 25, 27 ),
      ClassRanges: [Classes] int = ( 11, 16, 19, 21, 23 );

// Set of config constants that control size of problem and output options
config const probClass = classVals.S;	 // default to Sample class
config const maxIterations = 10;	 // can be dialed down for devel runs
config const DEBUG:        bool = false; // shows progress and summary info
config param printLoopTimings: bool = false; // prints timing info
config param printArrays:  bool = false; // prints out arrays (can be long)
config const printTime    = false;	 // turn off for test verification

// Standard modules
use Time, Random, BlockDist, BlockCycDist;
enum ISDistType { block, blockcyclic };
config param distType = ISDistType.block;

// Comparison numbers for partial verification (these are given)
var testD: domain(2) =  {0..4, 0..4};	// # of Classes, # of keys to verify
var testIndexArray: [testD] int = (	( 48427,17148,23627,62548,4431),
					( 357773,934767,875723,898999,404505),
					( 2112377,662041,5336171,3642833,4250760),
					( 41869,812306,5102857,18232239,26860214),
					( 44172927,72999161,74326391,129606274,21736814) );
const testRankArray: [testD]  int = (   ( 0,18,346,64917,65463),
					( 1249,11698,1039987,1043896,1048018),
					( 104,17523,123928,8288932,8388264),
					( 33422937,10244,59149,33135281,99),
					( 61147,882988,266290,133997595,133525895) );

var passedVerifications = 0;		// verification counter

// Set problem sizes and storage requirements
const log2range = ClassRanges(probClass);
const log2nkeys = ClassNkeys (probClass);
var range:    int = 1 << log2range;	// left shift for power of 2 values
var nkeys:    int = 1 << log2nkeys;
var npes:     int = numLocales*(if dataParTasksPerLocale==0
                                then here.maxTaskPar
                                  else dataParTasksPerLocale);  // npes is the total number of tasks
var nbuckets: int = if distType==ISDistType.block then npes else 4*npes;

var keybuff_pe: real = 3*nkeys/npes;
var keybuffsz:  int  = keybuff_pe:int;

// Domains for main arrays
const keySpace:    domain(1)= {0..#nkeys};

// mcahir: Alas, it seems a block cyclic dist array has to be 2D so ended up making countSpace a 2D domain
const countSpace = if distType==ISDistType.block then {0..#range}
                                                 else {0..#range, 0..0};
const bucketSpace: domain(2)= {0..#npes,0..#nbuckets};
const buffSpace:   domain(2)= {0..#nbuckets,0..#keybuffsz};

// ... block a 2D space into horizontal slabs
const MyLocaleView = {0..#numLocales, 1..1};
const MyLocales: [MyLocaleView] locale = reshape(Locales, MyLocaleView);
const bucketDom = bucketSpace dmapped Block(boundingBox=bucketSpace, targetLocales=MyLocales);
const buffDom = if distType==ISDistType.block
  then buffSpace dmapped Block(boundingBox=buffSpace, targetLocales=MyLocales)
  else buffSpace dmapped BlockCyclic (startIdx=buffSpace.low,
                                      blocksize=(dataParTasksPerLocale,
                                                 keybuffsz),
                                      targetLocales=MyLocales);

// Map domains to locales
const keyDom   = keySpace dmapped Block(boundingBox=keySpace);
const countDom = if distType==ISDistType.block
  then countSpace dmapped Block(boundingBox=countSpace)
  else countSpace dmapped BlockCyclic(startIdx=countSpace.low,
                                      blocksize=(dataParTasksPerLocale,1),
                                      targetLocales=MyLocales);

// Now declare the main distributed arrays
var key:      [keyDom] int;	// random numbers between 0 and range -1
var rank:     [keyDom] int;	// store ranks here
var sortkey:  [keyDom] int;	// used for verification
var keybuff:  [buffDom]int;	// used for remote storage
var keycount: [countDom] int = 0;	// store #occurrences of values
var keycountbuf: [countDom] int = 0;	// buffer for pre-fix scan

// these counter arrays are also distributed
var bucket_cnts: [bucketDom] int;	// there is a row here for each pe (rank, locale)
var bucket_ptrs: [bucketDom] int;  // similarly, a row for each pe, a column for each bucket

// Timers
var tsetup, tsort, tverif: Timer;
var tloops: [1..5] Timer;

// Compute log of the number of pes and num keys per pe
var log2npes: int = 0;
while (1<<log2npes < npes) do {log2npes += 1; }
var nkeys_per_pe: int = nkeys >> log2npes;
var log2nbuckets: int = 0;
while (1<<log2nbuckets < nbuckets) do {log2nbuckets += 1; }

proc main () {

  tsetup.start();

  writef ("NAS Parallel Benchmarks 2.4 -- IS Benchmark\n" );
  writef (" Size:                       #########  (class %t)\n", nkeys, probClass);
  writef (" Iterations:                 #########\n",maxIterations);
  writef (" Number of locales:          #########\n",numLocales);
  writef (" Number of tasks per locale: #########\n",dataParTasksPerLocale);
  writef (" \n");

  // initialize key values
  gen_keys();

  tsetup.stop();
  if (printTime)
    {writeln ("Elapsed time for setup    =",tsetup.elapsed()," seconds");}

  if (DEBUG) then writeln("npes =",npes," log2npes=",log2npes);
  // ," log2nbuckets=", if ISDistType==block then "N/A" else log2nbuckets);

  tsort.start();				// start the timed section
						// main iteration loop
  for iteration in 1..maxIterations {

    if (probClass != classVals.S) then writeln (iteration," Iteration ");

    key[iteration] = iteration;
    key[iteration+maxIterations] = range - iteration;

    rank_keys ( iteration );

    partialVerification(iteration, keycountbuf);

  }						// end main iteration loop

  tsort.stop();					// end the timed section
  if (printTime) {
    writeln ("Elapsed time for key sort =", tsort.elapsed()," seconds");}
  if (printTime) {
    writeln ("Time for individual loops");
    for i in 1..5 do writeln("  Loop ",i,"  ",tloops(i).elapsed()," seconds"); }

  tverif.start();  				// full verification (full sort)
  keycount = keycountbuf;
  fullVerify();
  tverif.stop();
  if (printTime)
    {writeln ("Elapsed time for full verify =",tverif.elapsed()," seconds");}

  // Now complete output

  writef("\n\n IS Benchmark Completed\n");
  writef(" Class           =                         %t\n",probClass);
  writef(" Size            = #########################\n",nkeys);
  writef(" Iterations      = #########################\n",maxIterations);
  if (printTime) {
      writef(" Time in seconds = ######################.##\n",
        tsort.elapsed(TimeUnits.seconds));
      writef(" Mop/s total     = ######################.##\n",
        (maxIterations*nkeys)/tsort.elapsed(TimeUnits.seconds)/1000000 );
  }
  writeln(" Operation type  = ","              keys ranked");
  if (passedVerifications ==  (maxIterations*5+1) ) then
    writeln(" Verification    =                SUCCESSFUL");
  else
    writeln(" Verification    =              UNSUCCESSFUL");

} // end of main program

proc rank_keys ( iteration ) {

  bucket_cnts = 0;			// initialize bucket_cnts and bucket_ptrs
  bucket_ptrs = 0;			//   for each iteration
  keycount    = 0;

  // count the number of occurrences of each key value (each pe can do their share independently)

  tloops(1).start();
  forall (k,i) in zip(key, keyDom) do {
    on k do {
      var ibucket: int = k >> (log2range-log2nbuckets);  // this is equivalent to (key(i)/range)*nbuckets
      var mype: int = i / nkeys_per_pe;
      if false {
        writeln("mype=",mype," i=",i," key=",k," ibucket=",ibucket);
      }
      bucket_cnts[mype, ibucket] += 1;
    }
  }

  tloops(1).stop();
  if (printLoopTimings) then writeln("time for first loop =",tloops(1).elapsed());
  if (DEBUG) {
    var totbucket: [0..nbuckets-1] int;
    for ibucket in 0..nbuckets-1 {
      totbucket[ibucket] = + reduce (bucket_cnts[0..npes-1,ibucket]);
    }
    writeln ("total # keys by bucket= ",totbucket);
  }
  if (printArrays) { writeln("bucket_cnts"); writeln (bucket_cnts); }

  // need to synchronize here - co/forall construct above takes care of that for us

  // figure out pointers for keybuff -- even though these are remote references,
  // each pe can pre-fix scan a column independently

  var hi0 = if distType==ISDistType.block then 0..npes-1
    else 0..nbuckets-1;
  var hi1 = if distType==ISDistType.block then 1..nbuckets-1
    else 1..npes-1;
  tloops(2).start();
  forall b in hi0 do {
    for i in hi1 {
      bucket_ptrs[i, b] = bucket_ptrs[i-1, b] + bucket_cnts[i-1, b];
      if false then writeln("bucket_ptrs", (i,b), "=", bucket_ptrs[i, b]);
    }
  }

  tloops(2).stop();
  if (printLoopTimings) then writeln("time for second loop =",tloops(2).elapsed());
  if (printArrays) { writeln ("bucket_ptrs"); writeln (bucket_ptrs); }

  // now send off the keys -- there are remote references here, but there are no race conditions,
  //                          that is, each pe can work independently
  tloops(3).start();
  forall (k,i) in zip(key, keyDom) do {
    on k do {
      var mype: int = i / nkeys_per_pe;   // change this to a shift?
      var ibucket: int;
      ibucket = k >> (log2range-log2nbuckets);
      if false {writeln("2 mype=",mype," i=",i," key=",k," ibucket=",ibucket);}
      if false then writeln("2 bucket_ptrs", (mype,ibucket), "=", bucket_ptrs[mype,ibucket]);
      keybuff(ibucket, bucket_ptrs[mype,ibucket]) = k;
      bucket_ptrs[mype, ibucket] += 1;
    }
  }
  tloops(3).stop();
  if (printLoopTimings) then writeln("time for 3rd loop =",tloops(3).elapsed());
  if (printArrays) { writeln ("keybuff");     writeln (keybuff); }
  if (printArrays) { writeln ("bucket_ptrs"); writeln (bucket_ptrs); }

  // need to sychronize here - implicit sync of coforall takes care of it for us

  // now think of operating in an alternate domain
  // each pe does this bit
  tloops(4).start();
  forall b in 0..nbuckets-1 do {
    on keybuff[b,0] do     {			// is this necessary?
      var mykeys: int = bucket_ptrs[npes-1, b];
      for i in 0..mykeys-1 {
        var bindx = if distType==ISDistType.block then keybuff[b,i]
          else (keybuff[b,i], 0);
        keycount[bindx] += 1;
      }
    }
  }
  tloops(4).stop();
  if (printLoopTimings) then writeln("time for 4th loop =",tloops(4).elapsed());
  if (printArrays) {writeln ("keycount"); writeln (keycount);}

  // prefix scan the keycount array - this is a global operation, everyone participates
  tloops(5).start();
  keycountbuf = + scan (keycount);
  /*
  if distType==ISDistType.block {
    for i in 1..range-1 {
      keycount[i] += keycount[i-1];
    }
    keycountbuf = keycount;
  } else {
  }
  // keycountbuf = + scan (keycount);
  */
  tloops(5).stop();
  if printLoopTimings then writeln("time for 5th loop =",tloops(5).elapsed());
  if printArrays { writeln ("scanned keycountbuf"); writeln (keycountbuf); }

}

proc partialVerification(iteration: int, keyBuff1 /*:[countDom]int*/ ) {

  //  Performs a partial verification after each iteration.
  //  This code is pretty much a straight translation from the C reference
  //  version except that we use enums to access the comparison values directly.

  for i in 0..4 {
    var k = key(testIndexArray(probClass,i));
    var kbidx = if distType==ISDistType.block then k-1 else (k-1,0);
    select probClass {
      when classVals.S do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i)+iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i)-iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.W do {
        if (i < 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + (iteration-2)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.A do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.B do {
        if (i == 1 || i == 2 || i == 4) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.C do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
    }
  }
  if (DEBUG) {writeln("After partial verification, passedVerifications=",passedVerifications); }
}	// end partial verification

proc fullVerify () {

  // Perform a full verification after all iterations are complete.
  // Since the keys were never actually sorted, we need to do that first.

  for mype in 0..nbuckets-1 do {
    var mykeys: int = bucket_ptrs[npes-1, mype];
    var val: int;
    for i in 0..mykeys-1 {
      var val = if distType==ISDistType.block then keybuff[mype,i]
        else (keybuff[mype,i],0);
      keycount[val] -= 1;
      keybuff[mype,i] = keycount[val];  // keycount is now the row, reuse keybuff for local storage
    }
  }
  if (printArrays) {writeln ("keycount"); writeln (keycount);}
  if (printArrays) {writeln ("keybuff "); writeln (keybuff);}

  // store the ranks in the rank array -- think of being back on the orginal domain
  // first, adjust the bucketptr array

  forall mype in 0..npes-1 do {
    var first: int = mype*nkeys_per_pe;
    var last:  int = first+nkeys_per_pe-1;
    var ibucket: int;
    // var r = last..first by -1;
    var i: int = last;
    for ii in first..last {			// going backwards
      ibucket = key(i) >> (log2range-log2nbuckets);
      bucket_ptrs[mype,ibucket] -= 1;
      if (DEBUG) {writeln(" indx=",i," key=",key(i)," ibucket=",ibucket,
                          " bucket_ptr=", bucket_ptrs[mype,ibucket],
                          " keybuff =",  keybuff[ibucket, bucket_ptrs[mype,ibucket]]); }
      rank[i] = keybuff[ibucket, bucket_ptrs[mype,ibucket]];
      i = i -1;
    }                         // end for
  }                           // end coforall

  // Finally, sort the keys
  for i in 0..nkeys-1 { sortkey[rank[i]] = key[i]; }

  if (printArrays) {writeln ("Rank "); writeln (rank);}
  if (printArrays) {writeln ("Sortd="); writeln(sortkey);}

  // Verify that the keys are sorted correctly
  if ( && reduce ( sortkey[0..nkeys-2] <= sortkey[1..nkeys-1] ) ) then {passedVerifications += 1;}
  if (DEBUG) {writeln ("full Verify: passedVerifications=",passedVerifications);}

}

proc gen_keys () {
  // initialize key values - do this on each rank or locale such that the key values
  // are the same no matter how many locales or ranks are used
  // -- come back later and see if this can be simplified w/ a local range

  coforall loc in Locales do {
    on loc do {
      var tmpreals: [1..4] real;
      var seed: int(64) = 314159265;

      var mype:  int = here.id;
      var first: int = mype*(nkeys/numLocales);
      var last : int = first+(nkeys/numLocales) - 1;

      var rs = new RandomStream(seed);
      rs.skipToNth(first*4+1);
      for i in first..last {
        rs.fillRandom(tmpreals);
        key(i) = ( (range>>2)*(+ reduce tmpreals ) ): int;
      }
      delete rs;
    }
  }
}
=======
/*
   NPB Integer Sort
   Chapel parallel version: multi-task, multi-locale
     based on bucket sort (same as NPB reference version)

   Originally written by Margaret Cahir (Cray, Inc.)

   NOTES on the Block version:
     Limitations:
       - works with 32-bit integers only
       - tested on 1, 2 and 4 locales, Classes S, A, B and C (not D)

   NOTES on the BlockCyclic version:
     Still need to
     - adjust to solve for Class D (means adjusting for int64)
     - can gen keys be made a little cleaner? (see note)
     - can a backwards index be made to work? see note in fullVerify
  	var mype: int = i / nkeys_per_pe;  //  can this be i >> log2nkeys_per_pe?
     - loop2: can we use scan in line below

   Modifications [sungeun@cray.com]: 12/2011

   - Combined Block and BlockCyclic versions

*/

// Problem size parameters
enum  classVals {S=0, W=1, A=2, B=3, C=4};
const Classes: domain(classVals);
const ClassNkeys:  [Classes] int = ( 16, 20, 23, 25, 27 ),
      ClassRanges: [Classes] int = ( 11, 16, 19, 21, 23 );

// Set of config constants that control size of problem and output options
config const probClass = classVals.S;	 // default to Sample class
config const maxIterations = 10;	 // can be dialed down for devel runs
config const DEBUG:        bool = false; // shows progress and summary info
config param printLoopTimings: bool = false; // prints timing info
config param printArrays:  bool = false; // prints out arrays (can be long)
config const printTime    = false;	 // turn off for test verification

// Standard modules
use Time, Random, BlockDist, BlockCycDist;
enum ISDistType { block, blockcyclic };
config param distType = ISDistType.block;

// Comparison numbers for partial verification (these are given)
var testD: domain(2) =  {0..4, 0..4};	// # of Classes, # of keys to verify
var testIndexArray: [testD] int = (	( 48427,17148,23627,62548,4431),
					( 357773,934767,875723,898999,404505),
					( 2112377,662041,5336171,3642833,4250760),
					( 41869,812306,5102857,18232239,26860214),
					( 44172927,72999161,74326391,129606274,21736814) );
const testRankArray: [testD]  int = (   ( 0,18,346,64917,65463),
					( 1249,11698,1039987,1043896,1048018),
					( 104,17523,123928,8288932,8388264),
					( 33422937,10244,59149,33135281,99),
					( 61147,882988,266290,133997595,133525895) );

var passedVerifications = 0;		// verification counter

// Set problem sizes and storage requirements
const log2range = ClassRanges(probClass);
const log2nkeys = ClassNkeys (probClass);
var range:    int = 1 << log2range;	// left shift for power of 2 values
var nkeys:    int = 1 << log2nkeys;
var tasksPerLocale = if dataParTasksPerLocale == 0
                     then here.maxTaskPar
                     else dataParTasksPerLocale;
var npes:     int = numLocales*tasksPerLocale;
var log2npes: int = 0;
while (1<<log2npes < npes) do {log2npes += 1; }
if 2**log2npes > npes {
  if DEBUG then writeln("Resizing npes to a power of 2");
  log2npes -= 1;
  npes = 2 ** log2npes;
  if npes % numLocales != 0 {
    writeln("Unable to resize number of tasks to a power of two.");
    writeln("The nearest power of two below npes is not divisible by numLocales");
    writeln("Provided number of tasks: ", numLocales * tasksPerLocale);
    halt();
  }
  tasksPerLocale = npes / numLocales;
}
var nbuckets: int = if distType==ISDistType.block then npes else 4*npes;

var keybuff_pe: real = 3*nkeys/npes;
var keybuffsz:  int  = keybuff_pe:int;

// Domains for main arrays
const keySpace:    domain(1)= {0..#nkeys};

// mcahir: Alas, it seems a block cyclic dist array has to be 2D so ended up making countSpace a 2D domain
const countSpace = if distType==ISDistType.block then {0..#range}
                                                 else {0..#range, 0..0};
const bucketSpace: domain(2)= {0..#npes,0..#nbuckets};
const buffSpace:   domain(2)= {0..#nbuckets,0..#keybuffsz};

// ... block a 2D space into horizontal slabs
const MyLocaleView = {0..#numLocales, 1..1};
const MyLocales: [MyLocaleView] locale = reshape(Locales, MyLocaleView);
const bucketDom = bucketSpace dmapped Block(boundingBox=bucketSpace, targetLocales=MyLocales, dataParTasksPerLocale=tasksPerLocale);
const buffDom = if distType==ISDistType.block
  then buffSpace dmapped Block(boundingBox=buffSpace, targetLocales=MyLocales, dataParTasksPerLocale=tasksPerLocale)
  else buffSpace dmapped BlockCyclic (startIdx=buffSpace.low,
                                      blocksize=(tasksPerLocale,
                                                 keybuffsz),
                                      targetLocales=MyLocales);

// Map domains to locales
const keyDom   = keySpace dmapped Block(boundingBox=keySpace, dataParTasksPerLocale=tasksPerLocale);
const countDom = if distType==ISDistType.block
  then countSpace dmapped Block(boundingBox=countSpace, dataParTasksPerLocale=tasksPerLocale)
  else countSpace dmapped BlockCyclic(startIdx=countSpace.low,
                                      blocksize=(tasksPerLocale,1),
                                      targetLocales=MyLocales);

// Now declare the main distributed arrays
var key:      [keyDom] int;	// random numbers between 0 and range -1
var rank:     [keyDom] int;	// store ranks here
var sortkey:  [keyDom] int;	// used for verification
var keybuff:  [buffDom]int;	// used for remote storage
var keycount: [countDom] int = 0;	// store #occurrences of values
var keycountbuf: [countDom] int = 0;	// buffer for pre-fix scan

// these counter arrays are also distributed
var bucket_cnts: [bucketDom] int;	// there is a row here for each pe (rank, locale)
var bucket_ptrs: [bucketDom] int;  // similarly, a row for each pe, a column for each bucket

// Timers
var tsetup, tsort, tverif: Timer;
var tloops: [1..5] Timer;

// Compute log of the number of pes and num keys per pe
var nkeys_per_pe: int = nkeys >> log2npes;
var log2nbuckets: int = 0;
while (1<<log2nbuckets < nbuckets) do {log2nbuckets += 1; }

proc main () {

  tsetup.start();

  writeln ("NAS Parallel Benchmarks 2.4 -- IS Benchmark" );
  writeln (" Size:                       ", format("#########",nkeys),"  (class ",probClass,")");
  writeln (" Iterations:                 ", format("#########",maxIterations));
  writeln (" Number of locales:          ", format("#########",numLocales));
  writeln (" Number of tasks per locale: ", format("#########",dataParTasksPerLocale));
  writeln (" ");

  // initialize key values
  gen_keys();

  tsetup.stop();
  if (printTime)
    {writeln ("Elapsed time for setup    =",tsetup.elapsed()," seconds");}

  if (DEBUG) then writeln("npes =",npes," log2npes=",log2npes);
  // ," log2nbuckets=", if ISDistType==block then "N/A" else log2nbuckets);

  tsort.start();				// start the timed section
						// main iteration loop
  for iteration in 1..maxIterations {

    if (probClass != classVals.S) then writeln (iteration," Iteration ");

    key[iteration] = iteration;
    key[iteration+maxIterations] = range - iteration;

    rank_keys ( iteration );

    partialVerification(iteration, keycountbuf);

  }						// end main iteration loop

  tsort.stop();					// end the timed section
  if (printTime) {
    writeln ("Elapsed time for key sort =", tsort.elapsed()," seconds");}
  if (printTime) {
    writeln ("Time for individual loops");
    for i in 1..5 do writeln("  Loop ",i,"  ",tloops(i).elapsed()," seconds"); }

  tverif.start();  				// full verification (full sort)
  keycount = keycountbuf;
  fullVerify();
  tverif.stop();
  if (printTime)
    {writeln ("Elapsed time for full verify =",tverif.elapsed()," seconds");}

  // Now complete output

  writeln("\n\n IS Benchmark Completed");
  writeln(" Class           = ","                        ",probClass);
  writeln(" Size            = ", format("#########################",nkeys));
  writeln(" Iterations      = ", format("#########################",maxIterations));
  if (printTime) {
      writeln(" Time in seconds = ", format("######################.##",
        tsort.elapsed(TimeUnits.seconds)) );
      writeln(" Mop/s total     = ", format("######################.##",
        (maxIterations*nkeys)/tsort.elapsed(TimeUnits.seconds)/1000000 ) );
  }
  writeln(" Operation type  = ","              keys ranked");
  if (passedVerifications ==  (maxIterations*5+1) ) then
    writeln(" Verification    =                SUCCESSFUL");
  else
    writeln(" Verification    =              UNSUCCESSFUL");

} // end of main program

proc rank_keys ( iteration ) {

  bucket_cnts = 0;			// initialize bucket_cnts and bucket_ptrs
  bucket_ptrs = 0;			//   for each iteration
  keycount    = 0;

  // count the number of occurrences of each key value (each pe can do their share independently)

  tloops(1).start();
  forall (k,i) in zip(key, keyDom) do {
    on k do {
      var ibucket: int = k >> (log2range-log2nbuckets);  // this is equivalent to (key(i)/range)*nbuckets
      var mype: int = i / nkeys_per_pe;
      if false {
        writeln("mype=",mype," i=",i," key=",k," ibucket=",ibucket);
      }
      bucket_cnts[mype, ibucket] += 1;
    }
  }

  tloops(1).stop();
  if (printLoopTimings) then writeln("time for first loop =",tloops(1).elapsed());
  if (DEBUG) {
    var totbucket: [0..nbuckets-1] int;
    for ibucket in 0..nbuckets-1 {
      totbucket[ibucket] = + reduce (bucket_cnts[0..npes-1,ibucket]);
    }
    writeln ("total # keys by bucket= ",totbucket);
  }
  if (printArrays) { writeln("bucket_cnts"); writeln (bucket_cnts); }

  // need to synchronize here - co/forall construct above takes care of that for us

  // figure out pointers for keybuff -- even though these are remote references,
  // each pe can pre-fix scan a column independently

  var hi0 = if distType==ISDistType.block then 0..npes-1
    else 0..nbuckets-1;
  var hi1 = if distType==ISDistType.block then 1..nbuckets-1
    else 1..npes-1;
  tloops(2).start();
  forall b in hi0 do {
    for i in hi1 {
      bucket_ptrs[i, b] = bucket_ptrs[i-1, b] + bucket_cnts[i-1, b];
      if false then writeln("bucket_ptrs", (i,b), "=", bucket_ptrs[i, b]);
    }
  }

  tloops(2).stop();
  if (printLoopTimings) then writeln("time for second loop =",tloops(2).elapsed());
  if (printArrays) { writeln ("bucket_ptrs"); writeln (bucket_ptrs); }

  // now send off the keys -- there are remote references here, but there are no race conditions,
  //                          that is, each pe can work independently
  tloops(3).start();
  forall (k,i) in zip(key, keyDom) do {
    on k do {
      var mype: int = i / nkeys_per_pe;   // change this to a shift?
      var ibucket: int;
      ibucket = k >> (log2range-log2nbuckets);
      if false {writeln("2 mype=",mype," i=",i," key=",k," ibucket=",ibucket);}
      if false then writeln("2 bucket_ptrs", (mype,ibucket), "=", bucket_ptrs[mype,ibucket]);
      keybuff(ibucket, bucket_ptrs[mype,ibucket]) = k;
      bucket_ptrs[mype, ibucket] += 1;
    }
  }
  tloops(3).stop();
  if (printLoopTimings) then writeln("time for 3rd loop =",tloops(3).elapsed());
  if (printArrays) { writeln ("keybuff");     writeln (keybuff); }
  if (printArrays) { writeln ("bucket_ptrs"); writeln (bucket_ptrs); }

  // need to sychronize here - implicit sync of coforall takes care of it for us

  // now think of operating in an alternate domain
  // each pe does this bit
  tloops(4).start();
  forall b in 0..nbuckets-1 do {
    on keybuff[b,0] do     {			// is this necessary?
      var mykeys: int = bucket_ptrs[npes-1, b];
      for i in 0..mykeys-1 {
        var bindx = if distType==ISDistType.block then keybuff[b,i]
          else (keybuff[b,i], 0);
        keycount[bindx] += 1;
      }
    }
  }
  tloops(4).stop();
  if (printLoopTimings) then writeln("time for 4th loop =",tloops(4).elapsed());
  if (printArrays) {writeln ("keycount"); writeln (keycount);}

  // prefix scan the keycount array - this is a global operation, everyone participates
  tloops(5).start();
  keycountbuf = + scan (keycount);
  /*
  if distType==ISDistType.block {
    for i in 1..range-1 {
      keycount[i] += keycount[i-1];
    }
    keycountbuf = keycount;
  } else {
  }
  // keycountbuf = + scan (keycount);
  */
  tloops(5).stop();
  if printLoopTimings then writeln("time for 5th loop =",tloops(5).elapsed());
  if printArrays { writeln ("scanned keycountbuf"); writeln (keycountbuf); }

}

proc partialVerification(iteration: int, keyBuff1 /*:[countDom]int*/ ) {

  //  Performs a partial verification after each iteration.
  //  This code is pretty much a straight translation from the C reference
  //  version except that we use enums to access the comparison values directly.

  for i in 0..4 {
    var k = key(testIndexArray(probClass,i));
    var kbidx = if distType==ISDistType.block then k-1 else (k-1,0);
    select probClass {
      when classVals.S do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i)+iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i)-iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.W do {
        if (i < 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + (iteration-2)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.A do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - (iteration-1)) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.B do {
        if (i == 1 || i == 2 || i == 4) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
      when classVals.C do {
        if (i <= 2) {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) + iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        } else {
          if (keyBuff1(kbidx) != testRankArray(probClass,i) - iteration) {
            writeln("Failed partial verification: iteration ",
                    iteration, ", test key ", i);
          } else {
            passedVerifications += 1;
          }
        }
      }
    }
  }
  if (DEBUG) {writeln("After partial verification, passedVerifications=",passedVerifications); }
}	// end partial verification

proc fullVerify () {

  // Perform a full verification after all iterations are complete.
  // Since the keys were never actually sorted, we need to do that first.

  for mype in 0..nbuckets-1 do {
    var mykeys: int = bucket_ptrs[npes-1, mype];
    var val: int;
    for i in 0..mykeys-1 {
      var val = if distType==ISDistType.block then keybuff[mype,i]
        else (keybuff[mype,i],0);
      keycount[val] -= 1;
      keybuff[mype,i] = keycount[val];  // keycount is now the row, reuse keybuff for local storage
    }
  }
  if (printArrays) {writeln ("keycount"); writeln (keycount);}
  if (printArrays) {writeln ("keybuff "); writeln (keybuff);}

  // store the ranks in the rank array -- think of being back on the orginal domain
  // first, adjust the bucketptr array

  forall mype in 0..npes-1 do {
    var first: int = mype*nkeys_per_pe;
    var last:  int = first+nkeys_per_pe-1;
    var ibucket: int;
    // var r = last..first by -1;
    var i: int = last;
    for ii in first..last {			// going backwards
      ibucket = key(i) >> (log2range-log2nbuckets);
      bucket_ptrs[mype,ibucket] -= 1;
      if (DEBUG) {writeln(" indx=",i," key=",key(i)," ibucket=",ibucket,
                          " bucket_ptr=", bucket_ptrs[mype,ibucket],
                          " keybuff =",  keybuff[ibucket, bucket_ptrs[mype,ibucket]]); }
      rank[i] = keybuff[ibucket, bucket_ptrs[mype,ibucket]];
      i = i -1;
    }                         // end for
  }                           // end coforall

  // Finally, sort the keys
  for i in 0..nkeys-1 { sortkey[rank[i]] = key[i]; }

  if (printArrays) {writeln ("Rank "); writeln (rank);}
  if (printArrays) {writeln ("Sortd="); writeln(sortkey);}

  // Verify that the keys are sorted correctly
  if ( && reduce ( sortkey[0..nkeys-2] <= sortkey[1..nkeys-1] ) ) then {passedVerifications += 1;}
  if (DEBUG) {writeln ("full Verify: passedVerifications=",passedVerifications);}

}

proc gen_keys () {
  // initialize key values - do this on each rank or locale such that the key values
  // are the same no matter how many locales or ranks are used
  // -- come back later and see if this can be simplified w/ a local range

  coforall loc in Locales do {
    on loc do {
      var tmpreals: [1..4] real;
      var seed: int(64) = 314159265;

      var mype:  int = here.id;
      var first: int = mype*(nkeys/numLocales);
      var last : int = first+(nkeys/numLocales) - 1;

      var rs = new RandomStream(seed);
      rs.skipToNth(first*4+1);
      for i in first..last {
        rs.fillRandom(tmpreals);
        key(i) = ( (range>>2)*(+ reduce tmpreals ) ): int;
      }
      delete rs;
    }
  }
}
>>>>>>> 74e3adc0e5f4cdcd097fcf198bf40559ba0ac2c4


Make quicksort example do fewer iterations.

I changed it to 2**15 because 2**20 runs for over 600 seconds on the
simulator for x1/x2.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13995 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// An example of a parallel quick sort implementation that uses
// "cobegin" to make each recursive call in parallel and "serial" to
// limit the number of threads.
//

use Random, Time; // for random number generation and the Timer class

var timer: Timer; // to time the sort

config var n: int = 2**15;      // the size of the array to be sorted
config var pdepth: int = 1;     // the recursive depth to serialize
                                //  use --pdepth=-1 for unlimited threads
config var verbose: int = 0;    // print out this many elements in array
config var timing: bool = true; // set timing to false to disable timer

var A: [1..n] real; // array of real numbers

//
// initialize array with random numbers
//
fillRandom(A);

//
// print out front of array if verbose flag is set
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// start timer, call parallel quick sort routine, stop timer
//
if timing then timer.start();
pqsort(pdepth, A);
if timing then timer.stop();

//
// report sort time
//
if timing then writeln("sorted in ", timer.elapsed(), " seconds");

//
// print out front of array if verbose flag is set
//   values should now be in sorted order
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// verify that array is sorted or halt
//
for i in 2..n do
  if A(i) < A(i-1) then
    halt("A(", i-1, ") == ", A(i-1), " > A(", i, ") == ", A(i));

writeln("verification success");

//
// pqsort -- parallel quick sort
//
//   pdepth: number of recursive calls to make before serializing
//   arr: generic 1D array of values (real, int, ...)
//   low: lower bound of array to start sort at, defaults to whole array
//   high: upper bound of array to stop sort at, defaults to whole array
//
def pqsort(pdepth: int,
           arr: [],
           low: int = arr.domain.low,
           high: int = arr.domain.high) where arr.rank == 1 {

  //
  // base case: arr[low..high] is small enough to bubble sort
  //
  if high - low < 8 {
    bubbleSort(arr, low, high);
    return;
  }

  //
  // determine pivot and partition arr[low..high]
  //
  const pivotVal = findPivot();
  const pivotLoc = partition(pivotVal);

  //
  // make recursive calls to parallel quick sort each unsorted half of
  // the array; if pdepth is 0, start serializing; note: once
  // serialization is turned on, it is never turned off
  //
  serial pdepth == 0 do cobegin {
    pqsort(pdepth-1, arr, low, pivotLoc-1);
    pqsort(pdepth-1, arr, pivotLoc+1, high);
  }

  //
  // findPivot -- helper routine to find pivot value using simple
  //              median-of-3 method, returns pivot value
  //
  def findPivot() {
    const mid = low + (high-low+1) / 2;

    if arr(mid) < arr(low) then arr(mid) <=> arr(low);
    if arr(high) < arr(low) then arr(high) <=> arr(low);
    if arr(high) < arr(mid) then arr(high) <=> arr(mid);

    const pivotVal = arr(mid);
    arr(mid) = arr(high-1);
    arr(high-1) = pivotVal;

    return pivotVal;
  }

  //
  // partition -- helper routine to partition array such that all
  //              values less than pivot are to its left and all
  //              values greater than pivot are to its right, returns
  //              pivot location
  //
  def partition(pivotVal) {
    var ilo = low, ihi = high-1;
    while (ilo < ihi) {
      do { ilo += 1; } while arr(ilo) < pivotVal;
      do { ihi -= 1; } while pivotVal < arr(ihi);
      if (ilo < ihi) {
        arr(ilo) <=> arr(ihi);
      }
    }
    arr(high-1) = arr(ilo);
    arr(ilo) = pivotVal;
    return ilo;
  }
}

//
// bubbleSort -- bubble sort for base case of quick sort
//
//   arr: generic 1D array of values (real, int, ...)
//   low: lower bound of array to start sort at
//   high: upper bound of array to stop sort at
//
def bubbleSort(arr: [], low: int, high: int) where arr.rank == 1 {
  for i in low..high do
    for j in low..high-1 do
      if arr(j) > arr(j+1) then
        arr(j) <=> arr(j+1);
}

//
// An example of a parallel quick sort implementation that uses
// "cobegin" to make each recursive call in parallel and "serial" to
// limit the number of threads.
//

use Random, Time; // for random number generation and the Timer class

var timer: Timer; // to time the sort

config var n: int = 2**20;      // the size of the array to be sorted
config var pdepth: int = 1;     // the recursive depth to serialize
                                //  use --pdepth=-1 for unlimited threads
config var verbose: int = 0;    // print out this many elements in array
config var timing: bool = true; // set timing to false to disable timer

var A: [1..n] real; // array of real numbers

//
// initialize array with random numbers
//
fillRandom(A);

//
// print out front of array if verbose flag is set
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// start timer, call parallel quick sort routine, stop timer
//
if timing then timer.start();
pqsort(pdepth, A);
if timing then timer.stop();

//
// report sort time
//
if timing then writeln("sorted in ", timer.elapsed(), " seconds");

//
// print out front of array if verbose flag is set
//   values should now be in sorted order
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// verify that array is sorted or halt
//
for i in 2..n do
  if A(i) < A(i-1) then
    halt("A(", i-1, ") == ", A(i-1), " > A(", i, ") == ", A(i));

writeln("verification success");

//
// pqsort -- parallel quick sort
//
//   pdepth: number of recursive calls to make before serializing
//   arr: generic 1D array of values (real, int, ...)
//   low: lower bound of array to start sort at, defaults to whole array
//   high: upper bound of array to stop sort at, defaults to whole array
//
def pqsort(pdepth: int,
           arr: [],
           low: int = arr.domain.low,
           high: int = arr.domain.high) where arr.rank == 1 {

  //
  // base case: arr[low..high] is small enough to bubble sort
  //
  if high - low < 8 {
    bubbleSort(arr, low, high);
    return;
  }

  //
  // determine pivot and partition arr[low..high]
  //
  const pivotVal = findPivot();
  const pivotLoc = partition(pivotVal);

  //
  // make recursive calls to parallel quick sort each unsorted half of
  // the array; if pdepth is 0, start serializing; note: once
  // serialization is turned on, it is never turned off
  //
  serial pdepth == 0 do cobegin {
    pqsort(pdepth-1, arr, low, pivotLoc-1);
    pqsort(pdepth-1, arr, pivotLoc+1, high);
  }

  //
  // findPivot -- helper routine to find pivot value using simple
  //              median-of-3 method, returns pivot value
  //
  def findPivot() {
    const mid = low + (high-low+1) / 2;

    if arr(mid) < arr(low) then arr(mid) <=> arr(low);
    if arr(high) < arr(low) then arr(high) <=> arr(low);
    if arr(high) < arr(mid) then arr(high) <=> arr(mid);

    const pivotVal = arr(mid);
    arr(mid) = arr(high-1);
    arr(high-1) = pivotVal;

    return pivotVal;
  }

  //
  // partition -- helper routine to partition array such that all
  //              values less than pivot are to its left and all
  //              values greater than pivot are to its right, returns
  //              pivot location
  //
  def partition(pivotVal) {
    var ilo = low, ihi = high-1;
    while (ilo < ihi) {
      do { ilo += 1; } while arr(ilo) < pivotVal;
      do { ihi -= 1; } while pivotVal < arr(ihi);
      if (ilo < ihi) {
        arr(ilo) <=> arr(ihi);
      }
    }
    arr(high-1) = arr(ilo);
    arr(ilo) = pivotVal;
    return ilo;
  }
}

//
// bubbleSort -- bubble sort for base case of quick sort
//
//   arr: generic 1D array of values (real, int, ...)
//   low: lower bound of array to start sort at
//   high: upper bound of array to stop sort at
//
def bubbleSort(arr: [], low: int, high: int) where arr.rank == 1 {
  for i in low..high do
    for j in low..high-1 do
      if arr(j) > arr(j+1) then
        arr(j) <=> arr(j+1);
}