Fixed another goof in test/studies/ssca2/vass that was made in r20177.

Now this should *really* match test/release/examples/benchmarks/ssca2 as of r20165, modulo the remarks in r20177.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20180 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module analyze_RMAT_graph_associative_array {

  //
  // If graphInputFile != "", read in the graph from the files with
  // that prefix; do not run edge generation or Kernel 1.
  // Otherwise, if graphOutputFile != "" then write out the graph
  // generated by edge generation and Kernel 1 into those files.
  //
  // The *DStyle strings control the debugging printouts during I/O.
  // See interpretDStyle() for details.
  //
  // If graphVerifyFile != "" and the graph was read from graphInputFile,
  // write it out to graphVerifyFile. Could be used for correctness checking.
  //
  // The graph size in graphInputFile must match the SCALE config constant.
  //
  config const graphInputFile    = "";
  config const graphInputDStyle  = "-";
  config const graphOutputFile   = "";
  config const graphOutputDStyle = "-";
  config const graphVerifyFile   = "";
  config const graphVerifyDStyle = "-";

  // used during graph allocation
  config const initialRMATNeighborListLength =
    if graphInputFile == "" then 16 else 0;

    record VertexData {
      var ndom = [1..initialRMATNeighborListLength];
      var neighborIDs: [ndom] int(64);
      var edgeWeights: [ndom] int(64);

      // This is used for graph construction only.
      // TODO: move to a separate array, to be deallocated after construction.
      // Note: beware of --noRefCount vs. deallocation.
      var firstAvailableNeighborPosition$: sync ndom.idxType = ndom.low;

      // keep some statistics; guarded by firstAvailableNeighborPosition$
      var growCount = 0, shrinkCount = 0;

      proc numNeighbors()  return ndom.numIndices;

      proc addEdgeOnVertex(uArg, vArg, wArg) {
        on this do {
          // todo: the compiler should make these values local automatically!
          const /*u = uArg,*/ v = vArg, w = wArg;
          local {
            // grab the vertex lock
            const edgePos = firstAvailableNeighborPosition$;
            const prevNdomLen = ndom.high;
            if edgePos > prevNdomLen {
              // grow our arrays, by 2x
              growCount += 1;
              ndom = [1..prevNdomLen * 2];
              // bounds checking below will ensure (edgePos <= ndom.high)
            }
            // release the lock - don't need it any more
            firstAvailableNeighborPosition$ = edgePos + 1;

            // store the edge
            neighborIDs[edgePos] = v;
            edgeWeights[edgePos] = w;
          }
        } // on
      }

      // not parallel-safe
      proc tidyNeighbors() {
        local {
          var edgeCount = firstAvailableNeighborPosition$.readFF() - 1;
          RemoveDuplicates(1, edgeCount);
          // TODO: ideally if we don't save much memory, do not resize
          if edgeCount != ndom.numIndices {
            shrinkCount += 1;
            ndom = 1..edgeCount;
          }
          // writeln("stats ", growCount, " ", shrinkCount, ".");
        }
      }

      //
      // Jargon: a "duplicate" is an edge v1->v2 for which
      // there is another edge v1->v2, possibly with a different weight.
      //
      proc RemoveDuplicates(lo, inout hi) {
        param showArrays = false;  // beware of 'local' in the caller
        const style = new iostyle(min_width = 3);
        if showArrays {
          writeln("starting ", lo, "..", hi);
          stdout.writeln(neighborIDs(lo..hi), style);
          stdout.writeln(edgeWeights(lo..hi), style);
        }

        // TODO: remove the duplicates as we sort
        // InsertionSort, keep duplicates
        for i in lo+1..hi {
          const ithNID = neighborIDs(i);
          const ithEDW = edgeWeights(i);
          var inserted = false;

          for j in lo..i-1 by -1 {
            if (ithNID < neighborIDs(j)) {
              neighborIDs(j+1) = neighborIDs(j);
              edgeWeights(j+1) = edgeWeights(j);
            } else {
              neighborIDs(j+1) = ithNID;
              edgeWeights(j+1) = ithEDW;
              inserted = true;
              break;
            }
          }

          if (!inserted) {
            neighborIDs(lo) = ithNID;
            edgeWeights(lo) = ithEDW;
          }
        }
        //writeln("sorted ", lo, "..", hi);

        // remove the duplicates
        var foundDup = false;
        var indexDup: int;
        var lastNID = neighborIDs(lo);

        for i in lo+1..hi {
          const currNID = neighborIDs(i);
          if lastNID == currNID {
            foundDup = true;
            indexDup = i;
            break;
          } else {
            lastNID = currNID;
          }
        }

        if foundDup {
          // indexDup points to a hole
          // the already-found dup is dropped before entering the loop
          for i in indexDup+1..hi {
            const currNID = neighborIDs(i);
            if lastNID == currNID {
              // dropping this duplicate
            } else {
              // moving a non-duplicate value
              neighborIDs(indexDup) = currNID;
              edgeWeights(indexDup) = edgeWeights(i);
              indexDup += 1;
              lastNID = currNID;
            }
          }
          hi = indexDup - 1;
        }
        //writeln("eliminated dups ", lo, "..", hi);

        // VerifySort
        if boundsChecking then
          for i in lo..hi-1 do
            if !( neighborIDs(i) < neighborIDs(i+1) ) then
              writeln("unsorted for i = ", i, "   ",
                      neighborIDs(i), " !< ", neighborIDs(i+1));

        if showArrays {
          writeln("sorted ", lo, "..", hi);
          stdout.writeln(neighborIDs(lo..hi), style);
          stdout.writeln(edgeWeights(lo..hi), style);
          writeln();
        }
      }  // RemoveDuplicates

    } // record VertexData

  // +========================================================================+
  // |  Define associative array-based representations for general sparse     |
  // |  graphs. Provide execution template to generate a random RMAT graph    |
  // |  of a specified size and execute and verify SSCA2 kernels 2 through 4. |
  // =========================================================================+

  proc  generate_and_analyze_associative_array_RMAT_graph_representation {

    // -----------------------------------------------------------------
    // compute a random power law graph with 2^SCALE vertices, using
    // the RMAT generator. Initially generate a list of triples.
    // Then convert it to a Chapel representation of a sparse graph,
    // timing this step (Kernel 1).  Finally, execute Kernels 2, 3 and 4
    // of SSCA #2, using identically the same code as in the various
    // torus cases.
    // -----------------------------------------------------------------

    use SSCA2_compilation_config_params, SSCA2_execution_config_consts;

    use SSCA2_driver, SSCA2_RMAT_graph_generator;

    use BlockDist;

    use io_RMAT_graph;

    var n_raw_edges = 8 * N_VERTICES;

    assert ( SCALE > 1, "SCALE must be greater than 1");

    select SCALE {
      when 2 do n_raw_edges = N_VERTICES / 2;
      when 3 do n_raw_edges = N_VERTICES;
      when 4 do n_raw_edges = 2 * N_VERTICES;
      when 5 do n_raw_edges = 4 * N_VERTICES;
      }

    writeln ('-------------------------------------');
    writeln ('Order of RMAT generated graph:', N_VERTICES);
    writeln ('          number of raw edges:', n_raw_edges);
    writeln ('-------------------------------------');
    writeln ();

    // ------------------------------------------------------------------------
    // The data structures below are chosen to implement an irregular (sparse)
    // graph using rectangular domains and arrays.
    // Each node in the graph has a list of neighbors and a corresponding list
    // of (integer) weights for the implicit edges.
    // ------------------------------------------------------------------------

    const vertex_domain =
      if DISTRIBUTION_TYPE == "BLOCK" then
        [1..N_VERTICES] dmapped Block ( [1..N_VERTICES] )
      else
	[1..N_VERTICES] ;

    class Associative_Graph {
      const vertices;
      var   Row      : [vertices] VertexData;

      iter FilteredNeighbors( v : index (vertices) ) {
        const neighbors => Row(v).neighborIDs;
        const weights => Row(v).edgeWeights;
        for n in Row(v).ndom do
          if !FILTERING || weights(n)%8 != 0 then
            yield neighbors(n);
      }

      // Stand-alone parallel iterator would be nice
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        pragma "no copy" pragma "no auto destroy"
        const myDom = Row(v).ndom._value; // Cache the value
        // 1-d, no stride assumed in the follower
        const wholeLow = myDom.ranges(1).low;
        for block in myDom.these(tag) do
          yield (block(1).low, block(1).high, wholeLow);
      }

      // WARNING: This can't be zippered with anything other than itself
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        pragma "no copy" pragma "no auto destroy"
        const myRow = Row(v); // Cache a copy of the record
        const neighbors => myRow.neighborIDs;
        const weights => myRow.edgeWeights;
        const (low, high, wholeLow) = followThis;
        // 1-d, no stride
        const myElems = (low..high) + wholeLow;
        for n in myElems {
          if !FILTERING || weights(n)%8 != 0 then
            yield neighbors(n);
        }
      }

      // Neighbors and edge_weight can be used where iterators are expected
      proc   Neighbors  ( v : index (vertices) ) {
        return Row (v).neighborIDs;
      }

      proc   edge_weight (v : index (vertices) ) {
        return Row (v).edgeWeights;
      }

      proc   n_Neighbors (v : index (vertices) )
      {return Row (v).numNeighbors();}

    } // class Associative_Graph

    writeln("allocating Associative_Graph");
    var G = new Associative_Graph (vertex_domain);

    // ------------------------------------------------------------------
    // generate RMAT graph of the specified size, based on input config
    // values for quadrant assignment.
    // ------------------------------------------------------------------

   if graphInputFile == "" {
    Gen_RMAT_graph ( RMAT_a, RMAT_b, RMAT_c, RMAT_d,
		     SCALE, N_VERTICES, n_raw_edges, MAX_EDGE_WEIGHT, G );

    if graphOutputFile != "" then
      Writeout_RMAT_graph ( G, graphOutputFile, graphOutputDStyle );

   } else {
    Readin_RMAT_graph ( G, graphInputFile, graphInputDStyle );

    if graphVerifyFile != "" then
      Writeout_RMAT_graph ( G, graphVerifyFile, graphVerifyDStyle );
   }

    execute_SSCA2 ( G );
    writeln (); writeln ();
    delete G;
  }

}

module analyze_RMAT_graph_associative_array {

  //
  // If graphInputFile != "", read in the graph from the files with
  // that prefix; do not run edge generation or Kernel 1.
  // Otherwise, if graphOutputFile != "" then write out the graph
  // generated by edge generation and Kernel 1 into those files.
  //
  // The *DStyle strings control the debugging printouts during I/O.
  // See interpretDStyle() for details.
  //
  // If graphVerifyFile != "" and the graph was read from graphInputFile,
  // write it out to graphVerifyFile. Could be used for correctness checking.
  //
  // The graph size in graphInputFile must match the SCALE config constant.
  //
  config const graphInputFile    = "";
  config const graphInputDStyle  = "-";
  config const graphOutputFile   = "";
  config const graphOutputDStyle = "-";
  config const graphVerifyFile   = "";
  config const graphVerifyDStyle = "-";

  // used during graph allocation
  config const initialRMATNeighborListLength =
    if graphInputFile == "" then 16 else 0;

    record VertexData {
      var ndom = [1..initialRMATNeighborListLength];
      var neighborList: [ndom] (int(64), int(64));
//      var neighborIDs: [ndom] int(64);
//      var edgeWeights: [ndom] int(64);

      // This is used for graph construction only.
      // TODO: move to a separate array, to be deallocated after construction.
      // Note: beware of --noRefCount vs. deallocation.
      var firstAvailableNeighborPosition$: sync ndom.idxType = ndom.low;

      // keep some statistics; guarded by firstAvailableNeighborPosition$
      var growCount = 0, shrinkCount = 0;

      proc numNeighbors()  return ndom.numIndices;

      proc addEdgeOnVertex(uArg, vArg, wArg) {
        on this do {
          // todo: the compiler should make these values local automatically!
          const /*u = uArg,*/ v = vArg, w = wArg;
          local {
            // grab the vertex lock
            const edgePos = firstAvailableNeighborPosition$;
            const prevNdomLen = ndom.high;
            if edgePos > prevNdomLen {
              // grow our arrays, by 2x
              growCount += 1;
              ndom = [1..prevNdomLen * 2];
              // bounds checking below will ensure (edgePos <= ndom.high)
            }
            // release the lock - don't need it any more
            firstAvailableNeighborPosition$ = edgePos + 1;

            // store the edge
            neighborList[edgePos] = (v, w);
          }
        } // on
      }

      // not parallel-safe
      proc tidyNeighbors() {
        local {
          var edgeCount = firstAvailableNeighborPosition$.readFF() - 1;
          RemoveDuplicates(1, edgeCount);
          // TODO: ideally if we don't save much memory, do not resize
          if edgeCount != ndom.numIndices {
            shrinkCount += 1;
            ndom = 1..edgeCount;
          }
          // writeln("stats ", growCount, " ", shrinkCount, ".");
        }
      }

      //
      // Jargon: a "duplicate" is an edge v1->v2 for which
      // there is another edge v1->v2, possibly with a different weight.
      //
      proc RemoveDuplicates(lo, inout hi) {
        param showArrays = false;  // beware of 'local' in the caller
        const style = new iostyle(min_width = 3);
        if showArrays {
          writeln("starting ", lo, "..", hi);
          stdout.writeln(neighborList(lo..hi), style);
        }

        // TODO: remove the duplicates as we sort
        // InsertionSort, keep duplicates
        for i in lo+1..hi {
          const (ithNID, ithEDW) = neighborList(i);
          var inserted = false;

          for j in lo..i-1 by -1 {
            if (ithNID < neighborIDs(j)) {
              neighborList(j+1) = neighborList(j);
            } else {
              neighborList(j+1) = (ithNID, ithEDW);
              inserted = true;
              break;
            }
          }

          if (!inserted) {
            neighborList(lo)= (ithNID, ithEDW);
          }
        }
        //writeln("sorted ", lo, "..", hi);

        // remove the duplicates
        var foundDup = false;
        var indexDup: int;
        var lastNID = neighborIDs(lo);

        for i in lo+1..hi {
          const currNID = neighborIDs(i);
          if lastNID == currNID {
            foundDup = true;
            indexDup = i;
            break;
          } else {
            lastNID = currNID;
          }
        }

        if foundDup {
          // indexDup points to a hole
          // the already-found dup is dropped before entering the loop
          for i in indexDup+1..hi {
            const currNID = neighborIDs(i);
            if lastNID == currNID {
              // dropping this duplicate
            } else {
              // moving a non-duplicate value
              neighborList(indexDup) = (currNID, nlWeight(neighborList(i)));
              indexDup += 1;
              lastNID = currNID;
            }
          }
          hi = indexDup - 1;
        }
        //writeln("eliminated dups ", lo, "..", hi);

        // VerifySort
        if boundsChecking then
          for i in lo..hi-1 do
            if !( neighborIDs(i) < neighborIDs(i+1) ) then
              writeln("unsorted for i = ", i, "   ",
                      neighborIDs(i), " !< ", neighborIDs(i+1));

        if showArrays {
          writeln("sorted ", lo, "..", hi);
          stdout.writeln(neighborList(lo..hi), style);
          writeln();
        }
      }  // RemoveDuplicates

    } // record VertexData

  // +========================================================================+
  // |  Define associative array-based representations for general sparse     |
  // |  graphs. Provide execution template to generate a random RMAT graph    |
  // |  of a specified size and execute and verify SSCA2 kernels 2 through 4. |
  // =========================================================================+

  proc  generate_and_analyze_associative_array_RMAT_graph_representation {

    // -----------------------------------------------------------------
    // compute a random power law graph with 2^SCALE vertices, using
    // the RMAT generator. Initially generate a list of triples.
    // Then convert it to a Chapel representation of a sparse graph,
    // timing this step (Kernel 1).  Finally, execute Kernels 2, 3 and 4
    // of SSCA #2, using identically the same code as in the various
    // torus cases.
    // -----------------------------------------------------------------

    use SSCA2_compilation_config_params, SSCA2_execution_config_consts;

    use SSCA2_driver, SSCA2_RMAT_graph_generator;

    use BlockDist;

    use io_RMAT_graph;

    var n_raw_edges = 8 * N_VERTICES;

    assert ( SCALE > 1, "SCALE must be greater than 1");

    select SCALE {
      when 2 do n_raw_edges = N_VERTICES / 2;
      when 3 do n_raw_edges = N_VERTICES;
      when 4 do n_raw_edges = 2 * N_VERTICES;
      when 5 do n_raw_edges = 4 * N_VERTICES;
      }

    writeln ('-------------------------------------');
    writeln ('Order of RMAT generated graph:', N_VERTICES);
    writeln ('          number of raw edges:', n_raw_edges);
    writeln ('-------------------------------------');
    writeln ();

    // ------------------------------------------------------------------------
    // The data structures below are chosen to implement an irregular (sparse)
    // graph using rectangular domains and arrays.
    // Each node in the graph has a list of neighbors and a corresponding list
    // of (integer) weights for the implicit edges.
    // ------------------------------------------------------------------------

    const vertex_domain =
      if DISTRIBUTION_TYPE == "BLOCK" then
        [1..N_VERTICES] dmapped Block ( [1..N_VERTICES] )
      else
	[1..N_VERTICES] ;

    class Associative_Graph {
      const vertices;
      var   Row      : [vertices] VertexData;

      iter FilteredNeighbors( v : index (vertices) ) {
        const neighbors => Row(v).neighborIDs;
        const weights => Row(v).edgeWeights;
        for n in Row(v).ndom do
          if !FILTERING || weights(n)%8 != 0 then
            yield neighbors(n);
      }

      // Stand-alone parallel iterator would be nice
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind)
      where tag == iterKind.leader {
        pragma "no copy" pragma "no auto destroy"
        const myDom = Row(v).ndom._value; // Cache the value
        // 1-d, no stride assumed in the follower
        const wholeLow = myDom.ranges(1).low;
        for block in myDom.these(tag) do
          yield (block(1).low, block(1).high, wholeLow);
      }

      // WARNING: This can't be zippered with anything other than itself
      iter FilteredNeighbors( v : index (vertices), param tag: iterKind, followThis)
      where tag == iterKind.follower {
        pragma "no copy" pragma "no auto destroy"
        const myRow = Row(v); // Cache a copy of the record
        const neighbors => myRow.neighborIDs;
        const weights => myRow.edgeWeights;
        const (low, high, wholeLow) = followThis;
        // 1-d, no stride
        const myElems = (low..high) + wholeLow;
        for n in myElems {
          if !FILTERING || weights(n)%8 != 0 then
            yield neighbors(n);
        }
      }

      // Neighbors and edge_weight can be used where iterators are expected
      proc   Neighbors  ( v : index (vertices) ) {
        return Row (v).neighborIDs;
      }

      proc   edge_weight (v : index (vertices) ) {
        return Row (v).edgeWeights;
      }

      proc   n_Neighbors (v : index (vertices) )
      {return Row (v).numNeighbors();}  CONTINUE HERE

    } // class Associative_Graph

    writeln("allocating Associative_Graph");
    var G = new Associative_Graph (vertex_domain);

    // ------------------------------------------------------------------
    // generate RMAT graph of the specified size, based on input config
    // values for quadrant assignment.
    // ------------------------------------------------------------------

   if graphInputFile == "" {
    Gen_RMAT_graph ( RMAT_a, RMAT_b, RMAT_c, RMAT_d,
		     SCALE, N_VERTICES, n_raw_edges, MAX_EDGE_WEIGHT, G );

    if graphOutputFile != "" then
      Writeout_RMAT_graph ( G, graphOutputFile, graphOutputDStyle );

   } else {
    Readin_RMAT_graph ( G, graphInputFile, graphInputDStyle );

    if graphVerifyFile != "" then
      Writeout_RMAT_graph ( G, graphVerifyFile, graphVerifyDStyle );
   }

    execute_SSCA2 ( G );
    writeln (); writeln ();
    delete G;
  }

}


Update to comments to reflect that these are stubs

/*
 * Copyright 2004-2014 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// ChapelTaskTable.chpl
//
pragma "no use ChapelStandard"
module ChapelTaskTable {

  ////////////////////////////////////////////////////////////////////////{
  //
  // Exported task table code.
  //
  // In the --no-task-table mode, these are just no-op stubs and a halt()
  // if someone tries to print the task table.
  //
  ////////////////////////////////////////////////////////////////////////}

  export proc chpldev_taskTable_add(taskID   : chpl_taskID_t,
                                    lineno   : uint(32),
                                    filename : c_string,
                                    tl_info  : uint(64))
  {
  }

  export proc chpldev_taskTable_remove(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_set_active(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_set_suspended(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_get_tl_info(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_print()
  {
    halt("To use task tracking, you must recompile with --task-tracking");
  }
}

/*
 * Copyright 2004-2014 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// ChapelTaskTable.chpl
//
pragma "no use ChapelStandard"
module ChapelTaskTable {

  ////////////////////////////////////////////////////////////////////////{
  //
  // Exported task table code.
  //
  // In general, tasks may have been created before the task table was
  // initialized.
  // In that case, operations may be attempted on the task table using
  // taskIDs that are unknown to it.  That is why we check
  // for membership on all operations.
  //
  ////////////////////////////////////////////////////////////////////////}

  export proc chpldev_taskTable_add(taskID   : chpl_taskID_t,
                                    lineno   : uint(32),
                                    filename : c_string,
                                    tl_info  : uint(64))
  {
  }

  export proc chpldev_taskTable_remove(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_set_active(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_set_suspended(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_get_tl_info(taskID : chpl_taskID_t)
  {
  }

  export proc chpldev_taskTable_print()
  {
    halt("To use task tracking, you must recompile with --task-tracking");
  }
}


Oops -- forgot to wrap an "if debugBlock1D" around my last change

You can tell it's been awhile since I did any coding of note...

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15320 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Block1D Distribution
//
//      Block1D       Block1DDom     Block1DArr
//
//   LocBlock1DDist  LocBlock1DDom  LocBlock1DArr
//

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: implement the slicing interface?

config param debugBlock1D = false;

class Block1D : BaseDist {
  type idxType = int(64); // distribution's index type, for domain

  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  //
  // sjd: note this is a variable but it shouldn't be changed after
  // privatization, because it won't be reflected everywhere.  should
  // this be a constant?  how should we handle such?
  //
  var tasksPerLoc: int;

  //
  // DOWN LINK: an array of local distribution class descriptors --
  // set up in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  def Block1D(type idxType = int(64), bbox: domain(1, idxType),
              targetLocales: [] locale = Locales,
	      tasksPerLocale = 0) {
    boundingBox = bbox;
    targetLocDom = [0..#targetLocales.numElements]; // 0-based for simplicity
    targetLocs = targetLocales;

    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    tasksPerLoc = tasksPerLocale;
    if (tasksPerLoc == 0) then tasksPerLoc = min reduce targetLocs.numCores;

    if debugBlock1D then
      for loc in locDist do writeln(loc);
  }

  //
  // builds up a privatized (replicated copy)
  //
  def Block1D(type idxType = int(64), other: Block1D(idxType)) {
    boundingBox = other.boundingBox;
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    tasksPerLoc = other.tasksPerLoc;
    locDist = other.locDist;
  }

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDom(param rank: int, type idxType, param stridable: bool) {
    if idxType != this.idxType then
      compilerError("Block1D domain index type does not match distribution's");
    if rank != 1 then
      compilerError("Block1D domains are restricted to 1D (rank 1)");

    var dom = new Block1DDom(idxType=idxType, dist=this, stridable=stridable);
    dom.setup();
    return dom;
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("Block1D");
    x.writeln("-------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    //    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

class LocBlock1DDist {
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1D(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) then
      halt("Creating a local distribution class on the wrong locale");

    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const (blo, bhi) = _computeBlock(min(idxType), numelems, lo,
                                     max(idxType), numlocs, localeIdx);

    myChunk = [blo..bhi];
  }

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

class Block1DDom: BaseArithmeticDom {
  type idxType;
  param rank = 1;
  param stridable: bool;

  //
  // LEFT LINK: a pointer to the parent distribution
  //
  const dist: Block1D(idxType);

  //
  // DOWN LINK: an array of local domain class descriptors -- set up in
  // setup() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType, stridable);

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  var pid: int = -1; // privatized object id

  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      for ind in blk do
        yield ind;
  }

  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  // TODO: Abstract this subtraction of low into a function?
  // Note relationship between this operation and the
  // order/position functions -- any chance for creating similar
  // support? (esp. given how frequent this seems likely to be?)
  //
  def these(param tag: iterator) where tag == iterator.leader {
    const precomputedNumTasks = dist.tasksPerLoc;
    coforall locDom in locDoms do on locDom {
      const locBlock = locDom.myBlock - whole.low,
            numTasks = precomputedNumTasks;
      if (numTasks == 1) {
        yield locBlock;
      } else {
        coforall taskid in 0..#numTasks {
          const (lo,hi) = _computeBlock(locBlock.low, locBlock.numIndices,
                                        locBlock.low, locBlock.high,
                                        numTasks, taskid);
          yield [lo..hi];
        }
      }
    }
  }

  //
  // TODO: Abstract the addition of low into a function?
  // Note relationship between this operation and the
  // order/position functions -- any chance for creating similar
  // support? (esp. given how frequent this seems likely to be?)
  //
  // TODO: Is there some clever way to invoke the leader/follower
  // iterator on the local blocks in here such that the per-core
  // parallelism is expressed at that level?  Seems like a nice
  // natural composition and might help with my fears about how
  // stencil communication will be done on a per-locale basis.
  //
  def these(param tag: iterator, follower) where tag == iterator.follower {
    const followThis = follower + whole.low;

    for i in followThis {
      yield i;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    var arr = new Block1DArr(idxType, elemType, stridable, this);
    arr.setup();
    return arr;
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices return whole.numIndices;
  def low return whole.low;
  def high return whole.high;

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setup();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setup();
  }

  def getIndices() {
    return whole;
  }

  def getDist(): Block1D(idxType) {
    return dist;
  }

  def setup() {
    coforall localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, stridable, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBlock1D then
      for loc in dist.targetLocDom do writeln(loc, " owns ", locDoms(loc));

  }

  def supportsPrivatization() param return true;
  def privatize() {
    var privateDist = new Block1D(idxType, dist);
    var c = new Block1DDom(idxType=idxType, rank=rank, stridable=stridable, dist=privateDist);
    c.locDoms = locDoms;
    c.whole = whole;
    return c;
  }
  def reprivatize(other) {
    locDoms = other.locDoms;
    whole = other.whole;
  }
}

class LocBlock1DDom {
  type idxType;
  param stridable: bool;

  //
  // UP LINK: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1, stridable);

  //
  // a local domain describing the indices owned by this locale
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  def these() {
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    halt("This is bogus");
    yield 2;
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

class Block1DArr: BaseArr {
  type idxType;
  type eltType;
  param stridable: bool;

  //
  // LEFT LINK: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1, stridable);

  //
  // DOWN LINK: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  // SJD update: maybe can do this again with privatize simplification
  // SJD: actually, this can only be const if the Chapel array is const?
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType, stridable);

  var pid: int = -1; // privatized object id

  def setup() {
    coforall localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, stridable, dom.locDoms(localeIdx));
  }

  def supportsPrivatization() param return true;
  def privatize() {
    var dompid = dom.pid;
    var thisdom = dom;
    var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
    var c = new Block1DArr(idxType, eltType, stridable, privdom);
    c.locArr = locArr;
    return c;
  }

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    const myLocArr = locArr(here.id);
    local {
      if myLocArr.locDom.myBlock.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  def these() var {
    for loc in dom.dist.targetLocDom {
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // TODO: Rewrite this to reuse more of the global domain iterator
  // logic?  (e.g., can we forward the forall to the global domain
  // somehow?
  //
  def these(param tag: iterator) where tag == iterator.leader {
    const precomputedNumTasks = dom.dist.tasksPerLoc;
    const precomputedWholeLow = dom.whole.low;
    coforall locDom in dom.locDoms do on locDom {
      const locBlock = locDom.myBlock - precomputedWholeLow,
            numTasks = precomputedNumTasks;
      if (numTasks == 1) {
        yield locBlock;
      } else {
        coforall taskid in 0..#numTasks {
          const (lo,hi) = _computeBlock(locBlock.low, locBlock.numIndices,
                                        locBlock.low, locBlock.high,
                                        numTasks, taskid);
          yield [lo..hi];
        }
      }
    }
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
    const followThis = follower + dom.low;

    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      //
      // we don't own all the elements we're following
      //
      def accessHelper(i) var {
        local {
          if myLocArr.locDom.myBlock.member(i) then
            return myLocArr.this(i);
        }
        return this(i);
      }
      for i in followThis {
        yield accessHelper(i);
      }
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

class LocBlock1DArr {
  type idxType;
  type eltType;
  param stridable: bool;

  //
  // LEFT LINK: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) var where tag == iterator.follower {
    yield myElems(0);
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }

  def owns(x) {
    return locDom.myBlock.dim(1).boundsCheck(x.dim(1));
  }
}

//
// helper function for blocking index ranges
//
def _computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def procToData(x, lo)
    return lo + (x: lo.type) + (x:real != x:int:real): lo.type;

  const blo =
    if (blocknum == 0) then waylo
      else procToData((numelems: real * blocknum) / numblocks, lo);
  const bhi =
    if (blocknum == numblocks - 1) then wayhi
      else procToData((numelems: real * (blocknum+1)) / numblocks, lo) - 1;

  return (blo, bhi);
}

//
// Block1D Distribution
//
//      Block1D       Block1DDom     Block1DArr
//
//   LocBlock1DDist  LocBlock1DDom  LocBlock1DArr
//

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: implement the slicing interface?

config param debugBlock1D = false;

class Block1D : BaseDist {
  type idxType = int(64); // distribution's index type, for domain

  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  //
  // sjd: note this is a variable but it shouldn't be changed after
  // privatization, because it won't be reflected everywhere.  should
  // this be a constant?  how should we handle such?
  //
  var tasksPerLoc: int;

  //
  // DOWN LINK: an array of local distribution class descriptors --
  // set up in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  def Block1D(type idxType = int(64), bbox: domain(1, idxType),
              targetLocales: [] locale = Locales,
	      tasksPerLocale = 0) {
    boundingBox = bbox;
    targetLocDom = [0..#targetLocales.numElements]; // 0-based for simplicity
    targetLocs = targetLocales;

    coforall locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    tasksPerLoc = tasksPerLocale;
    if (tasksPerLoc == 0) then tasksPerLoc = min reduce targetLocs.numCores;

    for loc in locDist do writeln(loc);
  }

  //
  // builds up a privatized (replicated copy)
  //
  def Block1D(type idxType = int(64), other: Block1D(idxType)) {
    boundingBox = other.boundingBox;
    targetLocDom = other.targetLocDom;
    targetLocs = other.targetLocs;
    tasksPerLoc = other.tasksPerLoc;
    locDist = other.locDist;
  }

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDom(param rank: int, type idxType, param stridable: bool) {
    if idxType != this.idxType then
      compilerError("Block1D domain index type does not match distribution's");
    if rank != 1 then
      compilerError("Block1D domains are restricted to 1D (rank 1)");

    var dom = new Block1DDom(idxType=idxType, dist=this, stridable=stridable);
    dom.setup();
    return dom;
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("Block1D");
    x.writeln("-------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    //    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

class LocBlock1DDist {
  type idxType;

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1D(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) then
      halt("Creating a local distribution class on the wrong locale");

    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const (blo, bhi) = _computeBlock(min(idxType), numelems, lo,
                                     max(idxType), numlocs, localeIdx);

    myChunk = [blo..bhi];
  }

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

class Block1DDom: BaseArithmeticDom {
  type idxType;
  param rank = 1;
  param stridable: bool;

  //
  // LEFT LINK: a pointer to the parent distribution
  //
  const dist: Block1D(idxType);

  //
  // DOWN LINK: an array of local domain class descriptors -- set up in
  // setup() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType, stridable);

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  var pid: int = -1; // privatized object id

  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      for ind in blk do
        yield ind;
  }

  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  // TODO: Abstract this subtraction of low into a function?
  // Note relationship between this operation and the
  // order/position functions -- any chance for creating similar
  // support? (esp. given how frequent this seems likely to be?)
  //
  def these(param tag: iterator) where tag == iterator.leader {
    const precomputedNumTasks = dist.tasksPerLoc;
    coforall locDom in locDoms do on locDom {
      const locBlock = locDom.myBlock - whole.low,
            numTasks = precomputedNumTasks;
      if (numTasks == 1) {
        yield locBlock;
      } else {
        coforall taskid in 0..#numTasks {
          const (lo,hi) = _computeBlock(locBlock.low, locBlock.numIndices,
                                        locBlock.low, locBlock.high,
                                        numTasks, taskid);
          yield [lo..hi];
        }
      }
    }
  }

  //
  // TODO: Abstract the addition of low into a function?
  // Note relationship between this operation and the
  // order/position functions -- any chance for creating similar
  // support? (esp. given how frequent this seems likely to be?)
  //
  // TODO: Is there some clever way to invoke the leader/follower
  // iterator on the local blocks in here such that the per-core
  // parallelism is expressed at that level?  Seems like a nice
  // natural composition and might help with my fears about how
  // stencil communication will be done on a per-locale basis.
  //
  def these(param tag: iterator, follower) where tag == iterator.follower {
    const followThis = follower + whole.low;

    for i in followThis {
      yield i;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    var arr = new Block1DArr(idxType, elemType, stridable, this);
    arr.setup();
    return arr;
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices return whole.numIndices;
  def low return whole.low;
  def high return whole.high;

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setup();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setup();
  }

  def getIndices() {
    return whole;
  }

  def getDist(): Block1D(idxType) {
    return dist;
  }

  def setup() {
    coforall localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, stridable, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBlock1D then
      for loc in dist.targetLocDom do writeln(loc, " owns ", locDoms(loc));

  }

  def supportsPrivatization() param return true;
  def privatize() {
    var privateDist = new Block1D(idxType, dist);
    var c = new Block1DDom(idxType=idxType, rank=rank, stridable=stridable, dist=privateDist);
    c.locDoms = locDoms;
    c.whole = whole;
    return c;
  }
  def reprivatize(other) {
    locDoms = other.locDoms;
    whole = other.whole;
  }
}

class LocBlock1DDom {
  type idxType;
  param stridable: bool;

  //
  // UP LINK: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1, stridable);

  //
  // a local domain describing the indices owned by this locale
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  def these() {
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    halt("This is bogus");
    yield 2;
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

class Block1DArr: BaseArr {
  type idxType;
  type eltType;
  param stridable: bool;

  //
  // LEFT LINK: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1, stridable);

  //
  // DOWN LINK: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  // SJD: note cannot do this anymore because constructor does not
  // setup (for privatization reasons)
  // SJD update: maybe can do this again with privatize simplification
  // SJD: actually, this can only be const if the Chapel array is const?
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType, stridable);

  var pid: int = -1; // privatized object id

  def setup() {
    coforall localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, stridable, dom.locDoms(localeIdx));
  }

  def supportsPrivatization() param return true;
  def privatize() {
    var dompid = dom.pid;
    var thisdom = dom;
    var privdom = __primitive("chpl_getPrivatizedClass", thisdom, dompid);
    var c = new Block1DArr(idxType, eltType, stridable, privdom);
    c.locArr = locArr;
    return c;
  }

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    const myLocArr = locArr(here.id);
    local {
      if myLocArr.locDom.myBlock.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  def these() var {
    for loc in dom.dist.targetLocDom {
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // TODO: Rewrite this to reuse more of the global domain iterator
  // logic?  (e.g., can we forward the forall to the global domain
  // somehow?
  //
  def these(param tag: iterator) where tag == iterator.leader {
    const precomputedNumTasks = dom.dist.tasksPerLoc;
    const precomputedWholeLow = dom.whole.low;
    coforall locDom in dom.locDoms do on locDom {
      const locBlock = locDom.myBlock - precomputedWholeLow,
            numTasks = precomputedNumTasks;
      if (numTasks == 1) {
        yield locBlock;
      } else {
        coforall taskid in 0..#numTasks {
          const (lo,hi) = _computeBlock(locBlock.low, locBlock.numIndices,
                                        locBlock.low, locBlock.high,
                                        numTasks, taskid);
          yield [lo..hi];
        }
      }
    }
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
    const followThis = follower + dom.low;

    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      //
      // we don't own all the elements we're following
      //
      def accessHelper(i) var {
        local {
          if myLocArr.locDom.myBlock.member(i) then
            return myLocArr.this(i);
        }
        return this(i);
      }
      for i in followThis {
        yield accessHelper(i);
      }
    }
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

class LocBlock1DArr {
  type idxType;
  type eltType;
  param stridable: bool;

  //
  // LEFT LINK: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) var where tag == iterator.follower {
    yield myElems(0);
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }

  def owns(x) {
    return locDom.myBlock.dim(1).boundsCheck(x.dim(1));
  }
}

//
// helper function for blocking index ranges
//
def _computeBlock(waylo, numelems, lo, wayhi, numblocks, blocknum) {
  def procToData(x, lo)
    return lo + (x: lo.type) + (x:real != x:int:real): lo.type;

  const blo =
    if (blocknum == 0) then waylo
      else procToData((numelems: real * blocknum) / numblocks, lo);
  const bhi =
    if (blocknum == numblocks - 1) then wayhi
      else procToData((numelems: real * (blocknum+1)) / numblocks, lo) - 1;

  return (blo, bhi);
}


Corrected my last check in.  This should work much better.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15489 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

_extern def chpl_malloc(number, size, description, userCode, lineno, filename);

chpl_malloc(max(int(32)), max(int(32)), "test big malloc", true, 3, "allocBig.chpl");

_extern def chpl_malloc(number, size, description, true, lineno, filename);

chpl_malloc(max(int(32)), max(int(32)), "test big malloc", 3, "allocBig.chpl");


Added more missing 'zip's.

Added more missing 'zip's.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20472 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// Use the distributions we need for this computation
//
use DimensionalDist2D, ReplicatedDim, BlockCycDim;

//
// The number of matrices and the element type of those matrices
// indexType can be int or int(64), elemType can be real or complex
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 8;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

// These are solely to make the testing system happy given the COMPOPTS file.
// To be removed once COMPOPTS becomes a non-issue.
config var reproducible = false, verbose = false;

//
// The program entry point, currently module initialization
//
  printConfiguration();

  //
  // Compute targetLocales - required for Dimensional.
  // We hard-code 2 dimensions.
  //
  var targetIds: domain(2);
  var targetLocales: [targetIds] locale;
  setupTargetLocalesArray(targetIds, targetLocales, Locales);

  // Here are the dimensions of our grid of locales.
  const tl1 = targetIds.dim(1).length,
        tl2 = targetIds.dim(2).length;
  if printParams && printStats then
    writeln("target locales ", tl1, " x ", tl2);

  // Create individual dimension descriptors
  const
    // block-cyclic for 1st dimension
    bdim1 = new BlockCyclicDim(lowIdx=1, blockSize=blkSize, numLocales=tl1),
    // replicated for 1st dimension
    rdim1 = new ReplicatedDim(tl1),

    // block-cyclic for 2nd dimension
    bdim2 = new BlockCyclicDim(lowIdx=1, blockSize=blkSize, numLocales=tl2),
    // replicated for 2nd dimension
    rdim2 = new ReplicatedDim(tl2);

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  // We use 'AbD' instead of 'MatVectSpace' throughout.
  //
  const AbD: domain(2, indexType)
          dmapped DimensionalDist2D(targetLocales, bdim1, bdim2, "dim")
          = {1..n, 1..n+1},
        MatrixSpace = AbD[.., ..n];

  var Ab : [AbD] elemType,           // the matrix A and vector b
      piv: [1..n] indexType;         // a vector of pivot values

  //
  // Create the 1-d replicated arrays for schurComplement().
  //
  const
    replAD = {1..n, 1..blkSize}
      dmapped DimensionalDist2D(targetLocales, bdim1, rdim2, "distBR"),
    replBD = {1..blkSize, 1..n+1}
      dmapped DimensionalDist2D(targetLocales, rdim1, bdim2, "distRB");

  var replA: [replAD] elemType,
      replB: [replBD] elemType;

  const
    replKD = {0..0, 1..n+1}
      dmapped new dmap(replBD._value.dist);
//    dmapped DimensionalDist2D(targetLocales, rdim1, bdim2, "distRB-replK");

  var replK: [replKD] elemType;

  initAB();

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, piv);                 // compute the LU factorization

  var x = backwardSub(n);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(x);
  printResults(validAnswer, execTime);

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
proc LUFactorize(n: indexType,
                piv: [1..n] indexType) {

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    if printStats then writeln("processing block ", blk);

    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(l, piv);
    updateBlockRow(tl, tr);

    //
    // update trailing submatrix (if any)
    //
    schurComplement(blk, bl, tr, br);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The 'a' region was 'bl' in the calling
//     |aaaaa|.....|.....|.....|  function but called AD here.  Similarly,
//     +-----+-----+-----+-----+  'b' was 'tr' in the calling code, but BD
//     |aaaaa|.....|.....|.....|  here.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
proc schurComplement(blk, AD: domain, BD: domain, Rest: domain) {

  // Prevent replication of unequal-sized slices
  if Rest.numIndices == 0 then return;

  //
  // Copy data into replicated arrays so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.
  //

  const AbSlice1 => Ab[1..n, AD.dim(2)],
        AbSlice2 => Ab[BD.dim(1), 1..n+1];

  forall (ab, ra) in zip(AbSlice1, replA) do
    local
      ra = ab;
  replicateA(blk);

  forall (ab, rb) in zip(AbSlice2, replB) do
    local
      rb = ab;
  replicateB(blk);

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in Rest by (blkSize, blkSize) {
    // localize Rest explicitly as a workaround;
    // also hoist the innerRange computation
    const outterRange = Rest.dim(1)(row..#blkSize),
          innerRange  = Rest.dim(2)(col..#blkSize);

    local {
      for a in outterRange do
        for w in 1..blkSize do
          for b in innerRange do
            Ab[a,b] -= replA[a,w] * replB[w,b];
    }
  }
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
proc panelSolve(
               panel: domain,
               piv: [] indexType) {

  const blk = panel.dim(1).low;
  const cornerLocale = Ab[blk,blk].locale;

  for k in panel.dim(2) {             // iterate through the columns
    const col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.numIndices == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const (_, (pivotRow, _)) = maxloc reduce(abs(Ab(col)), col);

    // Capture the pivot value explicitly (note that result of maxloc
    // is absolute value, so it can't be used directly).
    //
    const pivotVal = Ab[pivotRow, k];

    // Swap the current row with the pivot row and update the pivot vector
    // to reflect that
    Ab[k..k, ..] <=> Ab[pivotRow..pivotRow, ..];
    piv[k] <=> piv[pivotRow];

    if (pivotVal == 0) then
      halt("Matrix cannot be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot value
    Ab[k+1.., k..k] /= pivotVal;

    // update all other values below the pivot
    // (a) replicate
    on cornerLocale {
      local
        for j in panel.dim(2)[k+1..] do
          replK[0,j] = Ab[k,j];
      replicateK(blk);
    }
    // (b) compute
    forall (i,j) in panel[k+1.., k+1..] do
      local
        Ab[i,j] -= Ab[i,k] * replK[0,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
proc updateBlockRow(
                   tl: domain,
                   tr: domain) {

  for row in tr.dim(1) {
    const activeRow = tr[row..row, ..],
          prevRows = tr.dim(1).low..row-1;

    forall (i,j) in activeRow do
      for k in prevRows do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
  }
}

//
// compute the backwards substitution
//
proc backwardSub(n: indexType) {
  const bd = Ab.domain.dim(1);  // or simply 1..n
  var x: [bd] elemType;

  for i in bd by -1 do
    x[i] = (Ab[i,n+1] - (+ reduce [j in i+1..bd.high] (Ab[i,j] * x[j])))
            / Ab[i,i];

  return x;
}

//
// print out the problem size and block size if requested
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
proc initAB() {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
proc verifyResults(x) {
  initAB();

  const axmbNorm = norm(gaxpyMinus(Ab[.., 1..n], x, Ab[.., n+1..n+1]), normType.normInf);

  const a1norm   = norm(Ab[.., 1..n], normType.norm1),
        aInfNorm = norm(Ab[.., 1..n], normType.normInf),
        x1Norm   = norm(Ab[.., n+1..n+1], normType.norm1),
        xInfNorm = norm(Ab[.., n+1..n+1], normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

/////////////////////////////////////////////////////////////////////////////

proc replicateA(abIx) {
    const fromLocId2 = targetLocalesIndexForAbIndex(2, abIx);
    coforall lid1 in 0..#tl1 do
      on targetLocales(lid1, fromLocId2) do
        coforall lid2 in 0..#tl2 do
          if lid2 != fromLocId2 then
            replA._value.localAdescs[lid1,lid2].myStorageArr =
              replA._value.localAdescs[lid1,fromLocId2].myStorageArr;
}

proc replicateB(abIx) {
    const fromLocId1 = targetLocalesIndexForAbIndex(1, abIx);
    coforall lid2 in 0..#tl2 do
      on targetLocales(fromLocId1, lid2) do
        coforall lid1 in 0..#tl1 do
          if lid1 != fromLocId1 then
            replB._value.localAdescs[lid1,lid2].myStorageArr =
              replB._value.localAdescs[fromLocId1,lid2].myStorageArr;
}

// replicate only along one column of processors,
// starting on a good locale
proc replicateK(abIx) {
    const fromLocId1 = targetLocalesIndexForAbIndex(1, abIx);
    const fromLocId2 = targetLocalesIndexForAbIndex(2, abIx);
    // verify that we are on a good locale, optionally
    if boundsChecking {
      const fromLocale = targetLocales(fromLocId1, fromLocId2);
      if here != fromLocale then
        halt("replicateK(abIx=", abIx, ")  executing on ",
             here.id, "  expecting ", fromLocale.id, " ",
             (fromLocId1, fromLocId2));
    }
    const lid2 = fromLocId2;
        coforall lid1 in 0..#tl1 do
          if lid1 != fromLocId1 then
            replK._value.localAdescs[lid1,lid2].myStorageArr =
              replK._value.localAdescs[fromLocId1,lid2].myStorageArr;
}

proc targetLocalesIndexForAbIndex(param dim, abIx)
  return (abIx / blkSize) % (if dim == 1 then tl1 else tl2);

/////////////////////////////////////////////////////////////////////////////

//
// print success/failure, the execution time and the Gflop/s value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
proc gaxpyMinus(A: [],
                x: [?xD],
                y: [?yD]) {
  var res: [1..n] elemType;

  forall i in 1..n do
    // 'serial' to avoid spawning more tasks
    serial(true) { res[i] = (+ reduce [j in xD] (A[i,j] * x[j])) - y[i,n+1]; }

  return res;
}

//
// Use standard modules for vector and matrix Norms, Random numbers
// and Timing routines
//
use Norm, Random, Time;

//
// Use the user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// Use the distributions we need for this computation
//
use DimensionalDist2D, ReplicatedDim, BlockCycDim;

//
// The number of matrices and the element type of those matrices
// indexType can be int or int(64), elemType can be real or complex
//
const numMatrices = 1;
type indexType = int,
     elemType = real;

//
// Configuration constants indicating the problem size (n) and the
// block size (blkSize)
//
config const n = computeProblemSize(numMatrices, elemType, rank=2,
                                    memFraction=2, retType=indexType),
             blkSize = 8;

//
// Configuration constant used for verification thresholds
//
config const epsilon = 2.0e-15;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.currentTime else 31415;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

// These are solely to make the testing system happy given the COMPOPTS file.
// To be removed once COMPOPTS becomes a non-issue.
config var reproducible = false, verbose = false;

//
// The program entry point, currently module initialization
//
  printConfiguration();

  //
  // Compute targetLocales - required for Dimensional.
  // We hard-code 2 dimensions.
  //
  var targetIds: domain(2);
  var targetLocales: [targetIds] locale;
  setupTargetLocalesArray(targetIds, targetLocales, Locales);

  // Here are the dimensions of our grid of locales.
  const tl1 = targetIds.dim(1).length,
        tl2 = targetIds.dim(2).length;
  if printParams && printStats then
    writeln("target locales ", tl1, " x ", tl2);

  // Create individual dimension descriptors
  const
    // block-cyclic for 1st dimension
    bdim1 = new BlockCyclicDim(lowIdx=1, blockSize=blkSize, numLocales=tl1),
    // replicated for 1st dimension
    rdim1 = new ReplicatedDim(tl1),

    // block-cyclic for 2nd dimension
    bdim2 = new BlockCyclicDim(lowIdx=1, blockSize=blkSize, numLocales=tl2),
    // replicated for 2nd dimension
    rdim2 = new ReplicatedDim(tl2);

  //
  // MatVectSpace is a 2D domain of type indexType that represents the
  // n x n matrix adjacent to the column vector b.  MatrixSpace is a
  // subdomain that is created by slicing into MatVectSpace,
  // inheriting all of its rows and its low column bound.  As our
  // standard distribution library is filled out, MatVectSpace will be
  // distributed using a BlockCyclic(blkSize) distribution.
  //
  // We use 'AbD' instead of 'MatVectSpace' throughout.
  //
  const AbD: domain(2, indexType)
          dmapped DimensionalDist2D(targetLocales, bdim1, bdim2, "dim")
          = {1..n, 1..n+1},
        MatrixSpace = AbD[.., ..n];

  var Ab : [AbD] elemType,           // the matrix A and vector b
      piv: [1..n] indexType;         // a vector of pivot values

  //
  // Create the 1-d replicated arrays for schurComplement().
  //
  const
    replAD = {1..n, 1..blkSize}
      dmapped DimensionalDist2D(targetLocales, bdim1, rdim2, "distBR"),
    replBD = {1..blkSize, 1..n+1}
      dmapped DimensionalDist2D(targetLocales, rdim1, bdim2, "distRB");

  var replA: [replAD] elemType,
      replB: [replBD] elemType;

  const
    replKD = {0..0, 1..n+1}
      dmapped new dmap(replBD._value.dist);
//    dmapped DimensionalDist2D(targetLocales, rdim1, bdim2, "distRB-replK");

  var replK: [replKD] elemType;

  initAB();

  const startTime = getCurrentTime();     // capture the start time

  LUFactorize(n, piv);                 // compute the LU factorization

  var x = backwardSub(n);  // perform the back substitution

  const execTime = getCurrentTime() - startTime;  // store the elapsed time

  //
  // Validate the answer and print the results
  const validAnswer = verifyResults(x);
  printResults(validAnswer, execTime);

//
// blocked LU factorization with pivoting for matrix augmented with
// vector of RHS values.
//
proc LUFactorize(n: indexType,
                piv: [1..n] indexType) {

  // Initialize the pivot vector to represent the initially unpivoted matrix.
  piv = 1..n;

  /* The following diagram illustrates how we partition the matrix.
     Each iteration of the loop increments a variable blk by blkSize;
     point (blk, blk) is the upper-left location of the currently
     unfactored matrix (the dotted region represents the areas
     factored in prior iterations).  The unfactored matrix is
     partioned into four subdomains: tl, tr, bl, and br, and an
     additional domain (not shown), l, that is the union of tl and bl.

                (point blk, blk)
       +-------//------------------+
       |......//...................|
       |.....//....................|
       |....+-----+----------------|
       |....|     |                |
       |....| tl  |      tr        |
       |....|     |                |
       |....+-----+----------------|
       |....|     |                |
       |....|     |                |
       |....| bl  |      br        |
       |....|     |                |
       |....|     |                |
       +----+-----+----------------+
  */
  for blk in 1..n by blkSize {
    if printStats then writeln("processing block ", blk);

    const tl = AbD[blk..#blkSize, blk..#blkSize],
          tr = AbD[blk..#blkSize, blk+blkSize..],
          bl = AbD[blk+blkSize.., blk..#blkSize],
          br = AbD[blk+blkSize.., blk+blkSize..],
          l  = AbD[blk.., blk..#blkSize];

    //
    // Now that we've sliced and diced Ab properly, do the blocked-LU
    // computation:
    //
    panelSolve(l, piv);
    updateBlockRow(tl, tr);

    //
    // update trailing submatrix (if any)
    //
    schurComplement(blk, bl, tr, br);
  }
}

//
// Distributed matrix-multiply for HPL. The idea behind this algorithm is that
// some point the matrix will be partioned as shown in the following diagram:
//
//    [1]----+-----+-----+-----+
//     |     |bbbbb|bbbbb|bbbbb|  Solve for the dotted region by
//     |     |bbbbb|bbbbb|bbbbb|  multiplying the 'a' and 'b' region.
//     |     |bbbbb|bbbbb|bbbbb|  The 'a' region is a block column, the
//     +----[2]----+-----+-----+  'b' region is a block row.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|  The 'a' region was 'bl' in the calling
//     |aaaaa|.....|.....|.....|  function but called AD here.  Similarly,
//     +-----+-----+-----+-----+  'b' was 'tr' in the calling code, but BD
//     |aaaaa|.....|.....|.....|  here.
//     |aaaaa|.....|.....|.....|
//     |aaaaa|.....|.....|.....|
//     +-----+-----+-----+-----+
//
// Every locale with a block of data in the dotted region updates
// itself by multiplying the neighboring a-region block to its left
// with the neighboring b-region block above it and subtracting its
// current data from the result of this multiplication. To ensure that
// all locales have local copies of the data needed to perform this
// multiplication we copy the data A and B data into the replA and
// replB arrays, which will use a dimensional (block-cyclic,
// replicated-block) distribution (or vice-versa) to ensure that every
// locale only stores one copy of each block it requires for all of
// its rows/columns.
//
proc schurComplement(blk, AD: domain, BD: domain, Rest: domain) {

  // Prevent replication of unequal-sized slices
  if Rest.numIndices == 0 then return;

  //
  // Copy data into replicated arrays so every processor has a local copy
  // of the data it will need to perform a local matrix-multiply.
  //

  const AbSlice1 => Ab[1..n, AD.dim(2)],
        AbSlice2 => Ab[BD.dim(1), 1..n+1];

  forall (ab, ra) in (AbSlice1, replA) do
    local
      ra = ab;
  replicateA(blk);

  forall (ab, rb) in (AbSlice2, replB) do
    local
      rb = ab;
  replicateB(blk);

  // do local matrix-multiply on a block-by-block basis
  forall (row,col) in Rest by (blkSize, blkSize) {
    // localize Rest explicitly as a workaround;
    // also hoist the innerRange computation
    const outterRange = Rest.dim(1)(row..#blkSize),
          innerRange  = Rest.dim(2)(col..#blkSize);

    local {
      for a in outterRange do
        for w in 1..blkSize do
          for b in innerRange do
            Ab[a,b] -= replA[a,w] * replB[w,b];
    }
  }
}

//
// do unblocked-LU decomposition within the specified panel, update the
// pivot vector accordingly
//
proc panelSolve(
               panel: domain,
               piv: [] indexType) {

  const blk = panel.dim(1).low;
  const cornerLocale = Ab[blk,blk].locale;

  for k in panel.dim(2) {             // iterate through the columns
    const col = panel[k.., k..k];

    // If there are no rows below the current column return
    if col.numIndices == 0 then return;

    // Find the pivot, the element with the largest absolute value.
    const (_, (pivotRow, _)) = maxloc reduce(abs(Ab(col)), col);

    // Capture the pivot value explicitly (note that result of maxloc
    // is absolute value, so it can't be used directly).
    //
    const pivotVal = Ab[pivotRow, k];

    // Swap the current row with the pivot row and update the pivot vector
    // to reflect that
    Ab[k..k, ..] <=> Ab[pivotRow..pivotRow, ..];
    piv[k] <=> piv[pivotRow];

    if (pivotVal == 0) then
      halt("Matrix cannot be factorized");

    // divide all values below and in the same col as the pivot by
    // the pivot value
    Ab[k+1.., k..k] /= pivotVal;

    // update all other values below the pivot
    // (a) replicate
    on cornerLocale {
      local
        for j in panel.dim(2)[k+1..] do
          replK[0,j] = Ab[k,j];
      replicateK(blk);
    }
    // (b) compute
    forall (i,j) in panel[k+1.., k+1..] do
      local
        Ab[i,j] -= Ab[i,k] * replK[0,j];
  }
}

//
// Update the block row (tr for top-right) portion of the matrix in a
// blocked LU decomposition.  Each step of the LU decomposition will
// solve a block (tl for top-left) portion of a matrix. This function
// solves the rows to the right of the block.
//
proc updateBlockRow(
                   tl: domain,
                   tr: domain) {

  for row in tr.dim(1) {
    const activeRow = tr[row..row, ..],
          prevRows = tr.dim(1).low..row-1;

    forall (i,j) in activeRow do
      for k in prevRows do
        Ab[i, j] -= Ab[i, k] * Ab[k,j];
  }
}

//
// compute the backwards substitution
//
proc backwardSub(n: indexType) {
  const bd = Ab.domain.dim(1);  // or simply 1..n
  var x: [bd] elemType;

  for i in bd by -1 do
    x[i] = (Ab[i,n+1] - (+ reduce [j in i+1..bd.high] (Ab[i,j] * x[j])))
            / Ab[i,i];

  return x;
}

//
// print out the problem size and block size if requested
//
proc printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks();
    printProblemSize(elemType, numMatrices, n, rank=2);
    writeln("block size = ", blkSize, "\n");
  }
}

//
// construct an n by n+1 matrix filled with random values and scale
// it to be in the range -1.0..1.0
//
proc initAB() {
  fillRandom(Ab, seed);
  Ab = Ab * 2.0 - 1.0;
}

//
// calculate norms and residuals to verify the results
//
proc verifyResults(x) {
  initAB();

  const axmbNorm = norm(gaxpyMinus(Ab[.., 1..n], x, Ab[.., n+1..n+1]), normType.normInf);

  const a1norm   = norm(Ab[.., 1..n], normType.norm1),
        aInfNorm = norm(Ab[.., 1..n], normType.normInf),
        x1Norm   = norm(Ab[.., n+1..n+1], normType.norm1),
        xInfNorm = norm(Ab[.., n+1..n+1], normType.normInf);

  const resid1 = axmbNorm / (epsilon * a1norm * n),
        resid2 = axmbNorm / (epsilon * a1norm * x1Norm),
        resid3 = axmbNorm / (epsilon * aInfNorm * xInfNorm);

  if (printStats) {
    writeln("resid1: ", resid1);
    writeln("resid2: ", resid2);
    writeln("resid3: ", resid3);
  }

  return max(resid1, resid2, resid3) < 16.0;
}

/////////////////////////////////////////////////////////////////////////////

proc replicateA(abIx) {
    const fromLocId2 = targetLocalesIndexForAbIndex(2, abIx);
    coforall lid1 in 0..#tl1 do
      on targetLocales(lid1, fromLocId2) do
        coforall lid2 in 0..#tl2 do
          if lid2 != fromLocId2 then
            replA._value.localAdescs[lid1,lid2].myStorageArr =
              replA._value.localAdescs[lid1,fromLocId2].myStorageArr;
}

proc replicateB(abIx) {
    const fromLocId1 = targetLocalesIndexForAbIndex(1, abIx);
    coforall lid2 in 0..#tl2 do
      on targetLocales(fromLocId1, lid2) do
        coforall lid1 in 0..#tl1 do
          if lid1 != fromLocId1 then
            replB._value.localAdescs[lid1,lid2].myStorageArr =
              replB._value.localAdescs[fromLocId1,lid2].myStorageArr;
}

// replicate only along one column of processors,
// starting on a good locale
proc replicateK(abIx) {
    const fromLocId1 = targetLocalesIndexForAbIndex(1, abIx);
    const fromLocId2 = targetLocalesIndexForAbIndex(2, abIx);
    // verify that we are on a good locale, optionally
    if boundsChecking {
      const fromLocale = targetLocales(fromLocId1, fromLocId2);
      if here != fromLocale then
        halt("replicateK(abIx=", abIx, ")  executing on ",
             here.id, "  expecting ", fromLocale.id, " ",
             (fromLocId1, fromLocId2));
    }
    const lid2 = fromLocId2;
        coforall lid1 in 0..#tl1 do
          if lid1 != fromLocId1 then
            replK._value.localAdescs[lid1,lid2].myStorageArr =
              replK._value.localAdescs[fromLocId1,lid2].myStorageArr;
}

proc targetLocalesIndexForAbIndex(param dim, abIx)
  return (abIx / blkSize) % (if dim == 1 then tl1 else tl2);

/////////////////////////////////////////////////////////////////////////////

//
// print success/failure, the execution time and the Gflop/s value
//
proc printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    const GflopPerSec = ((2.0/3.0) * n**3 + (3.0/2.0) * n**2) / execTime * 10e-9;
    writeln("Performance (Gflop/s) = ", GflopPerSec);
  }
}

//
// simple matrix-vector multiplication, solve equation A*x-y
//
proc gaxpyMinus(A: [],
                x: [?xD],
                y: [?yD]) {
  var res: [1..n] elemType;

  forall i in 1..n do
    // 'serial' to avoid spawning more tasks
    serial(true) { res[i] = (+ reduce [j in xD] (A[i,j] * x[j])) - y[i,n+1]; }

  return res;
}