A few more comment changes to stream-ep and removed mention of maxChunks since
it doesn't provide complete control over the tasks per locale.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@16390 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Embarassingly Parallel Implementation of STREAM Triad
//
// This version of the stream benchmark is not as elegant as
// stream.chpl.  It is a per-locale code with no communication in the
// actual computation.  It highlights the ability of a Chapel
// programmer to escape the global-view programming model and write
// codes with a fragmented, per-locale model.
//
// Comments marked with '***' point out differences with the global
// version of this benchmark in stream.chpl.
//

//
// *** Since this benchmark is written on a per-locale basis, there is no
// *** need to use the Block distribution.  The following use omits
// *** BlockDist, included in stream.chpl.
//
// Use standard modules for Timing routines, Type utility functions,
// and Random numbers
//
use Time, Types, Random;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of vectors and element type of those vectors
//
const numVectors = 3;
type elemType = real(64);

//
// *** To ensure a local problem size, we spoof the number of vectors
// *** passed to the computeProblemSize function to be the number of
// *** vectors times the number of locales.
//
// Configuration constants to set the problem size (m) and the scalar
// multiplier, alpha
//
config const m = computeProblemSize(numVectors*numLocales, elemType),
             alpha = 3.0;

//
// Configuration constants to set the number of trials to run and the
// amount of error to permit in the verification
//
config const numTrials = 10,
             epsilon = 0.0;

//
// *** There isn't (yet) a way to set the number of tasks to use for
// *** implementing a forall loop over a default array.  When there is
// *** such a way, we will want to set it via this configuration
// *** constant to get functionality like we have with stream.chpl.
//
// config const tasksPerLocale = here.numCores;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

//
// *** To ensure determinism of output, there is no more printing of the
// *** arrays in initVectors and verifyResults.
//
//
// Configuration constants to control what's printed -- benchmark
// parameters and/or statistics
//
config const printParams = true,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // *** Aggregates for collecting per-locale results for the minimum
  // *** execution time per trial, and whether verification passed
  //
  var minTimes: [LocaleSpace] real;
  var validAnswers: [LocaleSpace] bool;

  //
  // *** Fragment control so that we have a single task running on
  // *** every locale.
  //
  coforall loc in Locales do on loc {

    //
    // *** We declare these variables outside of the local block since
    // *** we'll need to access them when we write back to the global
    // *** aggregates declared above.
    //
    var validAnswer: bool;
    var execTime: [1..numTrials] real;

    //
    // *** Indicates that all of the code in this block is local to
    // *** this locale.  There is no communication.  A violation will
    // *** result in an error, though error checking is disabled with
    // *** --fast or --no-checks.
    //
    local {

      //
      // *** A, B, and C are the three local vectors
      //
      var A, B, C: [1..m] elemType;

      initVectors(B, C);    // Initialize the input vectors, B and C

      for trial in 1..numTrials {                        // loop over the trials
        const startTime = getCurrentTime();              // capture the start time

        //
        // *** The main loop looks identical to stream.chpl.  However,
        // *** in this version we are iterating over arrays that are
        // *** not distributed.
        //
        forall (a, b, c) in (A, B, C) do
          a = b + alpha * c;

        execTime(trial) = getCurrentTime() - startTime;  // store the elapsed time
      }

      validAnswer = verifyResults(A, B, C);              // verify...
    }

    //
    // *** Write times and verification result into aggregates
    // *** declared above.  These are declared over LocaleSpace so we
    // *** can write to them in parallel.
    //
    minTimes[here.id] = min reduce execTime;
    validAnswers[here.id] = validAnswer;
  }

  //
  // *** Pass minimum, average, and maximum times to printResults
  //
  printResults(&& reduce validAnswers, minTimes);
}

//
// Print the problem size and number of trials
//
def printConfiguration() {
  if (printParams) {
    //
    // *** Here we multiply m by the number of locales so that we can
    // *** print out the global problem size.
    //
    printProblemSize(elemType, numVectors, m * numLocales);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

//
// *** Both initVectors and verifyResults are almost identical to
// *** stream.chpl even though they are called with arrays that are
// *** not distributed.  For initialization, the same random stream is
// *** used on each locale.  In the global version, a single logical
// *** stream of random numbers is used across all of the locales.
// ***
// *** In this version, we've omitted a way to print the arrays.  This
// *** ensures determinism of output.  Printing the arrays also
// *** violates the locality constraint imposed by the local block
// *** from which these functions are called.
//
// Initialize vectors B and C using a random stream of values
//
def initVectors(B, C) {
  var randlist = new RandomStream(seed);

  randlist.fillRandom(B);
  randlist.fillRandom(C);

  delete randlist;
}

//
// Verify that the computation is correct
//
def verifyResults(A, B, C) {

  //
  // recompute the computation, destructively storing into B to save space
  //
  forall (b, c) in (B, C) do
    b += alpha *c;

  //
  // Compute the infinity-norm by computing the maximum reduction of the
  // absolute value of A's elements minus the new result computed in B.
  // "[i in I]" represents an expression-level loop: "forall i in I"
  //
  const infNorm = max reduce [(a,b) in (A,B)] abs(a - b);

  return (infNorm <= epsilon);    // return whether the error is acceptable
}

//
// Print out success/failure, the timings, and the GB/s value.
//
// *** Here we report maximum, average, and minimum times instead of
// *** total, average, and minimum.
//
def printResults(successful, minTimes) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    const maxTime = max reduce minTimes,
          avgTime = + reduce minTimes / numLocales,
          minTime = min reduce minTimes;
    writeln("Execution time:");
    writeln("  max = ", maxTime);
    writeln("  avg = ", avgTime);
    writeln("  min = ", minTime);

    const GBPerSec = numVectors * numBytes(elemType) * (m / minTime) * 1e-9;
    writeln("Performance (GB/s) = ", GBPerSec);
  }
}

//
// Embarassingly Parallel Implementation of STREAM Triad
//
// This version of the stream benchmark is not as elegant as
// stream.chpl.  It is a per-locale code with no communication in the
// actual computation.  It highlights the ability of a Chapel
// programmer to escape the global-view programming model and write
// codes with a fragmented, per-locale model.
//
// Comments marked with '***' point out differences with the global
// version of this benchmark in stream.chpl.
//

//
// *** Since this benchmark is written on a per-locale basis, there is no
// *** need to use the Block distribution.  The following use omits
// *** BlockDist, included in stream.chpl.
//
// Use standard modules for Timing routines, Type utility functions,
// and Random numbers
//
use Time, Types, Random;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

//
// The number of vectors and element type of those vectors
//
const numVectors = 3;
type elemType = real(64);

//
// *** To ensure a local problem size, we spoof the number of vectors
// *** passed to the computeProblemSize function to be the number of
// *** vectors times the number of locales.
//
// Configuration constants to set the problem size (m) and the scalar
// multiplier, alpha
//
config const m = computeProblemSize(numVectors*numLocales, elemType),
             alpha = 3.0;

//
// Configuration constants to set the number of trials to run and the
// amount of error to permit in the verification
//
config const numTrials = 10,
             epsilon = 0.0;

//
// *** Use the global config constant maxChunks to set the number of
// *** tasks per locale since this value is used to control the number
// *** of tasks that can implement a forall loop over a default
// *** arithmetic array.
//
// config const tasksPerLocale = here.numCores;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

//
// *** To ensure determinism of output, there is no more printing of the
// *** arrays in initVectors and verifyResults.
//
//
// Configuration constants to control what's printed -- benchmark
// parameters and/or statistics
//
config const printParams = true,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();   // print the problem size, number of trials, etc.

  //
  // *** Aggregates for collecting per-locale results for the minimum
  // *** execution time per trial, and whether verification passed
  //
  var minTimes: [LocaleSpace] real;
  var validAnswers: [LocaleSpace] bool;

  //
  // *** Fragment control so that we have a single task running on
  // *** every locale.
  //
  coforall loc in Locales do on loc {

    //
    // *** We declare these variables outside of the local block since
    // *** we'll need to access them when we write back to the global
    // *** aggregates declared above.
    //
    var validAnswer: bool;
    var execTime: [1..numTrials] real;

    //
    // *** Indicates that all of the code in this block is local to
    // *** this locale.  There is no communication.  A violation will
    // *** result in an error, though error checking is disabled with
    // *** --fast or --no-checks.
    //
    local {

      //
      // *** A, B, and C are the three local vectors
      //
      var A, B, C: [1..m] elemType;

      initVectors(B, C);    // Initialize the input vectors, B and C

      for trial in 1..numTrials {                        // loop over the trials
        const startTime = getCurrentTime();              // capture the start time

        //
        // *** The main loop looks identical to stream.chpl.  However,
        // *** in this version we are iterating over arrays that are
        // *** not distributed.
        //
        forall (a, b, c) in (A, B, C) do
          a = b + alpha * c;

        execTime(trial) = getCurrentTime() - startTime;  // store the elapsed time
      }

      validAnswer = verifyResults(A, B, C);              // verify...
    }

    //
    // *** Write times and verification result into aggregates
    // *** declared above.  These are declared over LocaleSpace so we
    // *** can write to them in parallel.
    //
    minTimes[here.id] = min reduce execTime;
    validAnswers[here.id] = validAnswer;
  }

  //
  // *** Pass minimum, average, and maximum times to printResults
  //
  printResults(&& reduce validAnswers, minTimes);
}

//
// Print the problem size and number of trials
//
def printConfiguration() {
  if (printParams) {
    //
    // *** Here we multiple m by the number of locales so that we can
    // *** print out the global problem size.  We print out the value
    // *** of the global configuration constant maxChunks as tasks per
    // *** locale.
    //
    if (printStats) then printLocalesTasks(maxChunks);
    printProblemSize(elemType, numVectors, m * numLocales);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

//
// *** Both initVectors and verifyResults are almost identical to
// *** stream.chpl even though they are called with arrays that are
// *** not distributed.  The only difference is that the arrays are
// *** not printed.  This ensures determinism of output.  Printing the
// *** arrays also violates the locality constraint imposed by the
// *** local block from which these functions are called.
//
//
// Initialize vectors B and C using a random stream of values
//
def initVectors(B, C) {
  var randlist = new RandomStream(seed);

  randlist.fillRandom(B);
  randlist.fillRandom(C);

  delete randlist;
}

//
// Verify that the computation is correct
//
def verifyResults(A, B, C) {

  //
  // recompute the computation, destructively storing into B to save space
  //
  forall (b, c) in (B, C) do
    b += alpha *c;

  //
  // Compute the infinity-norm by computing the maximum reduction of the
  // absolute value of A's elements minus the new result computed in B.
  // "[i in I]" represents an expression-level loop: "forall i in I"
  //
  const infNorm = max reduce [(a,b) in (A,B)] abs(a - b);

  return (infNorm <= epsilon);    // return whether the error is acceptable
}

//
// Print out success/failure, the timings, and the GB/s value.
//
// *** Here we report maximum, average, and minimum times instead of
// *** total, average, and minimum.
//
def printResults(successful, minTimes) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    const maxTime = max reduce minTimes,
          avgTime = + reduce minTimes / numLocales,
          minTime = min reduce minTimes;
    writeln("Execution time:");
    writeln("  max = ", maxTime);
    writeln("  avg = ", avgTime);
    writeln("  min = ", minTime);

    const GBPerSec = numVectors * numBytes(elemType) * (m / minTime) * 1e-9;
    writeln("Performance (GB/s) = ", GBPerSec);
  }
}


Add an error message for trying to send sparsely packed numpy arrays to Chapel

With the arrays view work, we ignore the blk field, as no Chapel arrays will
normally be created with it having meaning.  However, in PyChapel we create
unusual arrays - numpy can give us an array to translate which is not densely
packed and without the blk field, we have no way of understanding that array.

For now, add an error message to pyChapel when a user tries to send Chapel such
an array.  Next commit, I will lock this in with some tests and update the
failing case that motivated this investigation.

// Wrapper around the pych_array c-struct
// to access NumPy arrays provided by the PyChapel
// Python module
use Random;

param PYCH_MAXDIM: int(32)= 16;

extern record pych_array {
    var two:int(32);
    var nd:int(32);
    var typekind:int(8);
    var itemsize:int(32);
    var flags:int(32);
    var shape:PYCH_MAXDIM*int(32);
    var strides:PYCH_MAXDIM*int(32);
    var ptr_d:_ddata(real);
}

proc pprint_pych_array(arr: pych_array) {
    writeln("[");
    writeln("  two=", arr.two, ",");
    writeln("  nd=", arr.nd, ",");
    writeln("  typekind=", arr.typekind, ",");
    writeln("  itemsize=", arr.itemsize, ",");

    write("  shape=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.shape(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    write("  stride=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.strides(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    writeln("  ptr_d=", arr.ptr_d);
    writeln("]");
}

proc rangify(shape) where isTuple(shape) {
    proc tail(first, remainder...) {
        return remainder;
    }
    if (shape.size == 1) {
        return (0..shape(1):int(64),);
    } else {
        return ((0..shape(1):int(64)), (...rangify(tail((...shape)))));
    }
}

proc pych_to_chpl1D(arr: pych_array) {
    pragma "no auto destroy"
    var dom = {0..(arr.shape(1):int(64)-1)};

    var stride = (1,);
    var block = (1,);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    dom._value.add_arr(ret);

    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        //ret.str(i) = (arr.strides(i):int(64) / (arr.itemsize));
        ret.str(i) = 1;
    }
    //writeln("Strides");
    //for str in ret.str {
    //    writeln(str);
    //}
    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        // The block seems to be what NumPy call strides.
      if (arr.strides(i):int(64) != arr.itemsize) then
        halt("pyChapel does not support sparsely packed ndarrays");
      ret.blk(i) = 1;
        //ret.blk(i) = (arr.strides(i):int(64) / arr.itemsize);
    }
    //ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();
    ret.initShiftedData();

    return ret;
}

proc pych_to_chpl2D(arr: pych_array) {
    pragma "no auto destroy"
    var dom = {(...rangify(arr.shape))};

    //writeln(pych_to_chplT(arr));
    var stride = (1, 1);
    var block = (3, 1);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    dom._value.add_arr(ret);

    // rank*int tuples

    // stride for each dimension
    // you might be able to get this out of the domain instead.
    //ret.str = stride;

    for i in 1..arr.nd {
        ret.str(i) = arr.strides(i):int(64);
    }

    // I'm not totally sure what this one is. I think it's the size of a row of data.
    // You can test this by writing code like:
    //var x = [1..10, 10..20]: int;
    //writeln("What is this?");

    ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();

    // if you're having segfaults, look into this method. You can also try
    // compiling with '-searlyShiftData=false' to disable this method.
    ret.initShiftedData();

    // only returns the underlying class, still need to turn it into a
    // first-class chapel array. I'm not sure why exactly, but something
    // about the way we return arrays here performs a copy and changes
    // made to the array aren't reflected in the c pointer.
    return ret;
}

// Wrapper around the pych_array c-struct
// to access NumPy arrays provided by the PyChapel
// Python module
use Random;

param PYCH_MAXDIM: int(32)= 16;

extern record pych_array {
    var two:int(32);
    var nd:int(32);
    var typekind:int(8);
    var itemsize:int(32);
    var flags:int(32);
    var shape:PYCH_MAXDIM*int(32);
    var strides:PYCH_MAXDIM*int(32);
    var ptr_d:_ddata(real);
}

proc pprint_pych_array(arr: pych_array) {
    writeln("[");
    writeln("  two=", arr.two, ",");
    writeln("  nd=", arr.nd, ",");
    writeln("  typekind=", arr.typekind, ",");
    writeln("  itemsize=", arr.itemsize, ",");

    write("  shape=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.shape(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    write("  stride=");
    for idx in 1..PYCH_MAXDIM {
        write(arr.strides(idx));
        if (PYCH_MAXDIM != idx) {
            write(",");
        }
    }
    writeln("");
    writeln("  ptr_d=", arr.ptr_d);
    writeln("]");
}

proc rangify(shape) where isTuple(shape) {
    proc tail(first, remainder...) {
        return remainder;
    }
    if (shape.size == 1) {
        return (0..shape(1):int(64),);
    } else {
        return ((0..shape(1):int(64)), (...rangify(tail((...shape)))));
    }
}

proc pych_to_chpl1D(arr: pych_array) {
    pragma "no auto destroy"
    var dom = {0..(arr.shape(1):int(64)-1)};

    var stride = (1,);
    var block = (1,);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    dom._value.add_arr(ret);

    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        //ret.str(i) = (arr.strides(i):int(64) / (arr.itemsize));
        ret.str(i) = 1;
    }
    //writeln("Strides");
    //for str in ret.str {
    //    writeln(str);
    //}
    for i in 1..arr.nd {
        // TODO: Is this conversion from bytes to elements correct?
        // The block seems to be what NumPy call strides.
        ret.blk(i) = (arr.strides(i):int(64) / arr.itemsize);
    }
    //ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();
    ret.initShiftedData();

    return ret;
}

proc pych_to_chpl2D(arr: pych_array) {
    pragma "no auto destroy"
    var dom = {(...rangify(arr.shape))};

    //writeln(pych_to_chplT(arr));
    var stride = (1, 1);
    var block = (3, 1);

    var ret = new DefaultRectangularArr(
        eltType = real,                 // Element type
        rank = dom.rank,                // Rank
        idxType = dom.idxType,          // Index type of domain
        stridable = dom.stridable,      // Boolean, can we stride?
        dom = dom._value,               // The underlying class implementation
        noinit_data = true              // Prevent overwriting your data
    );

    dom._value.add_arr(ret);

    // rank*int tuples

    // stride for each dimension
    // you might be able to get this out of the domain instead.
    //ret.str = stride;

    for i in 1..arr.nd {
        ret.str(i) = arr.strides(i):int(64);
    }

    // I'm not totally sure what this one is. I think it's the size of a row of data.
    // You can test this by writing code like:
    //var x = [1..10, 10..20]: int;
    //writeln("What is this?");

    ret.blk = block;

    // get offsets from domain
    for param i in 1..dom.rank {
        ret.off(i) = dom._value.dsiDim(i).low;
    }

    ret.data = arr.ptr_d;
    ret.origin = 0; // I believe 'origin' is the c ptr offset
    ret.computeFactoredOffs();

    // if you're having segfaults, look into this method. You can also try
    // compiling with '-searlyShiftData=false' to disable this method.
    ret.initShiftedData();

    // only returns the underlying class, still need to turn it into a
    // first-class chapel array. I'm not sure why exactly, but something
    // about the way we return arrays here performs a copy and changes
    // made to the array aren't reflected in the c pointer.
    return ret;
}