Dial back problem size to keep test runtime < 5 minutes.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@22536 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
 * File I/O Primer
 */

/*
   First example: Textual Array I/O
   Second and Third examples: Binary I/O
   Fourth example: reading UTF-8 lines and printing them out.
   Fifth example: error handling.
   Sixth example: object-at-a-time writing.
   Seventh example: binary I/O with bits at a time
*/

config var n = 9,            // the problem size for example 1
           filename = "Arr.dat";  // the filename for writing/reading the array
config const num = 128*1024;
config const example = 0;
config const testfile = "test.bin";

use IO;

/* Example 1
 * This is a simple example of using file I/O in Chapel.
 *
 * It initializes an array and writes its size and data to a file.  It
 * then opens the file, uses the size in the file to declare a new
 * domain and array, and reads in the array data.
 */
if example == 0 || example == 1 {
  const ADom = {1..n, 1..n};  // Create a domain of the specified problem size

  // Create and initialize an array of the specified size
  var A: [ADom] real = [(i,j) in ADom] i + j/10.0;

  // Write the problem size and array out to the specified filename
  writeSquareArray(n, A, filename);

  // Read an array in from the specified filename, storing in a new variable, B
  var B = readArray(filename);

  // Print out B as a debugging step
  writeln("B is:\n", B);

  //
  // verify that A and B contain the same values and print success or failure
  //
  const numErrors = + reduce [i in ADom] (A(i) != B(i));

  if (numErrors > 0) {
    writeln("FAILURE");
  } else {
    writeln("SUCCESS");
  }
}

//
// this procedure writes a square array out to a file
//
proc writeSquareArray(n, X, filename) {
  // Create and open an output file with the specified filename in write mode
  var outfile = open(filename, iomode.cw);
  var writer = outfile.writer();

  // Write the problem size in each dimension to the file
  writer.writeln(n, " ", n);

  // write out the array itself
  writer.write(X);

  // close the file
  writer.close();
  outfile.close();
}

//
// This procedure reads a new array out of a file and returns it
//
proc readArray(filename) {
   // Open an input file with the specified filename in read mode
  var infile = open(filename, iomode.r);
  var reader = infile.reader();

  // Read the number of rows and columns in the array in from the file
  var m = reader.read(int),
      n = reader.read(int);

  // Declare an array of the specified dimensions
  var X: [1..m, 1..n] real;

  //
  // Read in the array elements one by one (eventually, you should be
  // able to read in the array wholesale, but this isn't currently
  // supported.
  //
  for i in 1..m do
    for j in 1..n do
      reader.read(X(i,j));

  // Close the file
  reader.close();
  infile.close();

  // Return the array
  return X;
}

/*
   In Examples 2 and 3, we will write numbers 0..n-1 to a file in binary,
   and then we'll open it up, and read the numbers in reverse,
   just to show how to 'seek' in a file.

   We show two versions of this example; a simple version, and
   a slightly more complicated version that has some performance hints.
*/

if example == 0 || example == 2 {
  writeln("Running Example 2");

  // Here comes the simple version!

  // First, open up a test file. Chapel's I/O interface allows
  // us to open regular files, temporary files, memory, or file descriptors;
  var f = open(testfile, iomode.cwr);

  /* Since the typical 'file position' design leads to race conditions
   * all over, the Chapel I/O design separates a file from a channel.
   * A channel is a buffer to a particular spot in a file. Channels
   * can have a start and and end, so that if you're doing parallel I/O
   * to different parts of a file with different channels, you can
   * partition the file to be assured that they do not interfere.
   */

  {
    var w = f.writer(kind=ionative); // get a binary writing channel for the start of the file.

    for i in 0..#num { // for each i in [0,n)
      var tmp:uint(64) = i:uint(64);
      w.write(tmp); // writing a uint(64) will write 8 bytes.
    }

    // Now w goes out of scope, it will flush anything that is buffered.
    // Channels are reference-counted, so it should be easy, but if you want to make sure,
    // you can close the channel.
    w.close();
  }

  // OK, now we have written our data file. Now read it backwards.
  // Note -- this could be a forall loop to do I/O in parallel!
  for i in 0..#num by -1 {
    var r = f.reader(kind=ionative, start=8*i, end=8*i+8);
    var tmp:uint(64);
    r.read(tmp);
    assert(tmp == i:uint(64));
    r.close();
  }

  // Now close the file. Like channels, files are reference-counted,
  // and so should close when they go out of scope, but we close it
  // here anyway to be sure.
  f.close();

  // Finally, we can remove the test file.
  unlink(testfile);
}

if example == 0 || example == 3 {
  writeln("Running Example 3");

  /* Here comes a faster version of example 2, using some hints.
     This time we're not going to use a temporary file, because
     we want to open it twice for performance reasons.
     If you want to measure the performance difference, try:
       time ./fielIOv2 --example=2
       time ./fielIOv2 --example=3
   */

  // First, open up a file and write to it.
  {
    var f = open(testfile, iomode.cwr);
    /* When we create the writer, suppling locking=false will do unlocked I/O.
       That's fine as long as the channel is not shared between tasks.
      */
    var w = f.writer(kind=ionative, locking=false);

    for i in 0..#num {
      var tmp:uint(64) = i:uint(64);
      w.write(tmp);
    }

    w.close();
    f.close();
  }

  /* Now that we've created the file, when we open it for
     read access and hint 'random access' and 'keep data cached/assume data is cached',
     we can optimize better (using mmap, if you like details).
   */
  {
    var f = open(testfile, iomode.r,
                 hints=IOHINT_RANDOM|IOHINT_CACHED|IOHINT_PARALLEL);

    // Note -- this could be a forall loop to do I/O in parallel!
    forall i in 0..#num by -1 {
      /* When we create the reader, suppling locking=false will do unlocked I/O.
         That's fine as long as the channel is not shared between tasks;
         here it's just used as a local variable, so we are O.K.
        */
      var r = f.reader(kind=ionative, locking=false, start=8*i, end=8*i+8);
      var tmp:uint(64);
      r.read(tmp);
      assert(tmp == i:uint(64));
      r.close();
    }

    f.close();
  }

  // Finally, we can remove the test file.
  unlink(testfile);
}

/*
   In Example 4, we show that it's easy to read the lines in a file,
   including UTF-8 characters.
*/
if example == 0 || example == 4 {
  writeln("Running Example 4");

  var f = open(testfile, iomode.cwr);
  var w = f.writer();

  w.writeln("Hello");
  w.writeln("This");
  w.writeln(" is ");
  w.writeln(" a test ");
  // We only write the UTF-8 characters if unicode is supported,
  // and that depends on the current unix locale environment
  // (e.g. setting the environment variable LC_ALL=C will disable unicode support).
  // Note that since UTF-8 strings are C strings, this should work
  // even in a C locale. We don't do it all the time for testing sanity reasons.
  if unicodeSupported() then w.writeln(" of UTF-8 Euro Sign: €");

  // flush buffers, close the channel.
  w.close();

  var r = f.reader();
  var line:string;
  while( r.readline(line) ) {
    write("Read line: ", line);
  }
  r.close();

  // Or, if we just want all the lines in the file,
  // we can use file.lines, and we don't even have to make a reader:
  for line in f.lines() {
    write("Read line: ", line);
  }

  f.close();
  unlink(testfile);
}

/*
   In Example 5, we show that error handling is possible
   with the new I/O routines. Maybe one day this strategy will
   be replaced by exceptions, but until then... Chapel programs
   can still respond to errors.

*/
if example == 0 || example == 5 {
  writeln("Running Example 5");

  // Error handling.
  var err:syserr;
  // Who knows, maybe 1st unlink succeeds.
  unlink(testfile, error=err);

  // File does not exist by now, for sure.
  unlink(testfile, error=err);
  assert(err == ENOENT);

  // What happens if we try to open a non-existant file?
  var f = open(testfile, iomode.r, error=err);
  assert(err == ENOENT);

  /* Note that if an error= argument is not supplied to an
     I/O function, it will call ioerror, which will
     in turn halt with an error message.
   */
}

/*
   In Example 6, we demonstrate that output from multiple tasks
   can use the same channel, and each write() call will be
   completed entirely before another one is allowed to begin.
*/
if example == 0 || example == 6 {
  writeln("Running Example 6");

  forall i in 1..4 {
    writeln("This should be a chunk: {", "\n a", "\n b", "\n}");
  }

  record MyThing {
    proc writeThis(w:Writer) {
      w.writeln("This should be a chunk: {");
      w.writeln(" a");
      w.writeln(" b");
      w.writeln("}");
    }
  }

  forall i in 1..4 {
    var t:MyThing;
    write(t);
  }
}

/*
   In Example 7, we demonstrate bit-level I/O.
 */
if example == 0 || example == 7 {
  writeln("Running Example 7");

  var f = open(testfile, iomode.cwr);

  {
    var w = f.writer(kind=ionative);

    // Write 011 0110 011110000
    w.writebits(0b011, 3);
    w.writebits(0b0110, 4);
    w.writebits(0b011110000, 9);
    w.close();
  }

  // Try reading it back the way we wrote it.
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(64);

    r.readbits(tmp, 3);
    assert(tmp == 0b011);

    r.readbits(tmp, 4);
    assert(tmp == 0b0110);

    r.readbits(tmp, 9);
    assert(tmp == 0b011110000);

    r.close();
  }

  // Try reading it back all as one big chunk.
  // Read 01101100 11110000
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(8);

    r.read(tmp);
    assert(tmp == 0b01101100);

    r.read(tmp);
    assert(tmp == 0b11110000);

    r.close();
  }

  f.close();
}

/*
 * File I/O Primer
 */

/*
   First example: Textual Array I/O
   Second and Third examples: Binary I/O
   Fourth example: reading UTF-8 lines and printing them out.
   Fifth example: error handling.
   Sixth example: object-at-a-time writing.
   Seventh example: binary I/O with bits at a time
*/

config var n = 9,            // the problem size for example 1
           filename = "Arr.dat";  // the filename for writing/reading the array
config const num = 1*1024*1024;
config const example = 0;
config const testfile = "test.bin";

use IO;

/* Example 1
 * This is a simple example of using file I/O in Chapel.
 *
 * It initializes an array and writes its size and data to a file.  It
 * then opens the file, uses the size in the file to declare a new
 * domain and array, and reads in the array data.
 */
if example == 0 || example == 1 {
  const ADom = {1..n, 1..n};  // Create a domain of the specified problem size

  // Create and initialize an array of the specified size
  var A: [ADom] real = [(i,j) in ADom] i + j/10.0;

  // Write the problem size and array out to the specified filename
  writeSquareArray(n, A, filename);

  // Read an array in from the specified filename, storing in a new variable, B
  var B = readArray(filename);

  // Print out B as a debugging step
  writeln("B is:\n", B);

  //
  // verify that A and B contain the same values and print success or failure
  //
  const numErrors = + reduce [i in ADom] (A(i) != B(i));

  if (numErrors > 0) {
    writeln("FAILURE");
  } else {
    writeln("SUCCESS");
  }
}

//
// this procedure writes a square array out to a file
//
proc writeSquareArray(n, X, filename) {
  // Create and open an output file with the specified filename in write mode
  var outfile = open(filename, iomode.cw);
  var writer = outfile.writer();

  // Write the problem size in each dimension to the file
  writer.writeln(n, " ", n);

  // write out the array itself
  writer.write(X);

  // close the file
  writer.close();
  outfile.close();
}

//
// This procedure reads a new array out of a file and returns it
//
proc readArray(filename) {
   // Open an input file with the specified filename in read mode
  var infile = open(filename, iomode.r);
  var reader = infile.reader();

  // Read the number of rows and columns in the array in from the file
  var m = reader.read(int),
      n = reader.read(int);

  // Declare an array of the specified dimensions
  var X: [1..m, 1..n] real;

  //
  // Read in the array elements one by one (eventually, you should be
  // able to read in the array wholesale, but this isn't currently
  // supported.
  //
  for i in 1..m do
    for j in 1..n do
      reader.read(X(i,j));

  // Close the file
  reader.close();
  infile.close();

  // Return the array
  return X;
}

/*
   In Examples 2 and 3, we will write numbers 0..n-1 to a file in binary,
   and then we'll open it up, and read the numbers in reverse,
   just to show how to 'seek' in a file.

   We show two versions of this example; a simple version, and
   a slightly more complicated version that has some performance hints.
*/

if example == 0 || example == 2 {
  writeln("Running Example 2");

  // Here comes the simple version!

  // First, open up a test file. Chapel's I/O interface allows
  // us to open regular files, temporary files, memory, or file descriptors;
  var f = open(testfile, iomode.cwr);

  /* Since the typical 'file position' design leads to race conditions
   * all over, the Chapel I/O design separates a file from a channel.
   * A channel is a buffer to a particular spot in a file. Channels
   * can have a start and and end, so that if you're doing parallel I/O
   * to different parts of a file with different channels, you can
   * partition the file to be assured that they do not interfere.
   */

  {
    var w = f.writer(kind=ionative); // get a binary writing channel for the start of the file.

    for i in 0..#num { // for each i in [0,n)
      var tmp:uint(64) = i:uint(64);
      w.write(tmp); // writing a uint(64) will write 8 bytes.
    }

    // Now w goes out of scope, it will flush anything that is buffered.
    // Channels are reference-counted, so it should be easy, but if you want to make sure,
    // you can close the channel.
    w.close();
  }

  // OK, now we have written our data file. Now read it backwards.
  // Note -- this could be a forall loop to do I/O in parallel!
  for i in 0..#num by -1 {
    var r = f.reader(kind=ionative, start=8*i, end=8*i+8);
    var tmp:uint(64);
    r.read(tmp);
    assert(tmp == i:uint(64));
    r.close();
  }

  // Now close the file. Like channels, files are reference-counted,
  // and so should close when they go out of scope, but we close it
  // here anyway to be sure.
  f.close();

  // Finally, we can remove the test file.
  unlink(testfile);
}

if example == 0 || example == 3 {
  writeln("Running Example 3");

  /* Here comes a faster version of example 2, using some hints.
     This time we're not going to use a temporary file, because
     we want to open it twice for performance reasons.
     If you want to measure the performance difference, try:
       time ./fielIOv2 --example=2
       time ./fielIOv2 --example=3
   */

  // First, open up a file and write to it.
  {
    var f = open(testfile, iomode.cwr);
    /* When we create the writer, suppling locking=false will do unlocked I/O.
       That's fine as long as the channel is not shared between tasks.
      */
    var w = f.writer(kind=ionative, locking=false);

    for i in 0..#num {
      var tmp:uint(64) = i:uint(64);
      w.write(tmp);
    }

    w.close();
    f.close();
  }

  /* Now that we've created the file, when we open it for
     read access and hint 'random access' and 'keep data cached/assume data is cached',
     we can optimize better (using mmap, if you like details).
   */
  {
    var f = open(testfile, iomode.r,
                 hints=IOHINT_RANDOM|IOHINT_CACHED|IOHINT_PARALLEL);

    // Note -- this could be a forall loop to do I/O in parallel!
    forall i in 0..#num by -1 {
      /* When we create the reader, suppling locking=false will do unlocked I/O.
         That's fine as long as the channel is not shared between tasks;
         here it's just used as a local variable, so we are O.K.
        */
      var r = f.reader(kind=ionative, locking=false, start=8*i, end=8*i+8);
      var tmp:uint(64);
      r.read(tmp);
      assert(tmp == i:uint(64));
      r.close();
    }

    f.close();
  }

  // Finally, we can remove the test file.
  unlink(testfile);
}

/*
   In Example 4, we show that it's easy to read the lines in a file,
   including UTF-8 characters.
*/
if example == 0 || example == 4 {
  writeln("Running Example 4");

  var f = open(testfile, iomode.cwr);
  var w = f.writer();

  w.writeln("Hello");
  w.writeln("This");
  w.writeln(" is ");
  w.writeln(" a test ");
  // We only write the UTF-8 characters if unicode is supported,
  // and that depends on the current unix locale environment
  // (e.g. setting the environment variable LC_ALL=C will disable unicode support).
  // Note that since UTF-8 strings are C strings, this should work
  // even in a C locale. We don't do it all the time for testing sanity reasons.
  if unicodeSupported() then w.writeln(" of UTF-8 Euro Sign: €");

  // flush buffers, close the channel.
  w.close();

  var r = f.reader();
  var line:string;
  while( r.readline(line) ) {
    write("Read line: ", line);
  }
  r.close();

  // Or, if we just want all the lines in the file,
  // we can use file.lines, and we don't even have to make a reader:
  for line in f.lines() {
    write("Read line: ", line);
  }

  f.close();
  unlink(testfile);
}

/*
   In Example 5, we show that error handling is possible
   with the new I/O routines. Maybe one day this strategy will
   be replaced by exceptions, but until then... Chapel programs
   can still respond to errors.

*/
if example == 0 || example == 5 {
  writeln("Running Example 5");

  // Error handling.
  var err:syserr;
  // Who knows, maybe 1st unlink succeeds.
  unlink(testfile, error=err);

  // File does not exist by now, for sure.
  unlink(testfile, error=err);
  assert(err == ENOENT);

  // What happens if we try to open a non-existant file?
  var f = open(testfile, iomode.r, error=err);
  assert(err == ENOENT);

  /* Note that if an error= argument is not supplied to an
     I/O function, it will call ioerror, which will
     in turn halt with an error message.
   */
}

/*
   In Example 6, we demonstrate that output from multiple tasks
   can use the same channel, and each write() call will be
   completed entirely before another one is allowed to begin.
*/
if example == 0 || example == 6 {
  writeln("Running Example 6");

  forall i in 1..4 {
    writeln("This should be a chunk: {", "\n a", "\n b", "\n}");
  }

  record MyThing {
    proc writeThis(w:Writer) {
      w.writeln("This should be a chunk: {");
      w.writeln(" a");
      w.writeln(" b");
      w.writeln("}");
    }
  }

  forall i in 1..4 {
    var t:MyThing;
    write(t);
  }
}

/*
   In Example 7, we demonstrate bit-level I/O.
 */
if example == 0 || example == 7 {
  writeln("Running Example 7");

  var f = open(testfile, iomode.cwr);

  {
    var w = f.writer(kind=ionative);

    // Write 011 0110 011110000
    w.writebits(0b011, 3);
    w.writebits(0b0110, 4);
    w.writebits(0b011110000, 9);
    w.close();
  }

  // Try reading it back the way we wrote it.
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(64);

    r.readbits(tmp, 3);
    assert(tmp == 0b011);

    r.readbits(tmp, 4);
    assert(tmp == 0b0110);

    r.readbits(tmp, 9);
    assert(tmp == 0b011110000);

    r.close();
  }

  // Try reading it back all as one big chunk.
  // Read 01101100 11110000
  {
    var r = f.reader(kind=ionative);
    var tmp:uint(8);

    r.read(tmp);
    assert(tmp == 0b01101100);

    r.read(tmp);
    assert(tmp == 0b11110000);

    r.close();
  }

  f.close();
}


Add the type variable idxType as a global variable to an fft test.

This test failed because it used, but failed to define the variable idxType.
Added a definition for that variable so the test should start passing again.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@16996 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// Use standard modules for Bit operations, Random numbers, and Timing
//
use BitOps, Random, Time, BlockDist, CyclicDist;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

const radix = 4;               // the radix of this FFT implementation

const numVectors = 2;          // the number of vectors to be stored
type idxType = int(64);        // the type to use as the index to the domains
type elemType = complex(128);  // the element type of the vectors

//
// A configuration constant defining log2(problem size) -- n -- and a
// constant defining the problem size itself -- m
//
config const n = computeProblemSize(elemType, numVectors, returnLog2 = true);
const m = 2**n;

//
// The number of tasks to use per Chapel locale in parallel loops
//
config const tasksPerLocale = here.numCores;

//
// Configuration constants defining the epsilon and threshold values
// used to verify the result
//
config const epsilon = 2.0 ** -51.0,
             threshold = 16.0;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = false,
             seed = if useRandomSeed then SeedGenerator.currentTime else 314159265;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();          // print the problem size
  makeSizeAssertions();
  //
  // TwiddleDom describes the index set used to define the vector of
  // twiddle values and is a 1D domain indexed by 64-bit ints from 0
  // to m/4-1.  Twiddles is the vector of twiddle values.
  //
  //const TwiddleDist = new dmap(new Cyclic(startIdx=0:idxType, tasksPerLocale=tasksPerLocale));
  const TwiddleDist = new dmap(new Block(rank=1, idxType=idxType, boundingBox=[0..m/4-1], targetLocales=Locales));
  const TwiddleDom: domain(1, int(64)) dmapped TwiddleDist = [0..m/4-1];
  var Twiddles: [TwiddleDom] elemType;

  //
  // BlkDom describes the index set used to define the input and
  // output vectors and is also a 1D domain indexed by 64-bit ints
  // from 0 to m-1.  It is distributes the vectors Z and z across the
  // locales using the Block distribution.
  //
  const BlkDist = new dmap(new Block(rank=1, idxType=idxType, boundingBox=[0..m-1],
                                     targetLocales=Locales,
                                     dataParTasksPerLocale=tasksPerLocale,
                                     dataParIgnoreRunningTasks=true));
  const BlkDom: domain(1, int(64)) dmapped BlkDist = [0..m-1];
  var Z, z: [BlkDom] elemType;

  const CycDist = new dmap(new Cyclic(startIdx=0:idxType, targetLocales=Locales,
                                      dataParTasksPerLocale=tasksPerLocale,
                                      dataParIgnoreRunningTasks=true));
  const CycDom: domain(1, int(64)) dmapped CycDist = [0..m-1];
  var Zcyc: [CycDom] elemType;

  initVectors(Twiddles, z);            // initialize twiddles and input vector z

  const startTime = getCurrentTime();  // capture the start time

  Z = conjg(z);                        // store the conjugate of z in Z
  bitReverseShuffle(Z);                // permute Z
  dfft(Z, Twiddles, 0);   // compute the Fourier transform block phase
  forall (b, c) in (Z, Zcyc) do
    c = b;
  dfft(Zcyc, Twiddles, 1); // compute the Fourier transform cyclic phase
  forall (b, c) in (Z, Zcyc) do
    b = c;

  const execTime = getCurrentTime() - startTime;     // store the elapsed time

  const validAnswer = verifyResults(z, Z, Zcyc, Twiddles); // validate the answer
  printResults(validAnswer, execTime);               // print the results
}

//
// compute the discrete fast Fourier transform of a vector A declared
// over domain ADom using twiddle vector W
//
def dfft(A: [?ADom], W, phase) {
  const numElements = A.numElements;
  //
  // loop over the phases of the DFT sequentially using custom
  // iterator genDFTStrideSpan that yields the stride and span for
  // each bank of butterfly calculations
  //
  for (str, span) in genDFTStrideSpan(numElements, phase) {
    //
    // loop in parallel over each of the banks of butterflies with
    // shared twiddle factors, zippering with the unbounded range
    // 0.. to get the base twiddle indices
    //
    forall (bankStart, twidIndex) in (ADom by 2*span, 0..) {
      //
      // compute the first set of multipliers for the low bank
      //
      var wk2 = W(twidIndex),
          wk1 = W(2*twidIndex),
          wk3 = (wk1.re - 2 * wk2.im * wk1.im,
                 2 * wk2.im * wk1.re - wk1.im):elemType;
      //
      // loop in parallel over the low bank, computing butterflies
      // Note: lo..#num         == lo, lo+1, lo+2, ..., lo+num-1
      //       lo.. by str #num == lo, lo+str, lo+2*str, ... lo+(num-1)*str
      //
      forall lo in bankStart..#str {
        on ADom.dist.idxToLocale(lo) {
          local {
            butterfly(wk1, wk2, wk3, A.localSlice(lo..by str #radix));
          }
        }
      }
      //
      // update the multipliers for the high bank
      //
      wk1 = W(2*twidIndex+1);
      wk3 = (wk1.re - 2 * wk2.re * wk1.im,
             2 * wk2.re * wk1.re - wk1.im):elemType;
      wk2 *= 1.0i;

      //
      // loop in parallel over the high bank, computing butterflies
      //
      forall lo in bankStart+span..#str {
        on ADom.dist.idxToLocale(lo) {
          local {
            butterfly(wk1, wk2, wk3, A.localSlice(lo.. by str #radix));
          }
        }
      }
    }
  }

  if phase == 1 {
    //
    // Do the last set of butterflies...
    //
    const str = radix**log4(numElements-1);
    //
    // ...using the radix-4 butterflies with 1.0 multipliers if the
    // problem size is a power of 4
    //
    if (str*radix == numElements) then
      forall lo in 0..#str do
        butterfly(1.0, 1.0, 1.0, A, lo.. by str #radix);
    //
    // ...otherwise using a simple radix-2 butterfly scheme
    //
    else
      forall lo in 0..#str {
        const a = A(lo),
              b = A(lo+str);
        A(lo)     = a + b;
        A(lo+str) = a - b;
      }
  }
}

//
// this is the radix-4 butterfly routine that takes multipliers wk1,
// wk2, and wk3 and a 4-element array (slice) A.
//
def butterfly(wk1, wk2, wk3, X:[0..3]) {
  var x0 = X(0) + X(1),
      x1 = X(0) - X(1),
      x2 = X(2) + X(3),
      x3rot = (X(2) - X(3))*1.0i;

  X(0) = x0 + x2;                   // compute the butterfly in-place on X
  x0 -= x2;
  X(2) = wk2 * x0;
  x0 = x1 + x3rot;
  X(1) = wk1 * x0;
  x0 = x1 - x3rot;
  X(3) = wk3 * x0;
}

def butterfly(wk1, wk2, wk3, A, rng) {
  var X: [0..#radix] elemType;
  for (x,r) in (X, rng) do
    x = A(r);
  butterfly(wk1, wk2, wk3, X);
  for (x,r) in (X, rng) do
    A(r) = x;
}

//
// this iterator generates the stride and span values for the phases
// of the DFFT simply by yielding tuples: (radix**i, radix**(i+1))
//
def genDFTStrideSpan(numElements, phase) {
  const (start, end) =
    if phase == 0 then (1, numLocales:int(64)) else (numLocales, numElements-1);
  var stride = start;
  for i in log4(start)+1..log4(end):int {
    const span = stride * radix;
    yield (stride, span);
    stride = span;
  }
}

//
// Print the problem size
//
def printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks(tasksPerLocale=tasksPerLocale);
    printProblemSize(elemType, numVectors, m);
  }
}

def printLocalesTasks(tasksPerLocale=1) {
  writeln("Number of Locales = ", numLocales);
  writeln("Tasks per locale = ", tasksPerLocale);
}

def makeSizeAssertions() {
  assert(4**log4(numLocales) == numLocales, "numLocales must be a power of 4");
}

//
// Initialize the twiddle vector and random input vector and
// optionally print them to the console
//
def initVectors(Twiddles, z) {
  computeTwiddles(Twiddles);
  bitReverseShuffle(Twiddles);

  fillRandom(z, seed);

  if (printArrays) {
    writeln("After initialization, Twiddles is: ", Twiddles, "\n");
    writeln("z is: ", z, "\n");
  }
}

//
// Compute the twiddle vector values
//
def computeTwiddles(Twiddles) {
  const numTwdls = Twiddles.numElements,
        delta = 2.0 * atan(1.0) / numTwdls;

  Twiddles(0) = 1.0;
  Twiddles(numTwdls/2) = let x = cos(delta * numTwdls/2)
                          in (x, x): elemType;
  forall i in 1..numTwdls/2-1 {
    const x = cos(delta*i),
          y = sin(delta*i);
    Twiddles(i)            = (x, y): elemType;
    Twiddles(numTwdls - i) = (y, x): elemType;
  }
}

//
// Perform a permutation of the argument vector by reversing the bits
// of the indices
//
def bitReverseShuffle(Vect: [?Dom]) {
  const numBits = log2(Vect.numElements),
        Perm: [Dom] Vect.eltType = [i in Dom] Vect(bitReverse(i, revBits=numBits));
  Vect = Perm;
}

//
// Reverse the low revBits bits of val
//
def bitReverse(val: ?valType, revBits = 64) {
  param mask = 0x0102040810204080;
  const valReverse64 = bitMatMultOr(mask, bitMatMultOr(val:uint(64), mask)),
        valReverse = bitRotLeft(valReverse64, revBits);
  return valReverse: valType;
}

//
// Compute the log base 4 of x
//
def log4(x) return logBasePow2(x, 2);

//
// verify that the results are correct by reapplying the dfft and then
// calculating the maximum error, comparing against epsilon
//
def verifyResults(z, Z, Cyc, Twiddles) {
  if (printArrays) then writeln("After FFT, Z is: ", Z, "\n");

  Z = conjg(Z) / m;
  bitReverseShuffle(Z);
  dfft(Z, Twiddles, 0);
  forall (b, c) in (Z, Cyc) do
    c = b;
  dfft(Cyc, Twiddles, 1);
  forall (b, c) in (Z, Cyc) do
    b = c;

  if (printArrays) then writeln("After inverse FFT, Z is: ", Z, "\n");

  var maxerr = max reduce sqrt((z.re - Z.re)**2 + (z.im - Z.im)**2);
  maxerr /= (epsilon * n);
  if (printStats) then writeln("error = ", maxerr);

  return (maxerr < threshold);
}

//
// print out sucess/failure, the timing, and the Gflop/s value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (Gflop/s) = ", 5 * (m * n / execTime) * 1e-9);
  }
}

def tmp_localSlice(A, rng) {
  var low: A.rank*A._value.idxType;
  for param i in 1..A.rank {
    low(i) = rng.low;
  }
  return A._value.locArr(A._value.dom.dist.idxToLocaleInd(low)).myElems(rng);
}

//
// Use standard modules for Bit operations, Random numbers, and Timing
//
use BitOps, Random, Time, BlockDist, CyclicDist;

//
// Use shared user module for computing HPCC problem sizes
//
use HPCCProblemSize;

const radix = 4;               // the radix of this FFT implementation

const numVectors = 2;          // the number of vectors to be stored
type elemType = complex(128);  // the element type of the vectors

//
// A configuration constant defining log2(problem size) -- n -- and a
// constant defining the problem size itself -- m
//
config const n = computeProblemSize(elemType, numVectors, returnLog2 = true);
const m = 2**n;

//
// The number of tasks to use per Chapel locale in parallel loops
//
config const tasksPerLocale = here.numCores;

//
// Configuration constants defining the epsilon and threshold values
// used to verify the result
//
config const epsilon = 2.0 ** -51.0,
             threshold = 16.0;

//
// Configuration constants to indicate whether or not to use a
// pseudo-random seed (based on the clock) or a fixed seed; and to
// specify the fixed seed explicitly
//
config const useRandomSeed = false,
             seed = if useRandomSeed then SeedGenerator.currentTime else 314159265;

//
// Configuration constants to control what's printed -- benchmark
// parameters, input and output arrays, and/or statistics
//
config const printParams = true,
             printArrays = false,
             printStats = true;

//
// The program entry point
//
def main() {
  printConfiguration();          // print the problem size
  makeSizeAssertions();
  //
  // TwiddleDom describes the index set used to define the vector of
  // twiddle values and is a 1D domain indexed by 64-bit ints from 0
  // to m/4-1.  Twiddles is the vector of twiddle values.
  //
  //const TwiddleDist = new dmap(new Cyclic(startIdx=0:idxType, tasksPerLocale=tasksPerLocale));
  const TwiddleDist = new dmap(new Block(rank=1, idxType=int(64), boundingBox=[0..m/4-1], targetLocales=Locales));
  const TwiddleDom: domain(1, int(64)) dmapped TwiddleDist = [0..m/4-1];
  var Twiddles: [TwiddleDom] elemType;

  //
  // BlkDom describes the index set used to define the input and
  // output vectors and is also a 1D domain indexed by 64-bit ints
  // from 0 to m-1.  It is distributes the vectors Z and z across the
  // locales using the Block distribution.
  //
  const BlkDist = new dmap(new Block(rank=1, idxType=int(64), boundingBox=[0..m-1],
                                     targetLocales=Locales,
                                     dataParTasksPerLocale=tasksPerLocale,
                                     dataParIgnoreRunningTasks=true));
  const BlkDom: domain(1, int(64)) dmapped BlkDist = [0..m-1];
  var Z, z: [BlkDom] elemType;

  const CycDist = new dmap(new Cyclic(startIdx=0:idxType, targetLocales=Locales,
                                      dataParTasksPerLocale=tasksPerLocale,
                                      dataParIgnoreRunningTasks=true));
  const CycDom: domain(1, int(64)) dmapped CycDist = [0..m-1];
  var Zcyc: [CycDom] elemType;

  initVectors(Twiddles, z);            // initialize twiddles and input vector z

  const startTime = getCurrentTime();  // capture the start time

  Z = conjg(z);                        // store the conjugate of z in Z
  bitReverseShuffle(Z);                // permute Z
  dfft(Z, Twiddles, 0);   // compute the Fourier transform block phase
  forall (b, c) in (Z, Zcyc) do
    c = b;
  dfft(Zcyc, Twiddles, 1); // compute the Fourier transform cyclic phase
  forall (b, c) in (Z, Zcyc) do
    b = c;

  const execTime = getCurrentTime() - startTime;     // store the elapsed time

  const validAnswer = verifyResults(z, Z, Zcyc, Twiddles); // validate the answer
  printResults(validAnswer, execTime);               // print the results
}

//
// compute the discrete fast Fourier transform of a vector A declared
// over domain ADom using twiddle vector W
//
def dfft(A: [?ADom], W, phase) {
  const numElements = A.numElements;
  //
  // loop over the phases of the DFT sequentially using custom
  // iterator genDFTStrideSpan that yields the stride and span for
  // each bank of butterfly calculations
  //
  for (str, span) in genDFTStrideSpan(numElements, phase) {
    //
    // loop in parallel over each of the banks of butterflies with
    // shared twiddle factors, zippering with the unbounded range
    // 0.. to get the base twiddle indices
    //
    forall (bankStart, twidIndex) in (ADom by 2*span, 0..) {
      //
      // compute the first set of multipliers for the low bank
      //
      var wk2 = W(twidIndex),
          wk1 = W(2*twidIndex),
          wk3 = (wk1.re - 2 * wk2.im * wk1.im,
                 2 * wk2.im * wk1.re - wk1.im):elemType;
      //
      // loop in parallel over the low bank, computing butterflies
      // Note: lo..#num         == lo, lo+1, lo+2, ..., lo+num-1
      //       lo.. by str #num == lo, lo+str, lo+2*str, ... lo+(num-1)*str
      //
      forall lo in bankStart..#str {
        on ADom.dist.idxToLocale(lo) {
          local {
            butterfly(wk1, wk2, wk3, A.localSlice(lo..by str #radix));
          }
        }
      }
      //
      // update the multipliers for the high bank
      //
      wk1 = W(2*twidIndex+1);
      wk3 = (wk1.re - 2 * wk2.re * wk1.im,
             2 * wk2.re * wk1.re - wk1.im):elemType;
      wk2 *= 1.0i;

      //
      // loop in parallel over the high bank, computing butterflies
      //
      forall lo in bankStart+span..#str {
        on ADom.dist.idxToLocale(lo) {
          local {
            butterfly(wk1, wk2, wk3, A.localSlice(lo.. by str #radix));
          }
        }
      }
    }
  }

  if phase == 1 {
    //
    // Do the last set of butterflies...
    //
    const str = radix**log4(numElements-1);
    //
    // ...using the radix-4 butterflies with 1.0 multipliers if the
    // problem size is a power of 4
    //
    if (str*radix == numElements) then
      forall lo in 0..#str do
        butterfly(1.0, 1.0, 1.0, A, lo.. by str #radix);
    //
    // ...otherwise using a simple radix-2 butterfly scheme
    //
    else
      forall lo in 0..#str {
        const a = A(lo),
              b = A(lo+str);
        A(lo)     = a + b;
        A(lo+str) = a - b;
      }
  }
}

//
// this is the radix-4 butterfly routine that takes multipliers wk1,
// wk2, and wk3 and a 4-element array (slice) A.
//
def butterfly(wk1, wk2, wk3, X:[0..3]) {
  var x0 = X(0) + X(1),
      x1 = X(0) - X(1),
      x2 = X(2) + X(3),
      x3rot = (X(2) - X(3))*1.0i;

  X(0) = x0 + x2;                   // compute the butterfly in-place on X
  x0 -= x2;
  X(2) = wk2 * x0;
  x0 = x1 + x3rot;
  X(1) = wk1 * x0;
  x0 = x1 - x3rot;
  X(3) = wk3 * x0;
}

def butterfly(wk1, wk2, wk3, A, rng) {
  var X: [0..#radix] elemType;
  for (x,r) in (X, rng) do
    x = A(r);
  butterfly(wk1, wk2, wk3, X);
  for (x,r) in (X, rng) do
    A(r) = x;
}

//
// this iterator generates the stride and span values for the phases
// of the DFFT simply by yielding tuples: (radix**i, radix**(i+1))
//
def genDFTStrideSpan(numElements, phase) {
  const (start, end) =
    if phase == 0 then (1, numLocales:int(64)) else (numLocales, numElements-1);
  var stride = start;
  for i in log4(start)+1..log4(end):int {
    const span = stride * radix;
    yield (stride, span);
    stride = span;
  }
}

//
// Print the problem size
//
def printConfiguration() {
  if (printParams) {
    if (printStats) then printLocalesTasks(tasksPerLocale=tasksPerLocale);
    printProblemSize(elemType, numVectors, m);
  }
}

def printLocalesTasks(tasksPerLocale=1) {
  writeln("Number of Locales = ", numLocales);
  writeln("Tasks per locale = ", tasksPerLocale);
}

def makeSizeAssertions() {
  assert(4**log4(numLocales) == numLocales, "numLocales must be a power of 4");
}

//
// Initialize the twiddle vector and random input vector and
// optionally print them to the console
//
def initVectors(Twiddles, z) {
  computeTwiddles(Twiddles);
  bitReverseShuffle(Twiddles);

  fillRandom(z, seed);

  if (printArrays) {
    writeln("After initialization, Twiddles is: ", Twiddles, "\n");
    writeln("z is: ", z, "\n");
  }
}

//
// Compute the twiddle vector values
//
def computeTwiddles(Twiddles) {
  const numTwdls = Twiddles.numElements,
        delta = 2.0 * atan(1.0) / numTwdls;

  Twiddles(0) = 1.0;
  Twiddles(numTwdls/2) = let x = cos(delta * numTwdls/2)
                          in (x, x): elemType;
  forall i in 1..numTwdls/2-1 {
    const x = cos(delta*i),
          y = sin(delta*i);
    Twiddles(i)            = (x, y): elemType;
    Twiddles(numTwdls - i) = (y, x): elemType;
  }
}

//
// Perform a permutation of the argument vector by reversing the bits
// of the indices
//
def bitReverseShuffle(Vect: [?Dom]) {
  const numBits = log2(Vect.numElements),
        Perm: [Dom] Vect.eltType = [i in Dom] Vect(bitReverse(i, revBits=numBits));
  Vect = Perm;
}

//
// Reverse the low revBits bits of val
//
def bitReverse(val: ?valType, revBits = 64) {
  param mask = 0x0102040810204080;
  const valReverse64 = bitMatMultOr(mask, bitMatMultOr(val:uint(64), mask)),
        valReverse = bitRotLeft(valReverse64, revBits);
  return valReverse: valType;
}

//
// Compute the log base 4 of x
//
def log4(x) return logBasePow2(x, 2);

//
// verify that the results are correct by reapplying the dfft and then
// calculating the maximum error, comparing against epsilon
//
def verifyResults(z, Z, Cyc, Twiddles) {
  if (printArrays) then writeln("After FFT, Z is: ", Z, "\n");

  Z = conjg(Z) / m;
  bitReverseShuffle(Z);
  dfft(Z, Twiddles, 0);
  forall (b, c) in (Z, Cyc) do
    c = b;
  dfft(Cyc, Twiddles, 1);
  forall (b, c) in (Z, Cyc) do
    b = c;

  if (printArrays) then writeln("After inverse FFT, Z is: ", Z, "\n");

  var maxerr = max reduce sqrt((z.re - Z.re)**2 + (z.im - Z.im)**2);
  maxerr /= (epsilon * n);
  if (printStats) then writeln("error = ", maxerr);

  return (maxerr < threshold);
}

//
// print out sucess/failure, the timing, and the Gflop/s value
//
def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (Gflop/s) = ", 5 * (m * n / execTime) * 1e-9);
  }
}

def tmp_localSlice(A, rng) {
  var low: A.rank*A._value.idxType;
  for param i in 1..A.rank {
    low(i) = rng.low;
  }
  return A._value.locArr(A._value.dom.dist.idxToLocaleInd(low)).myElems(rng);
}


edit to identityMat

edit to identityMat

use LinearAlgebra,
    LinearAlgebra.Sparse,
    Time,
    NumSuch,
    Random;

config param batchsize = 10000;

/*
 A matrix of data with named rows and columns.
 */
class NamedMatrix {
  var D: domain(2),
      SD = CSRDomain(D),
      X: [SD] real,  // the actual data
      rows: BiMap = new BiMap(),
      cols: BiMap = new BiMap();

   proc init() {
     this.complete(); // NAMEDMATRIX DOESNT EXPLICITLY INHERIT ANYTHING SO IS THIS NECESSARY??
   }

   proc init(X) {
     this.D = {X.domain.dim(1), X.domain.dim(2)};
     this.SD = X.domain;
     this.complete();
     this.loadX(X);
   }

   proc init(X, names: [] string) {
     this.init(X);
     try! this.setRowNames(names);
     try! this.setColNames(names);
   }

   proc init(X, rownames: [] string, colnames: [] string) {
     this.init(X);
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(rownames: [] string, colnames: [] string) {
     this.D = {1..rownames.size, 1..colnames.size};
     this.complete();
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(N: NamedMatrix) {
     this.init(N.X);
     this.rows = N.rows;
     this.cols = N.cols;
   }

   proc init(X, rows: BiMap, cols: BiMap) {
     this.init(X);
     this.rows = rows;
     this.cols = cols;
   }
}

/*
Returns the number of rows in the matrix frame
 */
proc NamedMatrix.nrows() {
  return X.domain.dim(1).size;
}

/*
Returns the number of columns in the matrix frame
 */
proc NamedMatrix.ncols() {
  return X.domain.dim(2).size;
}

/*
 Finds the max along the row indicated by row number
 */

proc NamedMatrix.rowMax(i: int) {
  return aMax(this.X, 1)[i];
}
proc NamedMatrix.rowMax(s: string) {
  return this.rowMax(this.rows.get(s));
}

proc NamedMatrix.rowMax() {
  return aMax(this.X, 1);
}

proc NamedMatrix.colMax(i: int) {
  return aMax(this.X, 2)[i];
}
proc NamedMatrix.colMax(s: string) {
  return this.colMax(this.cols.get(s));
}

proc NamedMatrix.colMax() {
  return aMax(this.X, 2);
}

proc NamedMatrix.rowArgMax(i: int) {
  return argMax(this.X, 1)[i];
}
proc NamedMatrix.rowArgMax(s: string) {
  return this.rowArgMax(this.rows.get(s));
}

proc NamedMatrix.rowArgMax() {
  return argMax(this.X, 1);
}

proc NamedMatrix.colArgMax(i: int) {
  return argMax(this.X, 2)[i];
}
proc NamedMatrix.colArgMax(s: string) {
  return this.colArgMax(this.cols.get(s));
}

proc NamedMatrix.colArgMax() {
  return argMax(this.X, 2);
}

proc NamedMatrix.rowMin(i: int) {
  return aMin(this.X, 1)[i];
}
proc NamedMatrix.rowMin(s: string) {
  return this.rowMin(this.rows.get(s));
}

proc NamedMatrix.rowMin() {
  return aMin(this.X, 1);
}

proc NamedMatrix.colMin(i: int) {
  return aMin(this.X, 2)[i];
}
proc NamedMatrix.colMin(s: string) {
  return this.colMin(this.cols.get(s));
}

proc NamedMatrix.colMin() {
  return aMin(this.X, 2);
}

proc NamedMatrix.rowArgMin(i:int) {
  return argMin(this.X, 1)[i];
}
proc NamedMatrix.rowArgMin(s: string) {
  return this.rowArgMin(this.rows.get(s));
}

proc NamedMatrix.rowArgMin() {
  return argMin(this.X, 1);
}

proc NamedMatrix.colArgMin(i) {
  return argMin(this.X, 2)[i];
}
proc NamedMatrix.colArgMin(s: string) {
  return this.colArgMin(this.cols.get(s));
}

proc NamedMatrix.colArgMin() {
  return argMin(this.X, 2);
}

/* We don't have a good matPlus for Q + E yet, so doing this element
  by element. kinda sucks.
 */
proc NamedMatrix.matPlus(Y: NamedMatrix) {
  forall ij in Y.SD {
    this.X[ij] += Y.X[ij];
  }
}

/*
 Multiplies the underlying matrix in place.
 */
proc NamedMatrix.eTimes(a: real) {
  this.X = a * this.X;
}

/*
 Prints the matrix with row / column headers, not recommended for large matrices.
 Also width of entries should not exceed about 8 places.
 */
proc NamedMatrix.pprint() {
  write("     ");
  for c in this.cols.sorted() {
    write("       %5s  |".format(c[1]));
  }
  write("\n");
  for r in this.rows.sorted() {
    write(" %3s ".format(r[1]));
    for c in this.cols.sorted() {
      write(" %13.3dr ".format(this.get(r[2], c[2])));
    }
    write("\n");
  }

}

//proc NamedMatrix.argRowMax()

/*
Loads the data from X into the internal array, also called X.  We call them all X to keep it clear.

:arg real[]: Array representing the matrix
 */
proc NamedMatrix.loadX(X:[], shape: 2*int =(-1,-1)) {
  if shape(1) > 0 && shape(2) > 0 {
    this.D = {1..shape(1), 1..shape(2)};
  }
  for (i,j) in X.domain {
    this.SD += (i,j);
    this.X(i,j) = X(i,j);
  }
}

/*
Sets the row names for the matrix X
 */
proc NamedMatrix.setRowNames(rn:[]): string throws {
  //if (X.domain.dim(1).size > 0) && (rn.size != X.domain.dim(1).size) {
  //if (1 > 0) && (rn.size != X.domain.dim(1).size) {
  if rn.size != X.domain.dim(1).size {
    const err = new DimensionMatchError(expected = X.domain.dim(1).size, actual=rn.size);
    throw err;
    return "";
  } else {
    for i in 1..rn.size {
      this.rows.add(rn[i]);
    }
    return this.rows;
  }
}

/*
Sets the column names for the matrix X
*/
proc NamedMatrix.setColNames(cn:[]): string throws {
  if cn.size != X.domain.dim(2).size {
    const err = new DimensionMatchError(expected = X.domain.dim(2).size, actual=cn.size);
    throw err;
    return "";
  } else {
    for i in 1..cn.size {
      this.cols.add(cn[i]);
    }
  }
  return this.cols;
}

/*
 Gets the value of the (i,j) entry of the matrix X in the NamedMatrix
 */
proc NamedMatrix.get(i: int, j: int) {
   return this.X(i,j);
 }

 /*
 Get the values using the named rows and columns
  */
proc NamedMatrix.get(f: string, t: string) {
    return this.X(rows.get(f), cols.get(t));
 }

/*
 Set the value of this.X(i,j) by index row
 */
proc NamedMatrix.set(i: int, j: int, w: real) {
    if this.SD.member((i,j)) {
      this.X(i,j) = w;
    } else {
      this.SD += (i,j);
      this.X(i,j) = w;
    }
    return w;
}

/*
Set the values using the row and column names
*/
proc NamedMatrix.set(f: string, t: string, w: real) {
  return this.set(rows.get(f), cols.get(t), w);
}

/*
Update the value in X(i,j) with `w`
 */
proc NamedMatrix.update(i: int, j: int, w: real) {
  const x = this.get(i,j);
  return this.set(i,j, x + w);
}

/*
Update the value in X(i,j) with `w` using the row column names. This adds
`w` to the current value. To replace the current value, use `.set()`
 */
proc NamedMatrix.update(f: string, t: string, w: real) {
  const x = this.get(rows.get(f),cols.get(t));
  return this.set(rows.get(f),cols.get(t), x + w);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column ids
 */
proc NamedMatrix.remove(i: int, j:int) {
  this.SD -= (i,j);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column names
 */
proc NamedMatrix.remove(i: string, j:string) {
  this.remove(this.rows.get(i), this.cols.get(j));
}

/*
Find the number of non-zeroes in the matrix
 */
proc NamedMatrix.nnz() {
  var i: int = 0;
  for ij in this.X.domain {
    i += 1;
  }
  return i;
}

/*
Calculates the sparsity of the matrix: Number of entries / frame size
 */
proc NamedMatrix.sparsity() {
  const d = this.X.shape[1]:real * this.X.shape[2]: real;
  return this.nnz():real / d;
}

/*
 Takes a index (i,j) tuple and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(ij: 2*int) {
  return this.ncols() * ij[1] + ij[2] - this.ncols();
}
/*
 Takes a index values `i,j` and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(i: int, j: int) {
  return this.grid2seq((i,j));
}

/*
 Takes a sequential value and gives the row / column indices of that entry
 */
proc NamedMatrix.seq2grid(k: int) {
  return (divfloor(k, this.ncols())+1, k % this.ncols());
}

proc NamedMatrix.ntropic(N: NamedMatrix) {
  var T: NamedMatrix = new NamedMatrix(X = tropic(this.X, N.X), this.rows, N.cols);
  return T;
}

/*
 Multiplies the current NamedMatrix `X` against the argument `Y`, but first it aligns
 the names of `X.cols` with `Y.rows`.  Returns an appropriately named NamedMatrix
 :arg NamedMatrix Y:
 */
proc NamedMatrix.ndot(N: NamedMatrix) {
  var C: NamedMatrix = new NamedMatrix(X = dot(this.X,N.X), this.rows, N.cols);
  return C;
}

/*
 Routine to take two NamedMatrices and multiply them along just the row/column
 intersections.  If X has cols [red yellow blue green] and Y has rows [red blue green]
 this function produces a NamedMatrix with rows [red blue green]
 */
proc NamedMatrix.alignAndMultiply(Y: NamedMatrix) {
    var rcOverlap: domain(string) = this.cols.keys & Y.rows.keys;

    var xSD: sparse subdomain(this.D) dmapped CS(),
        ySD: sparse subdomain(Y.D) dmapped CS(),
        xX: [xSD] real,  // the actual data
        yX: [ySD] real,  // the actual data
        xrows: BiMap = new BiMap(),
        yrows: BiMap = new BiMap(),
        xcols: BiMap = new BiMap(),
        ycols: BiMap = new BiMap();

    for (left_row, right_col) in zip(this.D.dim(2), Y.D.dim(1)) {
      for rc in rcOverlap {
        const j = this.cols.get(rc);
        if this.SD.member(left_row, j) {
          xSD += (left_row, j);
          xX[left_row, j] = this.X[left_row, j];
        }
        if Y.SD.member(j, right_col) {
          ySD += (j, right_col);
          yX[j, right_col] = Y.X[j, right_col];
        }
      }
    }
    const z = xX.dot(yX);
    var n = new NamedMatrix(X=z);
    n.rows = this.rows;
    n.cols = Y.cols;
    return n;
}

/*
 Designed to be similar to SciPy's `amax() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amax.html>`_
 Useful on sparse matrices

 We'll get around to documenting here it eventually
 */
proc aMax(X:[], axis = 0) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    maxima[1] = (max reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  }
  return maxima;
}

/*
 Designed to be similar to SciPy's `amin() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amin.html>`_
 Useful on sparse matrices.

 We'll get around to documenting here it eventually
 */
proc aMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    minima[1] = (min reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  }
  return minima;
}

/*
Provides the "where" of the min along an axis in a sparse matrix
 */
proc argMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    minima[1] = miniLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = miniLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = miniLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  }
  return minima;
}

/*
 Provides the "where" of a max along a given axis in a sparse matrix
 */
proc argMax(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    maxima[1] = maxiLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = maxiLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = maxiLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  }
  return maxima;
}

// HELPER FUNCTIONS SHOULD NOT BE EXPOSED TO USER
proc miniLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      minimum: real = 1000000;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) < minimum {
        minimum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) < minimum {
        minimum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) < minimum {
        minimum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc maxiLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      maximum: real;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) > maximum {
        maximum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) > maximum {
        maximum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) > maximum {
        maximum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc tropic(A:[],B:[]) {
  var dom: domain(2) = {A.domain.dim(1),B.domain.dim(2)};
  var sps = CSRDomain(dom);
//  var BT = transpose(B);
  var T: [sps] real;
  for (i,j) in dom {
    var t: Timer;
    t.start();
    var mini = 100000000: real;
    if A.domain.member((i,j)) {
      mini = min(mini,A(i,j));
    }
    if B.domain.member((i,j)) {
      mini = min(mini,B(i,j));
    }
    for w in A.domain.dimIter(2,i) {
      if B.domain.member((w,j)) {
        mini = min(mini,A(i,w) + B(w,j));
      }
    }
    t.stop();
    if mini < 100000000 {
      sps += (i,j);
      T(i,j) = mini;
      writeln((i,j));
      writeln("",t.elapsed());
    }
  }
  return T;
}

proc tropic_(A:[],B:[]) {
  var dom: domain(2) = {A.domain.dim(1),B.domain.dim(2)};
  var sps = CSRDomain(dom);
  var BT = transpose(B);
  var T: [sps] real;
  for (i,j) in dom {
    var mini = 100000000: real;
    var wdom: domain(1);
    var wids: [wdom] int;
    for w in A.domain.dimIter(2,i) {
      wids.push_back(w);
    }
    //writeln(wids);
    for w2 in BT.domain.dimIter(2,j) {
      wids.push_back(w2);
    }
    var wDom: domain(1);
    var witness: [wDom] real;
    for w in wids {
      if A(i,w) > 0 && BT(j,w) > 0 {
        witness.push_back(A(i,w) + BT(j,w));
      }
    }
    if A.domain.member((i,j)) {
      witness.push_back(A(i,j));
    }
    if B.domain.member((i,j)) {
      witness.push_back(B(i,j));
    }
    mini = min reduce witness;
    if mini < 1000000000 {
      sps += (i,j);
      T(i,j) = mini;
    }
  }
  return T;
}

proc sparseEq(A:[], B:[]) {
  var s: bool = true;
  forall (i,j) in A.domain with (&& reduce s) {
    s &&= A(i,j) == B(i,j);
  }
  return s;
}

/*
proc adjacencyPolynomial(A:[]) {

}*/

proc tropicLimit(A:[] real,B:[] real): A.type {
 var R = tropic(A,B);
 if A.domain == R.domain {
   if sparseEq(R,A) {
     return R;
   } else {
     return tropicLimit(R,B);
   }
 } else {
   return tropicLimit(R,B);
 }
}

/*
 Build a random sparse matrix.  Good for testing;
 */
proc generateRandomSparseMatrix(size: int, sparsity: real) {
  const D: domain(2) = {1..size, 1..size};
  var SD: sparse subdomain(D) dmapped CS();
  var R: [SD] real;
  var da: domain(1) = {1..size};
  var array: [da] int = 1..size;
  var dom: domain(1) = {1..0};
  var indices: [dom] 2*int;
  var N = floor(size*(1-sparsity)): int;
  forall i in array {
    forall j in array {
      indices.push_back((i,j));
    }
  }
  shuffle(indices);
  var sparseids = indices[1..N];
  SD.bulkAdd(sparseids);
  forall (i,j) in sparseids {
    R(i,j) = 1;
  }
  //ndices = zip(array1, array2)
  return R;
}

/*
 General sparsity function for any `real` matrix.
 */
proc sparsity(X) {
  const d = X.shape[1]:real * X.shape[2]: real;
  var i: real = 0.0;
  for ij in X.domain {
    i += 1.0;
  }
  return i / d;
}

proc elemMult(A: [], B: []) {
  var dom: domain(2) = {A.domain.dim(1),A.domain.dim(2)};
  var sps = CSRDomain(dom);
  var C: [sps] real;
  for (i,j) in A.domain {
    if B.domain.member((i,j)) {
      sps += (i,j);
      C(i,j) = A(i,j) * B(i,j);
    }
  }
  return C;
}

/*
 Error classes
 */
class NumSuchError : Error {
  proc init() {
    super.init();
    this.complete();
  }
  proc message() {
    return "Generic NumSuch Error";
  }
}

proc identityMat(n: int) {
  var dom: domain(2) = {1..n,1..n};
  var sps = CSRDomain(dom);
  for i in 1..n {
    sps += (i,i);
  }
  var I: [sps] real;
  forall ij in sps {
    I(ij) = 1;
  }
  return I;
}

/*
 Used to indidcate dimension mismatches on NamedMatrices and vectors
 */
class DimensionMatchError : NumSuchError {
  var expected: int,
      actual: int;

  proc init(expected: int, actual:int) {
    super.init();
    this.complete();
    this.expected = expected;
    this.actual = actual;
  }

  proc message() {
    return "Error matching dimensions.  Expected: " + this.expected + " Actual: " + this.actual;
  }
}

use LinearAlgebra,
    LinearAlgebra.Sparse,
    Time,
    NumSuch,
    Random;

config param batchsize = 10000;

/*
 A matrix of data with named rows and columns.
 */
class NamedMatrix {
  var D: domain(2),
      SD = CSRDomain(D),
      X: [SD] real,  // the actual data
      rows: BiMap = new BiMap(),
      cols: BiMap = new BiMap();

   proc init() {
     this.complete(); // NAMEDMATRIX DOESNT EXPLICITLY INHERIT ANYTHING SO IS THIS NECESSARY??
   }

   proc init(X) {
     this.D = {X.domain.dim(1), X.domain.dim(2)};
     this.SD = X.domain;
     this.complete();
     this.loadX(X);
   }

   proc init(X, names: [] string) {
     this.init(X);
     try! this.setRowNames(names);
     try! this.setColNames(names);
   }

   proc init(X, rownames: [] string, colnames: [] string) {
     this.init(X);
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(rownames: [] string, colnames: [] string) {
     this.D = {1..rownames.size, 1..colnames.size};
     this.complete();
     try! this.setRowNames(rownames);
     try! this.setColNames(colnames);
   }

   proc init(N: NamedMatrix) {
     this.init(N.X);
     this.rows = N.rows;
     this.cols = N.cols;
   }

   proc init(X, rows: BiMap, cols: BiMap) {
     this.init(X);
     this.rows = rows;
     this.cols = cols;
   }
}

/*
Returns the number of rows in the matrix frame
 */
proc NamedMatrix.nrows() {
  return X.domain.dim(1).size;
}

/*
Returns the number of columns in the matrix frame
 */
proc NamedMatrix.ncols() {
  return X.domain.dim(2).size;
}

/*
 Finds the max along the row indicated by row number
 */

proc NamedMatrix.rowMax(i: int) {
  return aMax(this.X, 1)[i];
}
proc NamedMatrix.rowMax(s: string) {
  return this.rowMax(this.rows.get(s));
}

proc NamedMatrix.rowMax() {
  return aMax(this.X, 1);
}

proc NamedMatrix.colMax(i: int) {
  return aMax(this.X, 2)[i];
}
proc NamedMatrix.colMax(s: string) {
  return this.colMax(this.cols.get(s));
}

proc NamedMatrix.colMax() {
  return aMax(this.X, 2);
}

proc NamedMatrix.rowArgMax(i: int) {
  return argMax(this.X, 1)[i];
}
proc NamedMatrix.rowArgMax(s: string) {
  return this.rowArgMax(this.rows.get(s));
}

proc NamedMatrix.rowArgMax() {
  return argMax(this.X, 1);
}

proc NamedMatrix.colArgMax(i: int) {
  return argMax(this.X, 2)[i];
}
proc NamedMatrix.colArgMax(s: string) {
  return this.colArgMax(this.cols.get(s));
}

proc NamedMatrix.colArgMax() {
  return argMax(this.X, 2);
}

proc NamedMatrix.rowMin(i: int) {
  return aMin(this.X, 1)[i];
}
proc NamedMatrix.rowMin(s: string) {
  return this.rowMin(this.rows.get(s));
}

proc NamedMatrix.rowMin() {
  return aMin(this.X, 1);
}

proc NamedMatrix.colMin(i: int) {
  return aMin(this.X, 2)[i];
}
proc NamedMatrix.colMin(s: string) {
  return this.colMin(this.cols.get(s));
}

proc NamedMatrix.colMin() {
  return aMin(this.X, 2);
}

proc NamedMatrix.rowArgMin(i:int) {
  return argMin(this.X, 1)[i];
}
proc NamedMatrix.rowArgMin(s: string) {
  return this.rowArgMin(this.rows.get(s));
}

proc NamedMatrix.rowArgMin() {
  return argMin(this.X, 1);
}

proc NamedMatrix.colArgMin(i) {
  return argMin(this.X, 2)[i];
}
proc NamedMatrix.colArgMin(s: string) {
  return this.colArgMin(this.cols.get(s));
}

proc NamedMatrix.colArgMin() {
  return argMin(this.X, 2);
}

/* We don't have a good matPlus for Q + E yet, so doing this element
  by element. kinda sucks.
 */
proc NamedMatrix.matPlus(Y: NamedMatrix) {
  forall ij in Y.SD {
    this.X[ij] += Y.X[ij];
  }
}

/*
 Multiplies the underlying matrix in place.
 */
proc NamedMatrix.eTimes(a: real) {
  this.X = a * this.X;
}

/*
 Prints the matrix with row / column headers, not recommended for large matrices.
 Also width of entries should not exceed about 8 places.
 */
proc NamedMatrix.pprint() {
  write("     ");
  for c in this.cols.sorted() {
    write("       %5s  |".format(c[1]));
  }
  write("\n");
  for r in this.rows.sorted() {
    write(" %3s ".format(r[1]));
    for c in this.cols.sorted() {
      write(" %13.3dr ".format(this.get(r[2], c[2])));
    }
    write("\n");
  }

}

//proc NamedMatrix.argRowMax()

/*
Loads the data from X into the internal array, also called X.  We call them all X to keep it clear.

:arg real[]: Array representing the matrix
 */
proc NamedMatrix.loadX(X:[], shape: 2*int =(-1,-1)) {
  if shape(1) > 0 && shape(2) > 0 {
    this.D = {1..shape(1), 1..shape(2)};
  }
  for (i,j) in X.domain {
    this.SD += (i,j);
    this.X(i,j) = X(i,j);
  }
}

/*
Sets the row names for the matrix X
 */
proc NamedMatrix.setRowNames(rn:[]): string throws {
  //if (X.domain.dim(1).size > 0) && (rn.size != X.domain.dim(1).size) {
  //if (1 > 0) && (rn.size != X.domain.dim(1).size) {
  if rn.size != X.domain.dim(1).size {
    const err = new DimensionMatchError(expected = X.domain.dim(1).size, actual=rn.size);
    throw err;
    return "";
  } else {
    for i in 1..rn.size {
      this.rows.add(rn[i]);
    }
    return this.rows;
  }
}

/*
Sets the column names for the matrix X
*/
proc NamedMatrix.setColNames(cn:[]): string throws {
  if cn.size != X.domain.dim(2).size {
    const err = new DimensionMatchError(expected = X.domain.dim(2).size, actual=cn.size);
    throw err;
    return "";
  } else {
    for i in 1..cn.size {
      this.cols.add(cn[i]);
    }
  }
  return this.cols;
}

/*
 Gets the value of the (i,j) entry of the matrix X in the NamedMatrix
 */
proc NamedMatrix.get(i: int, j: int) {
   return this.X(i,j);
 }

 /*
 Get the values using the named rows and columns
  */
proc NamedMatrix.get(f: string, t: string) {
    return this.X(rows.get(f), cols.get(t));
 }

/*
 Set the value of this.X(i,j) by index row
 */
proc NamedMatrix.set(i: int, j: int, w: real) {
    if this.SD.member((i,j)) {
      this.X(i,j) = w;
    } else {
      this.SD += (i,j);
      this.X(i,j) = w;
    }
    return w;
}

/*
Set the values using the row and column names
*/
proc NamedMatrix.set(f: string, t: string, w: real) {
  return this.set(rows.get(f), cols.get(t), w);
}

/*
Update the value in X(i,j) with `w`
 */
proc NamedMatrix.update(i: int, j: int, w: real) {
  const x = this.get(i,j);
  return this.set(i,j, x + w);
}

/*
Update the value in X(i,j) with `w` using the row column names. This adds
`w` to the current value. To replace the current value, use `.set()`
 */
proc NamedMatrix.update(f: string, t: string, w: real) {
  const x = this.get(rows.get(f),cols.get(t));
  return this.set(rows.get(f),cols.get(t), x + w);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column ids
 */
proc NamedMatrix.remove(i: int, j:int) {
  this.SD -= (i,j);
}

/*
 Removes the matrix entry `(i,j)` from the matrix and the underlying domain
 by row, column names
 */
proc NamedMatrix.remove(i: string, j:string) {
  this.remove(this.rows.get(i), this.cols.get(j));
}

/*
Find the number of non-zeroes in the matrix
 */
proc NamedMatrix.nnz() {
  var i: int = 0;
  for ij in this.X.domain {
    i += 1;
  }
  return i;
}

/*
Calculates the sparsity of the matrix: Number of entries / frame size
 */
proc NamedMatrix.sparsity() {
  const d = this.X.shape[1]:real * this.X.shape[2]: real;
  return this.nnz():real / d;
}

/*
 Takes a index (i,j) tuple and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(ij: 2*int) {
  return this.ncols() * ij[1] + ij[2] - this.ncols();
}
/*
 Takes a index values `i,j` and turns it into the sequential entry number on the
 given matrix.
 */
proc NamedMatrix.grid2seq(i: int, j: int) {
  return this.grid2seq((i,j));
}

/*
 Takes a sequential value and gives the row / column indices of that entry
 */
proc NamedMatrix.seq2grid(k: int) {
  return (divfloor(k, this.ncols())+1, k % this.ncols());
}

proc NamedMatrix.ntropic(N: NamedMatrix) {
  var T: NamedMatrix = new NamedMatrix(X = tropic(this.X, N.X), this.rows, N.cols);
  return T;
}

/*
 Multiplies the current NamedMatrix `X` against the argument `Y`, but first it aligns
 the names of `X.cols` with `Y.rows`.  Returns an appropriately named NamedMatrix
 :arg NamedMatrix Y:
 */
proc NamedMatrix.ndot(N: NamedMatrix) {
  var C: NamedMatrix = new NamedMatrix(X = dot(this.X,N.X), this.rows, N.cols);
  return C;
}

/*
 Routine to take two NamedMatrices and multiply them along just the row/column
 intersections.  If X has cols [red yellow blue green] and Y has rows [red blue green]
 this function produces a NamedMatrix with rows [red blue green]
 */
proc NamedMatrix.alignAndMultiply(Y: NamedMatrix) {
    var rcOverlap: domain(string) = this.cols.keys & Y.rows.keys;

    var xSD: sparse subdomain(this.D) dmapped CS(),
        ySD: sparse subdomain(Y.D) dmapped CS(),
        xX: [xSD] real,  // the actual data
        yX: [ySD] real,  // the actual data
        xrows: BiMap = new BiMap(),
        yrows: BiMap = new BiMap(),
        xcols: BiMap = new BiMap(),
        ycols: BiMap = new BiMap();

    for (left_row, right_col) in zip(this.D.dim(2), Y.D.dim(1)) {
      for rc in rcOverlap {
        const j = this.cols.get(rc);
        if this.SD.member(left_row, j) {
          xSD += (left_row, j);
          xX[left_row, j] = this.X[left_row, j];
        }
        if Y.SD.member(j, right_col) {
          ySD += (j, right_col);
          yX[j, right_col] = Y.X[j, right_col];
        }
      }
    }
    const z = xX.dot(yX);
    var n = new NamedMatrix(X=z);
    n.rows = this.rows;
    n.cols = Y.cols;
    return n;
}

/*
 Designed to be similar to SciPy's `amax() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amax.html>`_
 Useful on sparse matrices

 We'll get around to documenting here it eventually
 */
proc aMax(X:[], axis = 0) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    maxima[1] = (max reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        maxima[i] = max reduce ws;
      }
    }
  }
  return maxima;
}

/*
 Designed to be similar to SciPy's `amin() <https://docs.scipy.org/doc/numpy/reference/generated/numpy.amin.html>`_
 Useful on sparse matrices.

 We'll get around to documenting here it eventually
 */
proc aMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] real;
  if axis == 0 {
    var wDom: domain(1) = 1..X.domain.size;
    var ws = reshape(X, wDom);
    sis += 1;
    minima[1] = (min reduce ws);
  }
  if axis == 2 {
    is = X.domain.dim(1);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in transpose(X).domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(a,i));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  } else if axis == 1 {
    is = X.domain.dim(2);
    for i in is {
      var wDom: domain(1);
      var ws: [wDom] real;
      for a in X.domain.dimIter(2, i) {
        sis += i;
        ws.push_back(X(i,a));
      }
      if sis.member(i) {
        minima[i] = min reduce ws;
      }
    }
  }
  return minima;
}

/*
Provides the "where" of the min along an axis in a sparse matrix
 */
proc argMin(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var minima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    minima[1] = miniLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = miniLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = miniLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        minima[i] = idm;
      }
    }
  }
  return minima;
}

/*
 Provides the "where" of a max along a given axis in a sparse matrix
 */
proc argMax(X:[], axis: int) {
  var is: domain(1) = 1..max(X.domain.dim(1).size,X.domain.dim(2).size);
  var sis: sparse subdomain(is);
  var maxima: [sis] 2*int;
  if axis == 0 {
    sis += 1;
    maxima[1] = maxiLoc_(axis == 0, id = 1, X);
  } else if axis == 2 {
    for i in is {
      var idm = maxiLoc_(1,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  } else if axis == 1 {
    for i in is {
      var idm = maxiLoc_(2,i,X);
      if idm != (0,0) {
        sis += i;
        maxima[i] = idm;
      }
    }
  }
  return maxima;
}

// HELPER FUNCTIONS SHOULD NOT BE EXPOSED TO USER
proc miniLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      minimum: real = 1000000;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) < minimum {
        minimum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) < minimum {
        minimum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) < minimum {
        minimum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc maxiLoc_(axis:int, id, X:[]) {
  var idm: (int,int),
      maximum: real;
  if axis == 0 {
    for (i,j) in X.domain {
      if X(i,j) > maximum {
        maximum = X(i,j);
        idm = (i,j);
      }
    }
  }
  if axis == 1 {
    for a in transpose(X).domain.dimIter(2, id) {
      if X(a,id) > maximum {
        maximum = X(a,id);
        idm = (a,id);
      }
    }
  }
  if axis == 2 {
    for a in X.domain.dimIter(2, id) {
      if X(id,a) > maximum {
        maximum = X(id,a);
        idm = (id,a);
      }
    }
  }
  return idm;
}

proc tropic(A:[],B:[]) {
  var dom: domain(2) = {A.domain.dim(1),B.domain.dim(2)};
  var sps = CSRDomain(dom);
//  var BT = transpose(B);
  var T: [sps] real;
  for (i,j) in dom {
    var t: Timer;
    t.start();
    var mini = 100000000: real;
    if A.domain.member((i,j)) {
      mini = min(mini,A(i,j));
    }
    if B.domain.member((i,j)) {
      mini = min(mini,B(i,j));
    }
    for w in A.domain.dimIter(2,i) {
      if B.domain.member((w,j)) {
        mini = min(mini,A(i,w) + B(w,j));
      }
    }
    t.stop();
    if mini < 100000000 {
      sps += (i,j);
      T(i,j) = mini;
      writeln((i,j));
      writeln("",t.elapsed());
    }
  }
  return T;
}

proc tropic_(A:[],B:[]) {
  var dom: domain(2) = {A.domain.dim(1),B.domain.dim(2)};
  var sps = CSRDomain(dom);
  var BT = transpose(B);
  var T: [sps] real;
  for (i,j) in dom {
    var mini = 100000000: real;
    var wdom: domain(1);
    var wids: [wdom] int;
    for w in A.domain.dimIter(2,i) {
      wids.push_back(w);
    }
    //writeln(wids);
    for w2 in BT.domain.dimIter(2,j) {
      wids.push_back(w2);
    }
    var wDom: domain(1);
    var witness: [wDom] real;
    for w in wids {
      if A(i,w) > 0 && BT(j,w) > 0 {
        witness.push_back(A(i,w) + BT(j,w));
      }
    }
    if A.domain.member((i,j)) {
      witness.push_back(A(i,j));
    }
    if B.domain.member((i,j)) {
      witness.push_back(B(i,j));
    }
    mini = min reduce witness;
    if mini < 1000000000 {
      sps += (i,j);
      T(i,j) = mini;
    }
  }
  return T;
}

proc sparseEq(A:[], B:[]) {
  var s: bool = true;
  forall (i,j) in A.domain with (&& reduce s) {
    s &&= A(i,j) == B(i,j);
  }
  return s;
}

proc tropicLimit(A:[] real,B:[] real): A.type {
 var R = tropic(A,B);
 if A.domain == R.domain {
   if sparseEq(R,A) {
     return R;
   } else {
     return tropicLimit(R,B);
   }
 } else {
   return tropicLimit(R,B);
 }
}

/*
 Build a random sparse matrix.  Good for testing;
 */
proc generateRandomSparseMatrix(size: int, sparsity: real) {
  const D: domain(2) = {1..size, 1..size};
  var SD: sparse subdomain(D) dmapped CS();
  var R: [SD] real;
  var da: domain(1) = {1..size};
  var array: [da] int = 1..size;
  var dom: domain(1) = {1..0};
  var indices: [dom] 2*int;
  var N = floor(size*(1-sparsity)): int;
  forall i in array {
    forall j in array {
      indices.push_back((i,j));
    }
  }
  shuffle(indices);
  var sparseids = indices[1..N];
  SD.bulkAdd(sparseids);
  forall (i,j) in sparseids {
    R(i,j) = 1;
  }
  //ndices = zip(array1, array2)
  return R;
}

/*
 General sparsity function for any `real` matrix.
 */
proc sparsity(X) {
  const d = X.shape[1]:real * X.shape[2]: real;
  var i: real = 0.0;
  for ij in X.domain {
    i += 1.0;
  }
  return i / d;
}

proc elemMult(A: [], B: []) {
  var dom: domain(2) = {A.domain.dim(1),A.domain.dim(2)};
  var sps = CSRDomain(dom);
  var C: [sps] real;
  for (i,j) in A.domain {
    if B.domain.member((i,j)) {
      sps += (i,j);
      C(i,j) = A(i,j) * B(i,j);
    }
  }
  return C;
}

/*
 Error classes
 */
class NumSuchError : Error {
  proc init() {
    super.init();
    this.complete();
  }
  proc message() {
    return "Generic NumSuch Error";
  }
}

proc identityMat(n: int) {
  var dom: domain(2) = {1..n,1..n};
  var sps = CSRDomain(dom);
  var I: [sps] real;
  for i in 1..n {
    sps += (i,i);
    I(i,i) = 1;
  }
  return I;
}

/*
 Used to indidcate dimension mismatches on NamedMatrices and vectors
 */
class DimensionMatchError : NumSuchError {
  var expected: int,
      actual: int;

  proc init(expected: int, actual:int) {
    super.init();
    this.complete();
    this.expected = expected;
    this.actual = actual;
  }

  proc message() {
    return "Error matching dimensions.  Expected: " + this.expected + " Actual: " + this.actual;
  }
}