Restore previous encoding of chpl_startTrackingMemory, to avoid a race in chplmemtrack.c

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@20880 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// MemTracking.chpl
//
pragma "no use ChapelStandard"
module MemTracking
{
  config const
    memTrack: bool = false,
    memStats: bool = false,
    memLeaks: bool = false,
    memLeaksTable: bool = false,
    memMax: int = 0,
    memThreshold: int = 0,
    memLog: string = "";

  pragma "no auto destroy"
  config const
    memLeaksLog: string = "";

  proc chpl_startTrackingMemory() {
    if Locales[0] == here then
      coforall loc in Locales do
//
// This code does not work because there is a race condition between starting
// memory tracking on node 0 and the allocation/deallocation of the task list
// at the end of _waitEndCount().
// Running the enable code on node 0 without "on loc do" causes the execution of the primitive
// and the coforall cleanup to run serially (in that order), thus avoiding the race.
//vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
//        on loc do
//          __primitive("chpl_setMemFlags", memTrack, memStats, memLeaks, memLeaksTable, memMax, memThreshold, memLog, memLeaksLog);
//^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
//
        if loc == here {
          __primitive("chpl_setMemFlags", memTrack, memStats, memLeaks, memLeaksTable, memMax, memThreshold, memLog, memLeaksLog);
        } else on loc {
          __primitive("chpl_setMemFlags", memTrack, memStats, memLeaks, memLeaksTable, memMax, memThreshold, memLog, memLeaksLog);
        }
  }

}

// MemTracking.chpl
//
pragma "no use ChapelStandard"
module MemTracking
{
  config const
    memTrack: bool = false,
    memStats: bool = false,
    memLeaks: bool = false,
    memLeaksTable: bool = false,
    memMax: int = 0,
    memThreshold: int = 0,
    memLog: string = "";

  pragma "no auto destroy"
  config const
    memLeaksLog: string = "";

  proc chpl_startTrackingMemory() {
    if Locales[0] == here then
      coforall loc in Locales do
        on loc do
          __primitive("chpl_setMemFlags", memTrack, memStats, memLeaks, memLeaksTable, memMax, memThreshold, memLog, memLeaksLog);
  }

}


Change a comment to explain VDebugTree and friends.

/*
 * Copyright 2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*

   Support for the visualization tool "chplvis"

   This module provides access to and enables hooks to dump out
   task and communication information for post-run visualization
   of the tasks and communication.

*/
module VisualDebug
{

  use String;

  private extern proc chpl_now_time():real;

  //
  // Data Generation for the Visual Debug tool  (offline)
  //

  private extern proc chpl_vdebug_start ( rootname: c_string, time:real);

  private extern proc chpl_vdebug_stop ();

  private extern proc chpl_vdebug_tag ( tagname: c_string);

  private extern proc chpl_vdebug_pause ();

  private extern proc chpl_vdebug_nolog ();

/* Instead of using a "coforall l in Locales" which is an O(n) operation
   at the present time, vis_op, hc_id2com, and VDebugTree implement
   a hyper-cube broadcast tree to execute on all locales.  The vis_op
   is used to select which of the above routine to run at each locale.
   This code is O(log n), n the number of Locales.
*/

pragma "no doc"
  enum vis_op {v_start, v_stop, v_tag, v_pause};

private iter hc_id2com ( id: int, off: int ) {
   var offset = off;
   var ix = 1;
   while (offset > 0) {
      yield (id+offset, ix);
      offset = offset >> 1;
      ix = ix + 1;
   }
}

private proc VDebugTree (what: vis_op, name: string, time: real, id: int = 0,
                   n: int = numLocales, off: int = -1) {
      var offset = 1;
      if off < 0 then
         while offset >> 1 + id < n do offset = offset << 1;
      else
         offset = off;

      chpl_vdebug_nolog();
      coforall (rid, shift) in hc_id2com(id, offset) do
         if rid < n then
             on Locales[rid] do VDebugTree (what, name, time, rid, n, offset >> shift);

     /* Do the op at the root  */
     select what {
         when vis_op.v_start    do chpl_vdebug_start (name.c_str(), time);
         when vis_op.v_stop     do chpl_vdebug_stop ();
         when vis_op.v_tag      do chpl_vdebug_tag (name.c_str());
         when vis_op.v_pause    do chpl_vdebug_pause ();
     }
}

/*
  Start logging events for VisualDebug.  Open a new set of data
  files, one for each locale, for chplvis.  This routine should be
  called only once for each program.  It creates a directory with the
  rootname and creates the files in that directory.  The files are
  named with the rootname and "-n" is added where n is the locale
  number.

  :arg rootname:  Directory name and rootname for files.
*/
  proc startVdebug ( rootname : string ) {
    var now = chpl_now_time();
    //coforall l in Locales do
    //  on l do chpl_vdebug_start (rootname.c_str(), now);
    VDebugTree (vis_op.v_start, rootname, now);
  }

/*
  Add a tag to the data for chplvis to allow "view points" in the data.

  :arg tagname: name of the tag
*/
  proc tagVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 0);
    //chpl_vdebug_tag (tagname.c_str(), 0);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

/*
  Stop collecting VisualDebug data and close the data files.
*/
  proc stopVdebug () {
    // chpl_vdebug_stop();
    // coforall l in Locales[1..] do
    //  on l do chpl_vdebug_stop();
    VDebugTree (vis_op.v_stop, "", 0);
  }

/*
  Suspend collection of VisualDebug data.
*/
  proc pauseVdebug () {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 1);
    //chpl_vdebug_tag(tagname.c_str(), 1);
    VDebugTree (vis_op.v_pause, "", 0);
  }

/*
  Resume collection of VisualDebug data for chplvis after a pauseVdebug().
  This also generates a tag record visible by chplvis.

  :arg tagname: name of the tag
*/
  proc resumeVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_resume ();
    //chpl_vdebug_resume(tagname);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

}

/*
 * Copyright 2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*

   Support for the visualization tool "chplvis"

   This module provides access to and enables hooks to dump out
   task and communication information for post-run visualization
   of the tasks and communication.

*/
module VisualDebug
{

  use String;

  private extern proc chpl_now_time():real;

  //
  // Data Generation for the Visual Debug tool  (offline)
  //

  private extern proc chpl_vdebug_start ( rootname: c_string, time:real);

  private extern proc chpl_vdebug_stop ();

  private extern proc chpl_vdebug_tag ( tagname: c_string);

  private extern proc chpl_vdebug_pause ();

  private extern proc chpl_vdebug_nolog ();

/* Tree "coforall procedure .... calls one of the above routines */

pragma "no doc"
  enum vis_op {v_start, v_stop, v_tag, v_pause};

private iter hc_id2com ( id: int, off: int ) {
   var offset = off;
   var ix = 1;
   while (offset > 0) {
      yield (id+offset, ix);
      offset = offset >> 1;
      ix = ix + 1;
   }
}

private proc VDebugTree (what: vis_op, name: string, time: real, id: int = 0,
                   n: int = numLocales, off: int = -1) {
      var offset = 1;
      if off < 0 then
         while offset >> 1 + id < n do offset = offset << 1;
      else
         offset = off;

      chpl_vdebug_nolog();
      coforall (rid, shift) in hc_id2com(id, offset) do
         if rid < n then
             on Locales[rid] do VDebugTree (what, name, time, rid, n, offset >> shift);

     /* Do the op at the root  */
     select what {
         when vis_op.v_start    do chpl_vdebug_start (name.c_str(), time);
         when vis_op.v_stop     do chpl_vdebug_stop ();
         when vis_op.v_tag      do chpl_vdebug_tag (name.c_str());
         when vis_op.v_pause    do chpl_vdebug_pause ();
     }
}

/*
  Start logging events for VisualDebug.  Open a new set of data
  files, one for each locale, for chplvis.  This routine should be
  called only once for each program.  It creates a directory with the
  rootname and creates the files in that directory.  The files are
  named with the rootname and "-n" is added where n is the locale
  number.

  :arg rootname:  Directory name and rootname for files.
*/
  proc startVdebug ( rootname : string ) {
    var now = chpl_now_time();
    //coforall l in Locales do
    //  on l do chpl_vdebug_start (rootname.c_str(), now);
    VDebugTree (vis_op.v_start, rootname, now);
  }

/*
  Add a tag to the data for chplvis to allow "view points" in the data.

  :arg tagname: name of the tag
*/
  proc tagVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 0);
    //chpl_vdebug_tag (tagname.c_str(), 0);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

/*
  Stop collecting VisualDebug data and close the data files.
*/
  proc stopVdebug () {
    // chpl_vdebug_stop();
    // coforall l in Locales[1..] do
    //  on l do chpl_vdebug_stop();
    VDebugTree (vis_op.v_stop, "", 0);
  }

/*
  Suspend collection of VisualDebug data.
*/
  proc pauseVdebug () {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_tag (tagname.c_str(), 1);
    //chpl_vdebug_tag(tagname.c_str(), 1);
    VDebugTree (vis_op.v_pause, "", 0);
  }

/*
  Resume collection of VisualDebug data for chplvis after a pauseVdebug().
  This also generates a tag record visible by chplvis.

  :arg tagname: name of the tag
*/
  proc resumeVdebug ( tagname : string ) {
    //coforall l in Locales[1..] do
    //  on l do chpl_vdebug_resume ();
    //chpl_vdebug_resume(tagname);
    VDebugTree (vis_op.v_tag, tagname, 0);
  }

}


Massage FFTW primer comments

Massage FFTW primer comments

Rework FFTW primer comments to try and better fit into an 80-column
format, format some things more consistently, add a mention of the
multithreaded implementation, etc.

/*
  Example usage of the FFTW module in Chapel. This particular file
  demonstrates the single-threaded version of the code.  In order to
  switch to a multi-threaded version of the code, substitute FFTW_MT
  for FFTW in the 'use' statement just below this comment (see the
  FFTW_MT module's documentation for further details).

  If FFTW is in your standard include/library paths, compile this code
  using:

    chpl testFFTW.chpl

  Otherwise, use the following (where $FFTW_DIR points to your FFTW
  installation):

    chpl testFFTW.chpl -I$FFTW_DIR/include -L$FFTW_DIR/lib

  The FFTW module uses the FFTW3 API and currently just implements the
  basic, double-precision interface.  We will assume that the reader
  is familiar with using FFTW; more details are at
  http://www.fftw.org.

  The code computes a series of 1D, 2D and 3D transforms, exercising
  the complex<->complex and real<->complex transforms (both in- and
  out-of-place). The output of the code should be a series of small
  numbers ( <= 10^-13 ); see testFFTW.good for example values, though
  it is possible that your values may differ in practice.

  The input data for these tests is in arr{1,2,3}d.dat. The format of
  these files is documented below.
*/

use FFTW;

//
// Perform the tests and then cleanup.
//
proc main() {
  testAllDims();
  cleanup();
}

//
// A helper function that invokes the test for each rank.
//
proc testAllDims() {
  writeln("1D");
  runtest(1, "arr1d.dat");
  writeln("2D");
  runtest(2, "arr2d.dat");
  writeln("3D");
  runtest(3, "arr3d.dat");
}

//
// This is main test code, parametrized by the number of dimensions,
// 'ndim'.  'fn' is the filename of the file that contains the test
// data.
//
proc runtest(param ndim : int, fn : string) {
  var dims : ndim*int(32);

  /* We define a number of different domains below, corresponding to
     complex input/output arrays, real (input)->complex (output)
     out-of-place arrays, and real<->complex in-place arrays.

     The domains are as follows:

     * D : for complex<->complex transforms. This domain is also
           used for the real array in a real->complex out-of-place
           transform.

     * cD : for the complex array in a real<->complex out-of-place
            transform.

     * rD : for the real array in a real<->complex in-place
            transform. This includes the padding needed by the
            in-place transform. D is a sub-domain of this, and can be
            used to extract the real array, without padding.

     * reD, imD : Utility domains that access the real/complex parts
                  of the complex array in a real<->complex in-place
                  transform.
  */
  var D : domain(ndim);
  var rD,cD,reD,imD : domain(ndim,int,true);

  /* Read in the arrays from the file below. 'A' and 'B' are the
     arrays that will be used in the FFTW calls, while 'goodA' and
     'goodB' store the true values.

     Note that 'goodA' is all real, allowing us to reuse the same
     array for the real<->complex tests.

     The file format is as follows (all little-endian):

        [ndim] int32 : dimensions of the array

        [narr] real64 : the real components of A (imaginary components
                        are all zero). narr = N_1*N_2*..*N_ndim
                        (product of dimensions)

        [narr] complex128 : the components of B (stored as
                            real-imaginary pairs).

        The arrays are all stored in row-major order, as in C.
  */
  var A,B,goodA,goodB : [D] complex(128);
  {
    var f = open(fn,iomode.r).reader(kind=iokind.little);
    //
    // Read in dimensions
    //
    for ii in 1..ndim {
      f.read(dims(ii));
    }

    //
    // The code below sets the domain D handling the different rank cases
    //
    select ndim {
      when 1 do D = 0.. #dims(1);
      when 2 do D = {0.. #dims(1), 0.. #dims(2)};
      when 3 do D = {0.. #dims(1), 0.. #dims(2), 0.. #dims(3)};
    }

    //
    // Read in the arrays
    //
    for val in goodA {
      f.read(val.re);
      val.im = 0;
    }
    for val in goodB {
      f.read(val);
    }
    f.close();
    writeln("Data read...");
  }

  /* Now set the remaining domains.

     Refer to the FFTW documentation on the storage order for the
     in-place transforms (Sec 2.4 and 4.3.4).
  */
  select ndim {
    when 1 {
      var ldim = dims(1)/2 + 1;
      //
      // Domains for real FFT
      //
      rD = 0.. #(2*ldim); // Padding to do in-place transforms
      cD = 0.. #ldim;
      //
      // Define domains to extract the real and imaginary parts for
      // in-place transforms
      //
      reD = rD[0..(2*ldim-1) by 2];
      imD = rD[1..(2*ldim-1) by 2];
    }
    when 2 {
      //
      // Domains for real FFT
      //
      var ldim = dims(2)/2+1;
      rD = {0.. #dims(1),0.. #(2*ldim)}; // Padding to do in-place transforms
      cD = {0.. #dims(1),0.. #ldim};
      //
      // Define domains to extract the real and imaginary parts for
      // in-place transforms
      //
      reD = rD[..,0..(2*ldim-1) by 2];
      imD = rD[..,1..(2*ldim-1) by 2];
    }
    when 3 {
      //
      // Domains for real FFT
      //
      var ldim = dims(3)/2+1;
      rD = {0.. #dims(1),0.. #dims(2),0.. #(2*ldim)}; // Padding to do in-place transforms
      cD = {0.. #dims(1),0.. #dims(2),0.. #ldim};
      //
      // Define domains to extract the real and imaginary parts for
      // in-place transforms
      //
      reD = rD[..,..,0..(2*ldim-1) by 2];
      imD = rD[..,..,1..(2*ldim-1) by 2];
    }
  }

  // FFTW does not normalize inverse transforms, so just compute the
  // normalization constant.

  var norm = * reduce dims;

  /* We start the FFT tests below. The structure is the same :
      * Define plans for forward and reverse transforms.
      * Execute forward transform A -> B.
      * Compare with goodB.
      * Execute reverse transform B -> A and normalize.
      * Compare with goodA.
      * Cleanup plans
  */

  /* Complex<->complex out-of-place transform.

     Unlike the basic FFTW interface, we do not have specific 1D/2D/3D
     planner routines.  For the complex <-> complex case, the
     dimensions of the array are inferred automatically.
  */
	var fwd = plan_dft(A, B, FFTW_FORWARD, FFTW_ESTIMATE);
	var rev = plan_dft(B, A, FFTW_BACKWARD, FFTW_ESTIMATE);
        //
	// Test forward and reverse transform
        //
	A = goodA;
	execute(fwd);
	printcmp(B,goodB);
	execute(rev);
	A /= norm;
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Complex <-> complex in-place transform.

     This is the same calling sequence as above, but using the
     in-place versions of the routine.
  */
	fwd = plan_dft(A, FFTW_FORWARD, FFTW_ESTIMATE);
	rev = plan_dft(A, FFTW_BACKWARD, FFTW_ESTIMATE);
        //
	// Test forward and reverse transform
        //
	A = goodA;
	execute(fwd);
	printcmp(A,goodB);
	execute(rev);
	A /= norm; // FFTW does an unnormalized transform
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Real <-> complex out-of-place transform

     As with FFTW, these use 'r2c' and 'c2r' suffixes to define the
     direction of the transform.

     plan_dft_r2c/plan_dft_c2r are overloaded; for the out-of-place
     transforms, they infer the dimensions from the sizes of the
     arrays passed in.
  */
  var rA : [D] real(64); // No padding for an out-of-place transform
  var cB : [cD] complex(128);
  fwd = plan_dft_r2c(rA,cB,FFTW_ESTIMATE);
  rev = plan_dft_c2r(cB,rA,FFTW_ESTIMATE);
  rA[D] = goodA.re;
  execute(fwd);
  printcmp(cB,goodB[cD]);
  execute(rev);
  rA /= norm;
  printcmp(rA[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

  /* Real <-> complex in-place transform

     In this case, the first argument to the planning routines is the
     domain of the *real* array WITHOUT padding (in both r2c and c2r
     cases). This breaks the ambiguity of whether the leading
     dimension of the real array is even or odd.

     This design decision was motivated by the fact that the user has
     likely already defined a domain to extract the unpadded real
     array from the fully padded array (or that such a domain is
     intrinsically useful).

     For both the r2c and c2r transforms, a real array is passed in.
  */
  var rA2 : [rD] real(64);
  fwd = plan_dft_r2c(D,rA2,FFTW_ESTIMATE);
  rev = plan_dft_c2r(D,rA2,FFTW_ESTIMATE);
  rA2[D] = goodA.re;
  execute(fwd);
  printcmp(rA2[reD],goodB[cD].re); // Check the real and complex parts separately.
  printcmp(rA2[imD],goodB[cD].im);
  execute(rev);
  rA2 /= norm;
  printcmp(rA2[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

  /* Real <-> complex in-place transform

     This is similar to the previous case, except that we pass in a
     complex array, instead of a real array. This can get a little
     ugly, so we just reverse engineer the previous case.

     Note that we reuse the rA2 and cB arrays, since they're the
     correct sizes.
  */
  fwd = plan_dft_r2c(D,cB,FFTW_ESTIMATE);
  rev = plan_dft_c2r(D,cB,FFTW_ESTIMATE);
  //
  // Zero out rA2 to ensure that anything extraneous in the padding
  // doesn't get passed in.
  //
  rA2 = 0.0;
  rA2[D] = goodA.re;
  cB.re = rA2[reD]; // Fill the complex array
  cB.im = rA2[imD];
  execute(fwd);
  printcmp(cB.re,goodB[cD].re); // Check the real and complex parts separately.
  printcmp(cB.im,goodB[cD].im);
  execute(rev);
  cB /= norm;
  //
  // Pull everything back out to the real array for simplicity
  //
  rA2 = 0.0;
  rA2[reD] = cB.re;
  rA2[imD] = cB.im;
  printcmp(rA2[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

}

//
// Utility function to print the maximum absolute deviation between
// values computed by this code, and "truth". The true values are
// computed using Mathematica v10
//
proc printcmp(x, y) {
  var err = max reduce abs(x-y);
  writeln(err);
}

/*

Example usage of the FFTW module in Chapel. This particular file
demonstrates the single-threaded version of the code.

To compile this code :

   chpl testFFTW.chpl

If FFTW is not on your include/library paths, you can do

   chpl testFFTW.chpl -I$FFTW_DIR/include -L$FFTW_DIR/lib

where $FFTW_DIR points to your FFTW installation.

Note that you do not need to explicitly include the header or library
files; the module uses an experimental feature of the Chapel compiler
to automatically include these.

The FFTW module uses the FFTW3 API, and currently just implements the
basic, double-precision interface.  We will assume that the reader is
familiar with using FFTW; more details are at http://www.fftw.org.

The code computes a series of 1,2 and 3D transforms, exercising the
complex <-> complex, and real<->complex transforms (both in and out of
place). The output of the code should be a series of small numbers (
<= 10^-13); see testFFTW.good for example values (it is possible that
your values might differ).

The data for these tests is in arr{1,2,3}d.dat. The format of these
files is documented below.
*/

use FFTW;

// Do the tests and then cleanup.
proc main() {
  testAllDims();
  cleanup();
}

// A helper function that calls all the tests.
proc testAllDims() {
  writeln("1D");
  runtest(1, "arr1d.dat");
  writeln("2D");
  runtest(2, "arr2d.dat");
  writeln("3D");
  runtest(3, "arr3d.dat");
}

/* This is main test code, parametrized on the number of dimensions ndim.
   fn is the file that contains the test data.
*/
proc runtest(param ndim : int, fn : string) {
  var dims : ndim*int(32);

  /* We define a number of different domains below, corresponding to complex input/output arrays,
     real (input) -> complex (output) out-of-place arrays, and real <-> complex in-place arrays.

     The domains are :
     * D : for complex <-> complex transforms. This domain is also used for the real array in
           a real->complex out-of-place transform.
     * cD : for the complex array in a real <-> complex out-of-place transform.
     * rD : for the real array in a real <-> complex in-place transform. This includes the padding needed
            by the in-place transform. D is a sub-domain of this, and can be used to extract the real array,
            without padding.
     * reD, imD : Utility domains that access the real/complex parts of the complex array in a real<->complex in-place transform.
  */
  var D : domain(ndim);
  var rD,cD,reD,imD : domain(ndim,int,true);

  /* Read in the arrays from the file below. A,B are the arrays that will be used in the FFTW calls,
     goodA and goodB store the true values.

     Note that goodA is all real, allowing us to reuse the same array for the real<->complex tests.

     The file format is as follows (all little-endian):
        [ndim] int32 : dimensions of the array
        [narr] real64 : the real components of A (imaginary components are all zero). narr = N_1*N_2*..*N_ndim (product of dimensions)
        [narr] complex128 : the components of B (stored as real-imaginary pairs).

        The arrays are all stored in C (row-major) order.
  */
  var A,B,goodA,goodB : [D] complex(128);
  {
    var f = open(fn,iomode.r).reader(kind=iokind.little);
    // Read in dimensions
    for ii in 1..ndim {
      f.read(dims(ii));
    }
    /* The code below sets the domain D handling the different rank cases */
    select ndim {
      when 1 do D = 0.. #dims(1);
      when 2 do D = {0.. #dims(1), 0.. #dims(2)};
      when 3 do D = {0.. #dims(1), 0.. #dims(2), 0.. #dims(3)};
    }

    // Read in the arrays
    for val in goodA {
      f.read(val.re);
      val.im = 0;
    }
    for val in goodB {
      f.read(val);
    }
    f.close();
    writeln("Data read...");
  }

  /* Now set the remaining domains.

     The reader is referred to the FFTW documentation on the storage order for the in-place
     transforms (Sec 2.4 and 4.3.4).
  */
  select ndim {
    when 1 {
      var ldim = dims(1)/2 + 1;
      // Domains for real FFT
      rD = 0.. #(2*ldim); // Padding to do in place transforms
      cD = 0.. #ldim;
      // Define domains to extract the real and imaginary parts for inplace transforms
      reD = rD[0..(2*ldim-1) by 2];
      imD = rD[1..(2*ldim-1) by 2];
    }
    when 2 {
      // Domains for real FFT
      var ldim = dims(2)/2+1;
      rD = {0.. #dims(1),0.. #(2*ldim)}; // Padding to do in place transforms
      cD = {0.. #dims(1),0.. #ldim};
      // Define domains to extract the real and imaginary parts for in place transforms
      reD = rD[..,0..(2*ldim-1) by 2];
      imD = rD[..,1..(2*ldim-1) by 2];
    }
    when 3 {
      // Domains for real FFT
      var ldim = dims(3)/2+1;
      rD = {0.. #dims(1),0.. #dims(2),0.. #(2*ldim)}; // Padding to do in place transforms
      cD = {0.. #dims(1),0.. #dims(2),0.. #ldim};
      // Define domains to extract the real and imaginary parts for in place transforms
      reD = rD[..,..,0..(2*ldim-1) by 2];
      imD = rD[..,..,1..(2*ldim-1) by 2];
    }
  }

  // FFTW does not normalize inverse transform, so just compute the normalization constant.
  var norm = * reduce dims;

  /* We start the FFT tests below. The structure is the same :
      * Define plans for forward and reverse transforms.
      * Execute forward transform A -> B.
      * Compare with goodB.
      * Execute reverse transform B -> A and normalize.
      * Compare with goodA.
      * Cleanup plans
  */

  /* Complex <-> complex out-of-place transform.

     Unlike the basic FFTW interface, we do not have specific 1,2,3d planner routines.
     For the complex <-> complex case, the dimensions of the array are inferred automatically.

  */
	var fwd = plan_dft(A, B, FFTW_FORWARD, FFTW_ESTIMATE);
	var rev = plan_dft(B, A, FFTW_BACKWARD, FFTW_ESTIMATE);
	// Test forward and reverse transform
	A = goodA;
	execute(fwd);
	printcmp(B,goodB);
	execute(rev);
	A /= norm;
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Complex <-> complex in-place transform.

     This is the same calling sequence as above, but using the
     in-place versions of the routine.
  */
	fwd = plan_dft(A, FFTW_FORWARD, FFTW_ESTIMATE);
	rev = plan_dft(A, FFTW_BACKWARD, FFTW_ESTIMATE);
	// Test forward and reverse transform
	A = goodA;
	execute(fwd);
	printcmp(A,goodB);
	execute(rev);
	A /= norm; // FFTW does an unnormalized transform
	printcmp(A,goodA);
	destroy_plan(fwd);
	destroy_plan(rev);

  /* Real <-> complex out-of-place transform

     As with FFTW, these use an r2c and c2r suffix to define the direction of the transform.

     plan_dft_r2c/plan_dft_c2r are overloaded; for the out-of-place transforms, they infer the
     dimensions from the sizes of the arrays passed in.
  */
  var rA : [D] real(64); // No padding for an out-of place transform
  var cB : [cD] complex(128);
  fwd = plan_dft_r2c(rA,cB,FFTW_ESTIMATE);
  rev = plan_dft_c2r(cB,rA,FFTW_ESTIMATE);
  rA[D] = goodA.re;
  execute(fwd);
  printcmp(cB,goodB[cD]);
  execute(rev);
  rA /= norm;
  printcmp(rA[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

  /* Real <-> complex in-place transform

     In this case, the first argument to the planning routines is the domain of the *real*
     array WITHOUT padding (in both r2c and c2r cases). This breaks the ambiguity of whether the leading
     dimension of the real array is even or odd.

     This design decision was motivated by the fact that the user has likely already defined a
     domain to extract the unpadded real array from the fully padded array (or that such a domain
     is intrinsically useful).

     For both the r2c and c2r transforms, a real array is passed in.

  */
  var rA2 : [rD] real(64);
  fwd = plan_dft_r2c(D,rA2,FFTW_ESTIMATE);
  rev = plan_dft_c2r(D,rA2,FFTW_ESTIMATE);
  rA2[D] = goodA.re;
  execute(fwd);
  printcmp(rA2[reD],goodB[cD].re); // Check the real and complex parts separately.
  printcmp(rA2[imD],goodB[cD].im);
  execute(rev);
  rA2 /= norm;
  printcmp(rA2[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

  /* Real <-> complex in-place transform

     This is similar to the previous case, except that we pass in a complex array, instead
     of a real array. This can get a little ugly, so we just reverse engineer the previous case.

     Note that we reuse the rA2 and cB arrays, since they're the correct sizes.
  */
  fwd = plan_dft_r2c(D,cB,FFTW_ESTIMATE);
  rev = plan_dft_c2r(D,cB,FFTW_ESTIMATE);
  // Zero out rA2 to ensure that anything extraneous in the padding doesn't get passed in.
  rA2 = 0.0;
  rA2[D] = goodA.re;
  cB.re = rA2[reD]; // Fill the complex array
  cB.im = rA2[imD];
  execute(fwd);
  printcmp(cB.re,goodB[cD].re); // Check the real and complex parts separately.
  printcmp(cB.im,goodB[cD].im);
  execute(rev);
  cB /= norm;
  // Pull everything back out to the real array for simplicity
  rA2 = 0.0;
  rA2[reD] = cB.re;
  rA2[imD] = cB.im;
  printcmp(rA2[D],goodA.re);
  destroy_plan(fwd);
  destroy_plan(rev);

}

/* Utility function to print the maximum absolute deviation between
   values computed by this code, and "truth". The true values are
   computed using Mathematica v10 */

proc printcmp(x, y) {
  var err = max reduce abs(x-y);
  writeln(err);
}