Finished documenting stuff, now playing with default flag values.

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Wrapper for FFTW version 3.

/*
  FFT computations via key routines from FFTW (version 3)

  This module defines Chapel wrappers for key routines from FFTW
  (http://www.fftw.org), version 3.  Over time, the intention is to
  expand this module to support additional routines, prioritizing
  based on requests and feedback from users.  Also, see the
  :mod:`FFTW_MT` module which provides support for calls to the
  multi-threaded FFTW implementation.

  To use this module:

  1. Ensure that FFTW (version 3) is installed on your system and that
     the header and library files (e.g., fftw3.h, libfftw3.*) are
     either installed in a standard system location or that your C
     compiler's environment variables are set up to find them.

  2. Add ``use FFTW;`` to your Chapel code.

  3. Compile and run your Chapel program as usual.

  As in standard FFTW usage, the flow is to:

  1. Create plan(s) using the :proc:`plan_dft* <plan_dft>` routines.

  2. Execute the plan(s) one or more times using :proc:`execute`.

  3. Destroy the plan(s) using :proc:`destroy_plan`.

  4. Call :proc:`cleanup`.
*/

//
// - How do we feel about FFTW_ALLCAPS names given that the routines
//   themselves don't have fftw_ prefixes, that they're defined as
//   c_int's, and that they're all-caps?
//
// - Related question about fftw_ types taking type c_int, though
//   maybe we don't care that much (as compared to other interfaces)
//   since symbolic values are taken in? (i.e., we won't run into
//   32- vs. 64-bit issues...)
//

module FFTW {

  use SysCTypes;
  use "fftw3.h", "-lfftw3";

  /*
    An opaque type used to store and reuse FFTW plans across multiple
    routines.
  */
  extern type fftw_plan; // opaque type

  // Planner functions
  // Complex : 4.3.1
  // NOTE : We pass in arrays using ref

  // TODO: Can we have the plan_dft() routine below take in native
  // Chapel types without changing the external C constants from
  // type c_int?  Probably given that Chapel's int will be >=
  // C's int for the forseeable future.  See also the TODO above
  // about whether we'd want to represent those constants using
  // more native Chapel types anyway.

  /*
    Creates a plan for a complex-to-complex DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `complex(128)`

    :arg output: The output array, whose rank must match the input array's
    :type output: [] `complex(128)`

    :arg sign: :const:`FFTW_FORWARD` or :const:`FFTW_BACKWARD`
    :type sign: c_int

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft(input: [] complex(128), output: [] complex(128),
                 sign: c_int, flags: c_uint = 0:c_int) : fftw_plan
  {
    param rank = input.rank;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size.safeCast(c_int);

    return C_FFTW.fftw_plan_dft(rank.safeCast(c_int), dims, input,
                                     output, sign, flags);
  }

  // Real-to-complex and complex-to-real planning routines There are
  // two cases that we treat independently here : in-place and
  // out-of-place transforms

  /*
    Create a plan for a real-to-complex, out-of-place DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `real(64)`

    :arg output: The output array, whose rank must match the input array's
    :type output: [] `complex(128)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft_r2c(input : [] real(64), output : [] complex(128), flags :c_uint = 0:c_int) : fftw_plan
  {
    param rank = input.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, input, output, flags);
  }

  // In-place routines, note that these take in the true leading dimension
  // TODO : We should put in checks to see that the sizes are consistent
  // TODO : We should check on types...
  /*
    Create a plan for a real-to-complex, in-place DFT.

    :arg realDom: Describes the indices of the 'real' view of the array
    :type realDom: `domain`

    :arg arr: The array to be used as the in-place input/output array.
    :type arr: [] `T` where `T` is of type `real` or `complex`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
   */
  proc plan_dft_r2c(realDom : domain, arr : [] ?t, flags : c_uint = 0:c_int) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, arr, arr, flags);
  }

  /*
    Create a plan for a complex-to-real, out-of-place DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `complex(128)`

    :arg output: The output array, whose rank must match the input array's
    :type output: [] `real(64)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft_c2r(input : [] complex(128), output : [] real(64), flags :c_uint=0:c_int) : fftw_plan
  {
    param rank = output.rank: c_int; // The dimensions are that of the real array

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = output.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, input, output, flags);
  }

  /*
    Create a plan for a complex-to-real, in-place DFT.

    :arg realDom: Describes the indices of the 'real' view of the array
    :type realDom: `domain`

    :arg arr: The array to be used as the in-place input/output array.
    :type arr: [] `T` where `T` is of type `real` or `complex`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
   */
  proc plan_dft_c2r(realDom : domain, arr: [] ?t, flags : c_uint=0:c_int) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, arr, arr, flags);
  }

  // Using plans

  /*
    Execute an FFTW plan.

    :arg plan: The plan to execute, as computed by a `plan_dft*()` routine.
    :type plan: fftw_plan
  */
  proc execute(const plan: fftw_plan) {
   C_FFTW.fftw_execute(plan);
  }

  /*
    Destroy an FFTW plan.

    :arg plan: The plan to destroy
    :type plan: fftw_plan
  */
  proc destroy_plan(plan: fftw_plan) {
    C_FFTW.fftw_destroy_plan(plan);
  }

  /*
    Clean up FFTW overall.
  */
  proc cleanup() {
    C_FFTW.fftw_cleanup();
  }

  // Define the various planner flags
  // See Sec. 4.3.2 of FFTW manual "Planner Flags"

  //
  // TODO: Is it weird that we use FFTW_ names here, but not in
  // the routines?  Should we wrap the external flags so that
  // they can be ints or enums and avoid the all-caps C scheme
  // or is the familiarity better?
  //

  // Direction flags

  /* Request a forward transform (i.e., use a negative exponent in the tranform). */
  extern const FFTW_FORWARD : c_int;
  /* Request a backward transform (i.e., use a positive exponent in the transofmr). */
  extern const FFTW_BACKWARD : c_int;

  // Planning-rigor flags

  /* Specify that a simple heuristic should be used to pick a plan
     quickly.  This will prevent the input/output arrays from being
     overwritten during planning. */
  extern const FFTW_ESTIMATE : c_uint;

  /* Specify that FFTW should try and find an optimized plan by
     computing several FFTs and measuring their execution time.  This
     option is the default planning option and can consume some
     time. */
  extern const FFTW_MEASURE : c_uint;

  /* Specify that FFTW should expend a greater effort finding an
     optimized plan. */
  extern const FFTW_PATIENT : c_uint;

  /* Specify that FFTW should expend an even greater effort finding an
     optimized plan. */
  extern const FFTW_EXHAUSTIVE : c_uint;

  /* This is a special planning mode that is useful for querying
     whether wisdom is available.  When using it, the plan is only
     created when wisdom is available for the given problem; otherwise
     a `null plan` is returned.  This can be combined with other flags
     to create a plan if the wisdom available was created in that mode
     (e.g., ``FFTW_WISDOM_ONLY | FFW_PATIENT``).  For more details on
     this flag and the previous four, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual
  */
  extern const FFTW_WISDOM_ONLY : c_uint;

  // Algorithm-restriction flags

  /* Specify that an out-of-place transform is permitted to overwrite
     its input array with arbitrary data.  This permits more efficient
     algorithms to be used in some cases.  This is the default for
     :proc:`plan_dft_c2r`. */ // NOTE: ...and hc2r once supported...
  extern const FFTW_DESTROY_INPUT : c_uint;

  /* Specify that an out-of-place transform cannot change its input
     array.  This is the default for :proc:`plan_dft` and
     :proc:`plan_dft_r2c`. */
  extern const FFTW_PRESERVE_INPUT : c_uint;

  /* Specify that the algorithm may not impose any unusual alignment
     requirements on the input/output arrays.  This flag should not be
     necessary for current Chapel use since the planner will
     automatically detect such cases.  For more details on this flag
     and the previous two, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual.
  */  // NOTE: But it will be if/when the new-array execute interface is supported
  extern const FFTW_UNALIGNED : c_uint;

  pragma "no doc"
  module C_FFTW {
    extern proc fftw_plan_dft(rank: c_int,
                              n,  // BLC: having trouble being specific
                              in1: [] complex(128),
                              out1: [] complex(128),
                              sign : c_int, c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_r2c(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_c2r(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_execute(const plan : fftw_plan);

    extern proc fftw_destroy_plan(plan : fftw_plan);

    extern proc fftw_cleanup();

    extern proc fftw_init_threads() : c_int;

    extern proc fftw_cleanup_threads();

    extern proc fftw_plan_with_nthreads(n : c_int);
  }
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Wrapper for FFTW version 3.

/*
  FFT computations via key routines from FFTW (version 3)

  This module defines Chapel wrappers for key routines from FFTW
  (http://www.fftw.org), version 3.  Over time, the intention is to
  expand this module to support additional routines, prioritizing
  based on requests and feedback from users.  Also, see the
  :mod:`FFTW_MT` module which provides support for calls to the
  multi-threaded FFTW implementation.

  To use this module:

  1. Ensure that FFTW (version 3) is installed on your system and that
     the header and library files (e.g., fftw3.h, libfftw3.*) are
     either installed in a standard system location or that your C
     compiler's environment variables are set up to find them.

  2. Add ``use FFTW;`` to your Chapel code.

  3. Compile and run your Chapel program as usual.

  As in standard FFTW usage, the flow is to:

  1. Create plan(s) using the :proc:`plan_dft* <plan_dft>` routines.

  2. Execute the plan(s) one or more times using :proc:`execute`.

  3. Destroy the plan(s) using :proc:`destroy_plan`.

  4. Call :proc:`cleanup`.
*/

//
// - How do we feel about FFTW_ALLCAPS names given that the routines
//   themselves don't have fftw_ prefixes, that they're defined as
//   c_int's, and that they're all-caps?
//
// - Related question about fftw_ types taking type c_int, though
//   maybe we don't care that much (as compared to other interfaces)
//   since symbolic values are taken in? (i.e., we won't run into
//   32- vs. 64-bit issues...)
//

module FFTW {

  use SysCTypes;
  use "fftw3.h", "-lfftw3";

  // Define the various planner flags
  // See Sec. 4.3.2 of FFTW manual "Planner Flags"

  //
  // TODO: Is it weird that we use FFTW_ names here, but not in
  // the routines?  Should we wrap the external flags so that
  // they can be ints or enums and avoid the all-caps C scheme
  // or is the familiarity better?
  //

  // Direction flags

  /* Request a forward transform (i.e., use a negative exponent in the tranform). */
  extern const FFTW_FORWARD : c_int;
  /* Request a backward transform (i.e., use a positive exponent in the transofmr). */
  extern const FFTW_BACKWARD : c_int;

  // Planning-rigor flags

  /* Specify that a simple heuristic should be used to pick a plan
     quickly.  This will prevent the input/output arrays from being
     overwritten during planning. */
  extern const FFTW_ESTIMATE : c_uint;

  /* Specify that FFTW should try and find an optimized plan by
     computing several FFTs and measuring their execution time.  This
     option is the default planning option and can consume some
     time. */
  extern const FFTW_MEASURE : c_uint;

  /* Specify that FFTW should expend a greater effort finding an
     optimized plan. */
  extern const FFTW_PATIENT : c_uint;

  /* Specify that FFTW should expend an even greater effort finding an
     optimized plan. */
  extern const FFTW_EXHAUSTIVE : c_uint;

  /* This is a special planning mode that is useful for querying
     whether wisdom is available.  When using it, the plan is only
     created when wisdom is available for the given problem; otherwise
     a `null plan` is returned.  This can be combined with other flags
     to create a plan if the wisdom available was created in that mode
     (e.g., ``FFTW_WISDOM_ONLY | FFW_PATIENT``).  For more details on
     this flag and the previous four, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual
  */
  extern const FFTW_WISDOM_ONLY : c_uint;

  // Algorithm-restriction flags

  /* Specify that an out-of-place transform is permitted to overwrite
     its input array with arbitrary data.  This permits more efficient
     algorithms to be used in some cases.  This is the default for
     :proc:`plan_dft_c2r`. */ // NOTE: ...and hc2r once supported...
  extern const FFTW_DESTROY_INPUT : c_uint;

  /* Specify that an out-of-place transform cannot change its input
     array.  This is the default for :proc:`plan_dft` and
     :proc:`plan_dft_r2c`. */
  extern const FFTW_PRESERVE_INPUT : c_uint;

  /* Specify that the algorithm may not impose any unusual alignment
     requirements on the input/output arrays.  This flag should not be
     necessary for current Chapel use since the planner will
     automatically detect such cases.  For more details on this flag
     and the previous two, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual.
  */  // NOTE: But it will be if/when the new-array execute interface is supported
  extern const FFTW_UNALIGNED : c_uint;

  /*
    'fftw_plan' is an opaque type used by FFTW for storing and passing
    plans between routines.
  */
  extern type fftw_plan; // opaque type

  // Planner functions
  // Complex : 4.3.1
  // NOTE : We pass in arrays using ref

  // TODO: Can we have the plan_dft() routine below take in native
  // Chapel types without changing the external C constants from
  // type c_int?  Probably given that Chapel's int will be >=
  // C's int for the forseeable future.  See also the TODO above
  // about whether we'd want to represent those constants using
  // more native Chapel types anyway.

  /*
    Creates a plan for a complex->complex DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `complex(128)`

    :arg output: The output array, with rank matching the input array's
    :type output: [] `complex(128)`

    :arg sign: :const:`FFTW_FORWARD` or :const:`FFTW_BACKWARD`
    :type sign: c_int

    :arg flags: planning-rigor and/or algorithm-restriction flags
    :type sign: c_int

    :returns: The fftw_plan representing the plan
  */
  proc plan_dft(input: [] complex(128), output: [] complex(128),
                 sign: c_int, flags: c_uint) : fftw_plan
  {
    param rank = input.rank;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size.safeCast(c_int);

    return C_FFTW.fftw_plan_dft(rank.safeCast(c_int), dims, input,
                                     output, sign, flags);
  }

  // Real-to-complex and complex-to-real planning routines There are
  // two cases that we treat independently here : in-place and
  // out-of-place transforms

  // Out-of-place routines
  proc plan_dft_r2c(input : [] real(64), output : [] complex(128), flags :c_uint) : fftw_plan
  {
    param rank = input.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, input, output, flags);
  }

  proc plan_dft_c2r(input : [] complex(128), output : [] real(64),  flags :c_uint) : fftw_plan
  {
    param rank = output.rank: c_int; // The dimensions are that of the real array

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = output.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, input, output, flags);
  }

  // In-place routines, note that these take in the true leading dimension
  // TODO : We should put in checks to see that the sizes are consistent
  // TODO : We should check on types...
  proc plan_dft_r2c(realDom : domain, input : [] ?t, flags : c_uint) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, input, input, flags);
  }
  proc plan_dft_c2r(realDom : domain, input: [] ?t, flags : c_uint) : fftw_plan
  {
    param rank = realDom.rank: c_int;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, input, input, flags);
  }

  // Using plans
  /*
    Executes the given plan.

    :arg plan: The plan to execute, as computed by a plan*() routine.
    :type plan: fftw_plan
  */
  proc execute(const plan: fftw_plan) {
   C_FFTW.fftw_execute(plan);
  }

  /*
    Destroys the given plan.

    :arg plan: The plan to destroy
    :type plan: fftw_plan
  */
  proc destroy_plan(plan: fftw_plan) {
    C_FFTW.fftw_destroy_plan(plan);
  }

  /*
    Cleans up FFTW overall.
  */
  proc cleanup() {
    C_FFTW.fftw_cleanup();
  }

  pragma "no doc"
  module C_FFTW {
    extern proc fftw_plan_dft(rank: c_int,
                              n,  // BLC: having trouble being specific
                              in1: [] complex(128),
                              out1: [] complex(128),
                              sign : c_int, c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_r2c(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_c2r(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_execute(const plan : fftw_plan);

    extern proc fftw_destroy_plan(plan : fftw_plan);

    extern proc fftw_cleanup();

    extern proc fftw_init_threads() : c_int;

    extern proc fftw_cleanup_threads();

    extern proc fftw_plan_with_nthreads(n : c_int);
  }
}


Removed locking from all iterator methods since there appears no need
for it. If there is such a need, then as suggested by Sam, an
iterator would have to make a copy of the accessed data structure
under lock before yielding any value.

Used the new coforall with "parent thread can run child tasks"
capability in building the madness tree.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14138 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/** FTree: The tree data structure used to store the numerical representation
    of a function.

    For 1d Madness, an FTree is a binary tree, for 2d it would
    be a quad tree, 3d an octtree, and so on.  The FTree used here uses level
    and index hashing to store the tree rather than storing connectivity
    information.  So, it is important that you do a has_coeffs() before
    indexing into the tree to be sure you don't create coeffs when you don't
    mean to.
 */

record Node {
    const lvl, idx : int;
    const loc = (lvl+idx)%numLocales;

    def this() {
        return (lvl, idx);
    }

    def path_upwards(node1: Node) {
        var l = idx;
        for n in node1.lvl..lvl by -1 {
           const node = new Node(n, l);
           yield node;
           l /= 2;
        }
    }

    def path_downwards(node1: Node) {
        for n in lvl..node1.lvl-1 {
            const node = new Node(n, node1.idx/(2**(node1.lvl-n)));
            yield node;
        }
    }

    def on_boundary() {
        return (idx < 0 || idx >= 2**lvl);
    }

    def get_children() {
        const child1 = new Node(lvl+1, 2*idx);
        const child2 = new Node(lvl+1, 2*idx+1);
        return (child1, child2);
    }

    def get_neighbors() {
        const neighbor1 = new Node(lvl, idx-1);
        const neighbor2 = new Node(lvl, idx+1);
        return (neighbor1, neighbor2);
    }
}

const rootNode = new Node(0, 0);
const emptyNode = new Node(-1, -1);

// The empty vector, returned when there are no Coeffs.
var None: [0..-1] real;

// Check if we have None.
def isNone(x) {
    return x.numElements == 0;
}

class LocTree {
    const coeffDom : domain(1);
    var nodes      : domain(Node);
    var coeffs     : [nodes] [coeffDom] real;

    var zeroes     : [coeffDom] real;    // Return zeroes from this() when
                                         // reading something that has not yet
                                         // been set.

    var oneAtATime$: sync bool = true;

    /** Access an element in the associative domain.  If the element
        doesn't exist it will be created.

        We have to be really careful about creating extra boxes when we don't
        mean to.  The algorithms tend to be really sensitive to the structure
        of the tree.  Scaling coefficients are used to mark the leaves in the
        sum tree, so if you accidentally create scaling coeffs somewhere in the
        middle of the tree you will short circuit a whole subtree!

     */
    def this(node: Node) var {
        oneAtATime$;
        if !nodes.member(node) {
            if setter {
              nodes += node;
            } else {
              // This is a getter so it shouldn't be modifying what
              // we return, should be safe to return the zero vector.
              // FIXME: Zeroes should really be a const, but can't
              //        return const from a var fcn.
              oneAtATime$ = true;
              return zeroes;
            }
        }

        const c => coeffs[node];
        oneAtATime$ = true;
        return c;
    }

    /** Access an element in the associative domain.  If it doesn't exist,
        return None.
     */
    def peek(node: Node) var {
        if has_coeffs(node) then
            return this(node);
        else
            return None;
    }

    /** Unordered iterator over all coefficients
     */
    def these() {
        for c in coeffs do
            yield c;
    }

    /** Unordered iterator over all boxes in a particular level.
        This is not a particularly fast way to do this, but for many of the
        Madness levelwise iterators that do all levels from 0..max_level this
        will be much faster when the data is not all the way down at max_level.
        A sparse array or an array of associative arrays may be more conducive
        to this type of iteration vs. the associative domain used here.
     */
    def coeffs_iter(lvl: int) {
        for i in nodes do
            if i.lvl == lvl then yield coeffs[i];
    }

    /** Check if there are coefficients in box (lvl, idx)
     */
    def has_coeffs(node: Node) {
        oneAtATime$;
        const b = nodes.member(node);
        oneAtATime$ = true;
        return b;
    }

    /** Remove an element from the associative domain.  If the element
        does not exist, it is ignored.
     */
    def remove(node: Node) {
        oneAtATime$;
        if nodes.member(node) then nodes.remove(node);
        oneAtATime$ = true;
    }

    def node_iter(lvl: int) {
        for i in nodes do
            if i.lvl == lvl then yield i;
    }

    def node_iter() {
        for i in nodes do
            yield i;
    }

    def copy(t: LocTree) {
        t.nodes = nodes;
        t.coeffs = coeffs;
    }
}

class FTree {
    const order    : int;
    const coeffDom = [0..order-1];

    var tree       : [LocaleSpace] LocTree;

    def initialize() {
        if order == 0 then
            halt("FTree must be initialized with an order > 0");

        coforall loc in LocaleSpace do
            on Locales(loc) do tree[loc] = new LocTree(coeffDom);
    }

    def this(node: Node) var {
        const t = tree[node.loc];
        return t[node];
    }

    def this(loc: int) {
        for coeffs in tree[loc] do
            yield coeffs;
    }

    def these() {
        for t in tree do
            for coeffs in t do
                yield coeffs;
    }

    def coeffs_iter(lvl: int) {
        for t in tree do
            for coeffs in t.coeffs_iter(lvl) do
                yield coeffs;
    }

    def peek(node: Node) var {
        const t = tree[node.loc];
        return t.peek(node);
    }

    def has_coeffs(node: Node) {
        const t = tree[node.loc];
        return t.has_coeffs(node);
    }

    def remove(node: Node) {
        const t = tree[node.loc];
        t.remove(node);
    }

    /** Return a copy of this FTree
     */
    def copy() {
        const f = new FTree(order);
        for loc in LocaleSpace do
            tree[loc].copy(f.tree[loc]);
        return f;
    }

    def node_iter(lvl: int) {
        for t in tree do
            for i in t.node_iter(lvl) do
                yield i;
    }

    def node_iter() {
        for t in tree do
            for i in t.node_iter() do
                yield i;
    }
}

def main() {
    var f = new FTree(2);

    for (i, j) in [1..3, 2..4] {
        const node = new Node(i, j);
        f[node] = (i, j);
    }

    for loc in LocaleSpace {
        writeln("\n\ntree on loc ", loc, " = ");
        for coeffs in f[loc] do
            writeln(coeffs);
    }

    var node = new Node(4, 5);
    writeln("\n\nf.has_coeffs((4, 5)) = ", f.has_coeffs(node));
    writeln("f.peek((4, 5)) = ", f.peek(node));
    writeln("f[(4, 5)] = ", f[node]);
    writeln("f.remove((4, 5))");
    f.remove(node);

    node = new Node(1, 2);
    writeln("\n\nf.has_coeffs((1, 2)) = ", f.has_coeffs(node));
    writeln("f.peek((1, 2)) = ", f.peek(node));
    writeln("f[(1, 2)] = ", f[node]);
    writeln("f.remove((1, 2))");
    f.remove(node);

    node = new Node(3, 4);
    writeln("\n\nf.remove((3, 4))");
    f.remove(node);

    for loc in LocaleSpace {
        writeln("\n\ntree on loc ", loc, " = ");
        for coeffs in f[loc] do
            writeln(coeffs);
    }

    writeln("\n\nentire tree = ");
    for coeffs in f do
        writeln(coeffs);

    var f1 = f.copy();

    node = new Node(3, 2);
    writeln("\n\nf.remove((3, 2))");
    f.remove(node);

    writeln("\n\nentire tree = ");
    for coeffs in f do
        writeln(coeffs);

    writeln("\n\ntree copy = ");
    for coeffs in f1 do
        writeln(coeffs);

    for lvl in 1..3 {
        writeln("\n\ncoeffs on lvl ", lvl, " = ");
        for coeffs in f.coeffs_iter(lvl) do
            writeln(coeffs);
    }

    for lvl in 1..3 {
        writeln("\n\nindices on lvl ", lvl, " = ");
            for i in f.node_iter(lvl) do
                writeln(i());
    }

    writeln("\n\nall tree indices = ");
    for i in f.node_iter() do
        writeln(i());
}

/** FTree: The tree data structure used to store the numerical representation
    of a function.

    For 1d Madness, an FTree is a binary tree, for 2d it would
    be a quad tree, 3d an octtree, and so on.  The FTree used here uses level
    and index hashing to store the tree rather than storing connectivity
    information.  So, it is important that you do a has_coeffs() before
    indexing into the tree to be sure you don't create coeffs when you don't
    mean to.
 */

record Node {
    const lvl, idx : int;
    const loc = (lvl+idx)%numLocales;

    def this() {
        return (lvl, idx);
    }

    def path_upwards(node1: Node) {
        var l = idx;
        for n in node1.lvl..lvl by -1 {
           const node = new Node(n, l);
           yield node;
           l /= 2;
        }
    }

    def path_downwards(node1: Node) {
        for n in lvl..node1.lvl-1 {
            const node = new Node(n, node1.idx/(2**(node1.lvl-n)));
            yield node;
        }
    }

    def on_boundary() {
        return (idx < 0 || idx >= 2**lvl);
    }

    def get_children() {
        const child1 = new Node(lvl+1, 2*idx);
        const child2 = new Node(lvl+1, 2*idx+1);
        return (child1, child2);
    }

    def get_neighbors() {
        const neighbor1 = new Node(lvl, idx-1);
        const neighbor2 = new Node(lvl, idx+1);
        return (neighbor1, neighbor2);
    }
}

const rootNode = new Node(0, 0);
const emptyNode = new Node(-1, -1);

// The empty vector, returned when there are no Coeffs.
var None: [0..-1] real;

// Check if we have None.
def isNone(x) {
    return x.numElements == 0;
}

class LocTree {
    const coeffDom : domain(1);
    var nodes      : domain(Node);
    var coeffs     : [nodes] [coeffDom] real;

    var zeroes     : [coeffDom] real;    // Return zeroes from this() when
                                         // reading something that has not yet
                                         // been set.

    var oneAtATime$: sync bool = true;

    /** Access an element in the associative domain.  If the element
        doesn't exist it will be created.

        We have to be really careful about creating extra boxes when we don't
        mean to.  The algorithms tend to be really sensitive to the structure
        of the tree.  Scaling coefficients are used to mark the leaves in the
        sum tree, so if you accidentally create scaling coeffs somewhere in the
        middle of the tree you will short circuit a whole subtree!

     */
    def this(node: Node) var {
        oneAtATime$;
        if !nodes.member(node) {
            if setter {
              nodes += node;
            } else {
              // This is a getter so it shouldn't be modifying what
              // we return, should be safe to return the zero vector.
              // FIXME: Zeroes should really be a const, but can't
              //        return const from a var fcn.
              oneAtATime$ = true;
              return zeroes;
            }
        }

        const c => coeffs[node];
        oneAtATime$ = true;
        return c;
    }

    /** Access an element in the associative domain.  If it doesn't exist,
        return None.
     */
    def peek(node: Node) var {
        if has_coeffs(node) then
            return this(node);
        else
            return None;
    }

    /** Unordered iterator over all coefficients
     */
    def these() {
        oneAtATime$;
        for c in coeffs do
            yield c;
        oneAtATime$ = true;
    }

    /** Unordered iterator over all boxes in a particular level.
        This is not a particularly fast way to do this, but for many of the
        Madness levelwise iterators that do all levels from 0..max_level this
        will be much faster when the data is not all the way down at max_level.
        A sparse array or an array of associative arrays may be more conducive
        to this type of iteration vs. the associative domain used here.
     */
    def coeffs_iter(lvl: int) {
        oneAtATime$;
        for i in nodes do
            if i.lvl == lvl then yield coeffs[i];
        oneAtATime$ = true;
    }

    /** Check if there are coefficients in box (lvl, idx)
     */
    def has_coeffs(node: Node) {
        oneAtATime$;
        const b = nodes.member(node);
        oneAtATime$ = true;
        return b;
    }

    /** Remove an element from the associative domain.  If the element
        does not exist, it is ignored.
     */
    def remove(node: Node) {
        oneAtATime$;
        if nodes.member(node) then nodes.remove(node);
        oneAtATime$ = true;
    }

    def node_iter(lvl: int) {
        oneAtATime$;
        for i in nodes do
            if i.lvl == lvl then yield i;
        oneAtATime$ = true;
    }

    def node_iter() {
        oneAtATime$;
        for i in nodes do
            yield i;
        oneAtATime$ = true;
    }

    def copy(t: LocTree) {
        oneAtATime$;
        t.nodes = nodes;
        t.coeffs = coeffs;
        oneAtATime$ = true;
    }
}

class FTree {
    const order    : int;
    const coeffDom = [0..order-1];

    var tree       : [LocaleSpace] LocTree;

    def initialize() {
        if order == 0 then
            halt("FTree must be initialized with an order > 0");

        for loc in LocaleSpace do
            on Locales(loc) do tree[loc] = new LocTree(coeffDom);
    }

    def this(node: Node) var {
        const t = tree[node.loc];
        return t[node];
    }

    def this(loc: int) {
        for coeffs in tree[loc] do
            yield coeffs;
    }

    def these() {
        for t in tree do
            for coeffs in t do
                yield coeffs;
    }

    def coeffs_iter(lvl: int) {
        for t in tree do
            for coeffs in t.coeffs_iter(lvl) do
                yield coeffs;
    }

    def peek(node: Node) var {
        const t = tree[node.loc];
        return t.peek(node);
    }

    def has_coeffs(node: Node) {
        const t = tree[node.loc];
        return t.has_coeffs(node);
    }

    def remove(node: Node) {
        const t = tree[node.loc];
        t.remove(node);
    }

    /** Return a copy of this FTree
     */
    def copy() {
        const f = new FTree(order);
        for loc in LocaleSpace do
            tree[loc].copy(f.tree[loc]);
        return f;
    }

    def node_iter(lvl: int) {
        for t in tree do
            for i in t.node_iter(lvl) do
                yield i;
    }

    def node_iter() {
        for t in tree do
            for i in t.node_iter() do
                yield i;
    }
}

def main() {
    var f = new FTree(2);

    for (i, j) in [1..3, 2..4] {
        const node = new Node(i, j);
        f[node] = (i, j);
    }

    for loc in LocaleSpace {
        writeln("\n\ntree on loc ", loc, " = ");
        for coeffs in f[loc] do
            writeln(coeffs);
    }

    var node = new Node(4, 5);
    writeln("\n\nf.has_coeffs((4, 5)) = ", f.has_coeffs(node));
    writeln("f.peek((4, 5)) = ", f.peek(node));
    writeln("f[(4, 5)] = ", f[node]);
    writeln("f.remove((4, 5))");
    f.remove(node);

    node = new Node(1, 2);
    writeln("\n\nf.has_coeffs((1, 2)) = ", f.has_coeffs(node));
    writeln("f.peek((1, 2)) = ", f.peek(node));
    writeln("f[(1, 2)] = ", f[node]);
    writeln("f.remove((1, 2))");
    f.remove(node);

    node = new Node(3, 4);
    writeln("\n\nf.remove((3, 4))");
    f.remove(node);

    for loc in LocaleSpace {
        writeln("\n\ntree on loc ", loc, " = ");
        for coeffs in f[loc] do
            writeln(coeffs);
    }

    writeln("\n\nentire tree = ");
    for coeffs in f do
        writeln(coeffs);

    var f1 = f.copy();

    node = new Node(3, 2);
    writeln("\n\nf.remove((3, 2))");
    f.remove(node);

    writeln("\n\nentire tree = ");
    for coeffs in f do
        writeln(coeffs);

    writeln("\n\ntree copy = ");
    for coeffs in f1 do
        writeln(coeffs);

    for lvl in 1..3 {
        writeln("\n\ncoeffs on lvl ", lvl, " = ");
        for coeffs in f.coeffs_iter(lvl) do
            writeln(coeffs);
    }

    for lvl in 1..3 {
        writeln("\n\nindices on lvl ", lvl, " = ");
            for i in f.node_iter(lvl) do
                writeln(i());
    }

    writeln("\n\nall tree indices = ");
    for i in f.node_iter() do
        writeln(i());
}