Changed quicksort example to cut off all parallelism after thresh

Our previous version of quicksort used == to cut off parallelism
using the serial statement which means that additional parallelism
would've been used at thresh == -1 under our current implementation
and semantics.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@15415 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

//
// An example of a parallel quick sort implementation that uses
// "cobegin" to make each recursive call in parallel and "serial" to
// limit the number of threads.
//

use Random, Time; // for random number generation and the Timer class

var timer: Timer; // to time the sort

config var n: int = 2**15;      // the size of the array to be sorted
config var thresh: int = 1;     // the recursive depth to serialize
                                //  use --thresh=-1 for unlimited threads
config var verbose: int = 0;    // print out this many elements in array
config var timing: bool = true; // set timing to false to disable timer

var A: [1..n] real; // array of real numbers

//
// initialize array with random numbers
//
fillRandom(A);

//
// print out front of array if verbose flag is set
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// start timer, call parallel quick sort routine, stop timer
//
if timing then timer.start();
pqsort(A, thresh);
if timing then timer.stop();

//
// report sort time
//
if timing then writeln("sorted in ", timer.elapsed(), " seconds");

//
// print out front of array if verbose flag is set
//   values should now be in sorted order
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// verify that array is sorted or halt
//
for i in 2..n do
  if A(i) < A(i-1) then
    halt("A(", i-1, ") == ", A(i-1), " > A(", i, ") == ", A(i));

writeln("verification success");

//
// pqsort -- parallel quick sort
//
//   arr: generic 1D array of values (real, int, ...)
//   thresh: number of recursive calls to make before serializing
//   low: lower bound of array to start sort at, defaults to whole array
//   high: upper bound of array to stop sort at, defaults to whole array
//
def pqsort(arr: [],
           thresh: int,
           low: int = arr.domain.low,
           high: int = arr.domain.high) where arr.rank == 1 {

  //
  // base case: arr[low..high] is small enough to bubble sort
  //
  if high - low < 8 {
    bubbleSort(arr, low, high);
    return;
  }

  //
  // determine pivot and partition arr[low..high]
  //
  const pivotVal = findPivot();
  const pivotLoc = partition(pivotVal);

  //
  // make recursive calls to parallel quick sort each unsorted half of
  // the array; if thresh is 0 or less, start executing conquer tasks
  // serially
  //
  serial thresh <= 0 do cobegin {
    pqsort(arr, thresh-1, low, pivotLoc-1);
    pqsort(arr, thresh-1, pivotLoc+1, high);
  }

  //
  // findPivot -- helper routine to find pivot value using simple
  //              median-of-3 method, returns pivot value
  //
  def findPivot() {
    const mid = low + (high-low+1) / 2;

    if arr(mid) < arr(low) then arr(mid) <=> arr(low);
    if arr(high) < arr(low) then arr(high) <=> arr(low);
    if arr(high) < arr(mid) then arr(high) <=> arr(mid);

    const pivotVal = arr(mid);
    arr(mid) = arr(high-1);
    arr(high-1) = pivotVal;

    return pivotVal;
  }

  //
  // partition -- helper routine to partition array such that all
  //              values less than pivot are to its left and all
  //              values greater than pivot are to its right, returns
  //              pivot location
  //
  def partition(pivotVal) {
    var ilo = low, ihi = high-1;
    while (ilo < ihi) {
      do { ilo += 1; } while arr(ilo) < pivotVal;
      do { ihi -= 1; } while pivotVal < arr(ihi);
      if (ilo < ihi) {
        arr(ilo) <=> arr(ihi);
      }
    }
    arr(high-1) = arr(ilo);
    arr(ilo) = pivotVal;
    return ilo;
  }
}

//
// bubbleSort -- bubble sort for base case of quick sort
//
//   arr: generic 1D array of values (real, int, ...)
//   low: lower bound of array to start sort at
//   high: upper bound of array to stop sort at
//
def bubbleSort(arr: [], low: int, high: int) where arr.rank == 1 {
  for i in low..high do
    for j in low..high-1 do
      if arr(j) > arr(j+1) then
        arr(j) <=> arr(j+1);
}

//
// An example of a parallel quick sort implementation that uses
// "cobegin" to make each recursive call in parallel and "serial" to
// limit the number of threads.
//

use Random, Time; // for random number generation and the Timer class

var timer: Timer; // to time the sort

config var n: int = 2**15;      // the size of the array to be sorted
config var thresh: int = 1;     // the recursive depth to serialize
                                //  use --thresh=-1 for unlimited threads
config var verbose: int = 0;    // print out this many elements in array
config var timing: bool = true; // set timing to false to disable timer

var A: [1..n] real; // array of real numbers

//
// initialize array with random numbers
//
fillRandom(A);

//
// print out front of array if verbose flag is set
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// start timer, call parallel quick sort routine, stop timer
//
if timing then timer.start();
pqsort(A, thresh);
if timing then timer.stop();

//
// report sort time
//
if timing then writeln("sorted in ", timer.elapsed(), " seconds");

//
// print out front of array if verbose flag is set
//   values should now be in sorted order
//
if verbose > 0 then
  writeln("A[1..", verbose, "] = ", A[1..verbose]);

//
// verify that array is sorted or halt
//
for i in 2..n do
  if A(i) < A(i-1) then
    halt("A(", i-1, ") == ", A(i-1), " > A(", i, ") == ", A(i));

writeln("verification success");

//
// pqsort -- parallel quick sort
//
//   arr: generic 1D array of values (real, int, ...)
//   thresh: number of recursive calls to make before serializing
//   low: lower bound of array to start sort at, defaults to whole array
//   high: upper bound of array to stop sort at, defaults to whole array
//
def pqsort(arr: [],
           thresh: int,
           low: int = arr.domain.low,
           high: int = arr.domain.high) where arr.rank == 1 {

  //
  // base case: arr[low..high] is small enough to bubble sort
  //
  if high - low < 8 {
    bubbleSort(arr, low, high);
    return;
  }

  //
  // determine pivot and partition arr[low..high]
  //
  const pivotVal = findPivot();
  const pivotLoc = partition(pivotVal);

  //
  // make recursive calls to parallel quick sort each unsorted half of
  // the array; if thresh is 0, start serializing; note: once
  // serialization is turned on, it is never turned off
  //
  serial thresh == 0 do cobegin {
    pqsort(arr, thresh-1, low, pivotLoc-1);
    pqsort(arr, thresh-1, pivotLoc+1, high);
  }

  //
  // findPivot -- helper routine to find pivot value using simple
  //              median-of-3 method, returns pivot value
  //
  def findPivot() {
    const mid = low + (high-low+1) / 2;

    if arr(mid) < arr(low) then arr(mid) <=> arr(low);
    if arr(high) < arr(low) then arr(high) <=> arr(low);
    if arr(high) < arr(mid) then arr(high) <=> arr(mid);

    const pivotVal = arr(mid);
    arr(mid) = arr(high-1);
    arr(high-1) = pivotVal;

    return pivotVal;
  }

  //
  // partition -- helper routine to partition array such that all
  //              values less than pivot are to its left and all
  //              values greater than pivot are to its right, returns
  //              pivot location
  //
  def partition(pivotVal) {
    var ilo = low, ihi = high-1;
    while (ilo < ihi) {
      do { ilo += 1; } while arr(ilo) < pivotVal;
      do { ihi -= 1; } while pivotVal < arr(ihi);
      if (ilo < ihi) {
        arr(ilo) <=> arr(ihi);
      }
    }
    arr(high-1) = arr(ilo);
    arr(ilo) = pivotVal;
    return ilo;
  }
}

//
// bubbleSort -- bubble sort for base case of quick sort
//
//   arr: generic 1D array of values (real, int, ...)
//   low: lower bound of array to start sort at
//   high: upper bound of array to stop sort at
//
def bubbleSort(arr: [], low: int, high: int) where arr.rank == 1 {
  for i in low..high do
    for j in low..high-1 do
      if arr(j) > arr(j+1) then
        arr(j) <=> arr(j+1);
}


Fixed a typo in the fragmented stream that Steve caught

...but that has nothing to do with the performance problems I was
seeing... :(

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14887 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use Time, Types, Random;
use hpccMultilocale;

use HPCCProblemSize;

param numVectors = 3;
type elemType = real(64),
     indexType = int(64);

config const m = computeProblemSize(numVectors, elemType),
             alpha = 3.0;

config const numTrials = 10,
             epsilon = 0.0;

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

config const printParams = true,
             printArrays = false,
             printStats = true;

def main() {
  printConfiguration();

  const ProblemSpace: domain(1, indexType) = [1..m];

  var localGBs: [LocaleSpace] real,
      allValidAnswer: [LocaleSpace] bool;

  coforall loc in Locales {
    on loc {
      const myProblemSpace: domain(1, indexType)
                          = BlockPartition(ProblemSpace, here.id, numLocales);

      var myA, myB, myC: [myProblemSpace] elemType;

      initVectors(myB, myC, myProblemSpace);

      const localTimings: [1..numTrials] real;

      for trial in 1..numTrials {
        const startTime = getCurrentTime();
        local {
	forall (a, b, c) in (myA, myB, myC) do
	  a = b + alpha * c;
        }
        localTimings(trial) = getCurrentTime() - startTime;
      }

      const myMinTime = min reduce localTimings;

      localGBs(here.id) = numVectors * numBytes(elemType) * (m / myMinTime) * 1e-9;
      allValidAnswer(here.id) = verifyResults(myA, myB, myC);
    }
  }

  const minGBs = min reduce localGBs,
        maxGBs = max reduce localGBs,
        avgGBs = + reduce localGBs / numLocales;

  const validAnswer = & reduce allValidAnswer;

  printResults(validAnswer, minGBs, avgGBs, maxGBs);
}

def printConfiguration() {
  if (printParams) {
    printProblemSize(elemType, numVectors, m);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

def initVectors(B, C, ProblemSpace) {
  var randlist = new RandomStream(seed);

  randlist.skipToNth(B.domain.low);
  randlist.fillRandom(B);
  randlist.skipToNth(ProblemSpace.numIndices + C.domain.low);
  randlist.fillRandom(C);

  if (printArrays) {
    writelnFragArray("B is: ", B, "\n");
    writelnFragArray("C is: ", C, "\n");
  }
}

def verifyResults(A, B, C) {
  if (printArrays) then writelnFragArray("A is: ", A, "\n");

  const infNorm = max reduce [i in A.domain] abs(A(i) - (B(i) + alpha * C(i)));

  return (infNorm <= epsilon);
}

def printResults(successful, minGBs, avgGBs, maxGBs) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Fragmented, per-locale GBs:");
    writeln("  min = ", minGBs);
    writeln("  avg = ", avgGBs);
    writeln("  max = ", maxGBs);
  }
}

use Time, Types, Random;
use hpccMultilocale;

use HPCCProblemSize;

param numVectors = 3;
type elemType = real(64),
     indexType = int(64);

config const m = computeProblemSize(numVectors, elemType),
             alpha = 3.0;

config const numTrials = 10,
             epsilon = 0.0;

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

config const printParams = true,
             printArrays = false,
             printStats = true;

def main() {
  printConfiguration();

  const ProblemSpace: domain(1, indexType) = [1..m];

  var localGBs: [LocaleSpace] real,
      allValidAnswer: [LocaleSpace] bool;

  coforall loc in Locales {
    on loc {
      const myProblemSpace: domain(1, indexType)
                          = BlockPartition(ProblemSpace, here.id, numLocales);

      var myA, myB, myC: [myProblemSpace] elemType;

      initVectors(myB, myC, ProblemSpace);

      const localTimings: [1..numTrials] real;

      for trial in 1..numTrials {
        const startTime = getCurrentTime();
        local {
	forall (a, b, c) in (myA, myB, myC) do
	  a = b + alpha * c;
        }
        localTimings(trial) = getCurrentTime() - startTime;
      }

      const myMinTime = min reduce localTimings;

      localGBs(here.id) = numVectors * numBytes(elemType) * (m / myMinTime) * 1e-9;
      allValidAnswer(here.id) = verifyResults(myA, myB, myC);
    }
  }

  const minGBs = min reduce localGBs,
        maxGBs = max reduce localGBs,
        avgGBs = + reduce localGBs / numLocales;

  const validAnswer = & reduce allValidAnswer;

  printResults(validAnswer, minGBs, avgGBs, maxGBs);
}

def printConfiguration() {
  if (printParams) {
    printProblemSize(elemType, numVectors, m);
    writeln("Number of trials = ", numTrials, "\n");
  }
}

def initVectors(B, C, ProblemSpace) {
  var randlist = new RandomStream(seed);

  randlist.skipToNth(B.domain.low);
  randlist.fillRandom(B);
  randlist.skipToNth(ProblemSpace.numIndices + C.domain.low);
  randlist.fillRandom(C);

  if (printArrays) {
    writelnFragArray("B is: ", B, "\n");
    writelnFragArray("C is: ", C, "\n");
  }
}

def verifyResults(A, B, C) {
  if (printArrays) then writelnFragArray("A is: ", A, "\n");

  const infNorm = max reduce [i in A.domain] abs(A(i) - (B(i) + alpha * C(i)));

  return (infNorm <= epsilon);
}

def printResults(successful, minGBs, avgGBs, maxGBs) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Fragmented, per-locale GBs:");
    writeln("  min = ", minGBs);
    writeln("  avg = ", avgGBs);
    writeln("  max = ", maxGBs);
  }
}


Refactored one of the remaining error message cases

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Wrapper for FFTW version 3.

/*
  FFT computations via key routines from FFTW (version 3)

  This module defines Chapel wrappers for key 64-bit routines from
  FFTW (http://www.fftw.org), version 3.  Over time, the intention is
  to expand this module to support additional routines, prioritizing
  based on requests and feedback from users.  Also, see the
  :mod:`FFTW_MT` module which provides support for calls to the
  multi-threaded FFTW implementation.

  To use this module:

  1. Ensure that FFTW (version 3) is installed on your system and that
     the header and library files (e.g., fftw3.h, libfftw3.*) are
     either installed in a standard system location or that your C
     compiler's environment variables are set up to find them.

  2. Add ``use FFTW;`` to your Chapel code.

  3. Compile and run your Chapel program as usual.

  As in standard FFTW usage, the flow is to:

  1. Create plan(s) using the :proc:`plan_dft* <plan_dft>` routines.

  2. Execute the plan(s) one or more times using :proc:`execute`.

  3. Destroy the plan(s) using :proc:`destroy_plan`.

  4. Call :proc:`cleanup`.

  Note that each of the Chapel :proc:`plan_dft* <plan_dft>` routines
  support both `in-place` and `out-of-place` versions of the
  transforms, where the former versions use a single array for both
  input and output, and the latter use two distinct arrays.

  In future versions of this module, we anticipate improving the
  plan_dft*() interfaces to make better use of Chapel features and
  move further away from C-isms (like the overloaded role of `flags`
  and the use of C-based types).  Such features are expected to take
  advantage of Chapel's support for default argument values and
  keyword-based argument passing.  We are also thinking about changing
  the interface for the in-place routines to use array slicing rather
  than separate arguments for the array and domain.
*/

// Possible TODOs:
//
// - How do we feel about FFTW_ALLCAPS names given that the routines
//   themselves don't have fftw_ prefixes, that they're defined as
//   c_int's, and that they're all-caps?  What if we were to replace
//   these with bool/enum arguments representing the separate planning
//   aspects that are overloaded into 'flags'?  This would also allow
//   us to move away from c_ints (which could be done in other ways
//   as well, such as safe-casting and providing our own versions of
//   the C variables).
//
// - It seems that rather than passing in a domain and an array for
//   the in-place cases, we could probably pass in an array slice
//   where the array's allocating domain reflected the padding and
//   the slicing domain reflected the area over which the FFT should
//   be performed.  Would this be cleaner?  Would the overhead be
//   significant?
//

module FFTW {

  use SysCTypes;
  use "fftw3.h", "-lfftw3";

  /*
    Disable execution-time array size checks in the FFTW
    :proc:`plan_dft` routine.
   */
  config param noFFTWsizeChecks = false;

  /*
    An opaque type used to store and reuse FFTW plans across multiple
    routines.
  */
  extern type fftw_plan; // opaque type

  // Planner functions
  // Complex : 4.3.1
  // NOTE : We pass in arrays using ref

  // TODO: Can we have the plan_dft() routine below take in native
  // Chapel types without changing the external C constants from
  // type c_int?  Probably given that Chapel's int will be >=
  // C's int for the forseeable future.  See also the TODO above
  // about whether we'd want to represent those constants using
  // more native Chapel types anyway.

  /*
    Creates a plan for an out-of-place complex-to-complex DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `complex(128)`

    :arg output: The output array, whose size and shape must match the input array's
    :type output: [] `complex(128)`

    :arg sign: :const:`FFTW_FORWARD` or :const:`FFTW_BACKWARD`
    :type sign: c_int

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft(input: [?Din] complex(128), output: [?Dout] complex(128),
                 sign: c_int, flags: c_uint) : fftw_plan
  {
    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..input.rank do
        error |= Private_FFTW.checkDimMismatch(Din, Dout, i, "plan_dft()");

      if error then
        halt("Incorrect array sizes in plan_dft()");
    }

    return plan_dft_help(input, output, sign, flags);
  }

  /*
    Creates a plan for an in-place complex-to-complex DFT.

    :arg arr: The array to use as the in-place input/output array.
    :type arr: [] `complex(128)`

    :arg sign: :const:`FFTW_FORWARD` or :const:`FFTW_BACKWARD`
    :type sign: c_int

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft(arr: [] complex(128), sign: c_int, flags: c_uint): fftw_plan {
    return plan_dft_help(arr, arr, sign, flags);
  }

  //
  // Though not strictly necessary, this helper routine is to avoid
  // doing the size check for the in-place case.  This is the kind
  // of thing we'd like to mark as "private" once we have that
  // capability.  I could move it into a sub-module for the time
  // being, but want to keep the diff with the previous version of
  // the code minimal.  -BLC
  //
  pragma "no doc"
  proc plan_dft_help(input: [] complex(128), output: [] complex(128),
                 sign: c_int, flags: c_uint) : fftw_plan
  {
    param rank = input.rank;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size.safeCast(c_int);

    return C_FFTW.fftw_plan_dft(rank.safeCast(c_int), dims, input,
                                     output, sign, flags);
  }

  // Real-to-complex and complex-to-real planning routines There are
  // two cases that we treat independently here : in-place and
  // out-of-place transforms

  /*
    Create a plan for a real-to-complex, out-of-place DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `real(64)`

    :arg output: The output array, whose size and shape must match the input array's, except for the leading dimension which should be n/2 + 1, where n is the size of the input array's leading dimension.  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type output: [] `complex(128)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft_r2c(input : [?Din] real(64), output : [?Dout] complex(128),
                    flags :c_uint) : fftw_plan
  {
    param rank = input.rank: c_int;

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(Din, Dout, i, "plan_dft_r2c()");

      error |= Private_FFTW.checkRealCplxDimMismatch(Din, Dout, rank, "plan_dft_r2c()",
                                                     "output");

      /*
      {
        const inputDim = input.domain.dim(rank).size/2+1;
        const outputDim = output.domain.dim(rank).size;
        if (inputDim != outputDim) {
          writeln("Error: In plan_dft_r2c(), output array's leading dimension is not of the appropriate size (expected ", inputDim, ", got ", outputDim, ")");
          error = true;
        }
      }
      */
      if (error) then
        halt("Incorrect array sizes in plan_dft_r2c()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, input, output, flags);
  }

  // In-place routines, note that these take in the true leading dimension
  /*
    Create a plan for a real-to-complex, in-place DFT.

    :arg realDom: Describes the indices of the 'real' view of the array
    :type realDom: `domain`

    :arg arr: The array to be used as the in-place input/output array.  If passing in an array of `real` elements, the leading dimension of the array must be padded to store 2(n/2 + 1) elements, where `n` is the size of the corresponding dimension of `realDom`.  If passing in an array of `complex` elements, the leading dimension should be (n/2 + 1).  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type arr: [] `T` where `T` is of type `real(640` or `complex(128)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
   */
  proc plan_dft_r2c(realDom : domain, arr : [?D] ?t, flags : c_uint) : fftw_plan
    where t == real || t == complex
  {
    param rank = realDom.rank: c_int;

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(realDom, D, i, "plan_dft_r2c()",
                                               inplace=true);

      if t == real {
        const arrDim = arr.domain.dim(rank).size;
        const domDim = 2*(realDom.dim(rank).size/2+1);
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_r2c(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
      } else {
        error |= Private_FFTW.checkRealCplxDimMismatch(realDom, D, rank, "plan_dft_r2c()");
        /*
        const arrDim = arr.domain.dim(rank).size;
        const domDim = realDom.dim(rank).size/2+1;
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_r2c(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
        */
      }
      if (error) then
        halt("Incorrect array sizes in plan_dft_r2c()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, arr, arr, flags);
  }

  //
  // Error overload
  //
  pragma "no doc"
  proc plan_dft_r2c(realDom : domain, arr: [] ?t, flags : c_uint) : fftw_plan {
    compilerError("plan_dft_r2c() is only supported for arrays of type real(64) and complex(128)");
  }

  /*
    Create a plan for a complex-to-real, out-of-place DFT.

    :arg input: The input array, whose size and shape must match the output array's, except for the leading dimension which should be n/2 + 1, where n is the size of the output array's leading dimension.  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type input: [] `complex(128)`

    :arg output: The output array
    :type output: [] `real(64)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft_c2r(input : [?Din] complex(128), output : [?Dout] real(64),
                    flags :c_uint) : fftw_plan
  {
    param rank = output.rank: c_int; // The dimensions are that of the real array

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(Din, Dout, i, "plan_dft_c2r()");

      error |= Private_FFTW.checkRealCplxDimMismatch(Dout, Din, rank, "plan_dft_c2r()",
                                                     "input");

      /*
      {
        const inputDim = input.domain.dim(rank).size;
        const outputDim = output.domain.dim(rank).size/2+1;
        if (inputDim != outputDim) {
          writeln("Error: In plan_dft_c2r(), input array's leading dimension is not of the appropriate size (expected ", outputDim, ", got ", inputDim, ")");
          error = true;
        }
      }
      */
      if (error) then
        halt("Incorrect array sizes in plan_dft_c2r()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = output.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, input, output, flags);
  }

  /*
    Create a plan for a complex-to-real, in-place DFT.

    :arg realDom: Describes the indices of the 'real' view of the array
    :type realDom: `domain`

    :arg arr: The array to be used as the in-place input/output array.  If passing in an array of `real` elements, the leading dimension of the array must be padded to store 2(n/2 + 1) elements, where `n` is the size of the corresponding dimension of `realDom`.  If passing in an array of `complex` elements, the leading dimension should be (n/2 + 1).  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type arr: [] `T` where `T` is of type `real(64)` or `complex(128)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
   */
  proc plan_dft_c2r(realDom : domain, arr: [?D] ?t, flags : c_uint) : fftw_plan
    where t == real || t == complex
  {
    param rank = realDom.rank: c_int;

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(realDom, D, i, "plan_dft_c2r()",
                                               inplace=true);

      if t == real {
        const arrDim = arr.domain.dim(rank).size;
        const domDim = 2*(realDom.dim(rank).size/2+1);
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_c2r(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
      } else {
        error |= Private_FFTW.checkRealCplxDimMismatch(realDom, D, rank, "plan_dft_c2r()");
        /*
        const arrDim = arr.domain.dim(rank).size;
        const domDim = realDom.dim(rank).size/2+1;
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_c2r(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
        */
      }
      if (error) then
        halt("Incorrect array sizes in plan_dft_c2r()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, arr, arr, flags);
  }

  pragma "no doc"
  proc plan_dft_c2r(realDom : domain, arr: [] ?t, flags : c_uint) : fftw_plan {
    compilerError("plan_dft_c2r() is only supported for arrays of type real(64) and complex(128)");
  }

  // Using plans

  /*
    Execute an FFTW plan.

    :arg plan: The plan to execute, as computed by a `plan_dft*()` routine.
    :type plan: fftw_plan
  */
  proc execute(const plan: fftw_plan) {
   C_FFTW.fftw_execute(plan);
  }

  /*
    Destroy an FFTW plan.

    :arg plan: The plan to destroy
    :type plan: fftw_plan
  */
  proc destroy_plan(plan: fftw_plan) {
    C_FFTW.fftw_destroy_plan(plan);
  }

  /*
    Clean up FFTW overall.
  */
  proc cleanup() {
    C_FFTW.fftw_cleanup();
  }

  // Define the various planner flags
  // See Sec. 4.3.2 of FFTW manual "Planner Flags"

  //
  // TODO: Is it weird that we use FFTW_ names here, but not in
  // the routines?  Should we wrap the external flags so that
  // they can be ints or enums and avoid the all-caps C scheme
  // or is the familiarity better?
  //

  // Direction flags

  /* Request a forward transform (i.e., use a negative exponent in the tranform). */
  extern const FFTW_FORWARD : c_int;
  /* Request a backward transform (i.e., use a positive exponent in the transofmr). */
  extern const FFTW_BACKWARD : c_int;

  // Planning-rigor flags

  /* Specify that a simple heuristic should be used to pick a plan
     quickly.  This will prevent the input/output arrays from being
     overwritten during planning. */
  extern const FFTW_ESTIMATE : c_uint;

  /* Specify that FFTW should try and find an optimized plan by
     computing several FFTs and measuring their execution time.
     This can consume some time.
  */
  // TODO: If/when we support defaults, might say something like: This
  // is the default planning option.
  extern const FFTW_MEASURE : c_uint;

  /* Specify that FFTW should expend a greater effort finding an
     optimized plan. */
  extern const FFTW_PATIENT : c_uint;

  /* Specify that FFTW should expend an even greater effort finding an
     optimized plan. */
  extern const FFTW_EXHAUSTIVE : c_uint;

  /* This is a special planning mode that is useful for querying
     whether wisdom is available.  When using it, the plan is only
     created when wisdom is available for the given problem; otherwise
     a `null plan` is returned.  This can be combined with other flags
     to create a plan if the wisdom available was created in that mode
     (e.g., ``FFTW_WISDOM_ONLY | FFW_PATIENT``).  For more details on
     this flag and the previous four, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual
  */
  extern const FFTW_WISDOM_ONLY : c_uint;

  // Algorithm-restriction flags

  /* Specify that an out-of-place transform is permitted to overwrite
     its input array with arbitrary data.  This permits more efficient
     algorithms to be used in some cases. */
  // TODO: When we're ready to mention defaults, add: "This is the default for
  // :proc:`plan_dft_c2r`. // NOTE: ...and hc2r once supported...
  extern const FFTW_DESTROY_INPUT : c_uint;

  /* Specify that an out-of-place transform cannot change its input
     array. */
  // TODO: When we're ready to mention defaults, add: This is the
  // default for :proc:`plan_dft` and :proc:`plan_dft_r2c`. */
  extern const FFTW_PRESERVE_INPUT : c_uint;

  /* Specify that the algorithm may not impose any unusual alignment
     requirements on the input/output arrays.  This flag should not be
     necessary for current Chapel use since the planner will
     automatically detect such cases.  For more details on this flag
     and the previous two, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual.
  */  // NOTE: But it will be if/when the new-array execute interface is supported
  extern const FFTW_UNALIGNED : c_uint;

  //
  // This is a helper module to give us a "private" notion.  Once we
  // have a private keyword, these could be hoisted to module scope
  // and labeled private.
  //
  pragma "no doc"
  module Private_FFTW {
    //
    // Check that two domain dimensions match, print an error if they
    // don't, and return whether or not an error occurred.  Note that
    // the 'inplace' argument is used to customize the error message
    // for in-place and out-of-place cases.
    //
    proc checkDimMismatch(inDom, outDom, dim, fnname, inplace=false) {
      const inputDim = inDom.dim(dim).size;
      const outputDim = outDom.dim(dim).size;
      if (inputDim == outputDim) then
        return false;

      const first = if inplace then "domain" else "input";
      const second = if inplace then "array" else "output";

      writeln("Error: In ", fnname, ", the ", first, " and ", second,
              if !inplace then " arrays" else "",
              " don't have same size in dimension ", dim,
              " (", first, " = ", inputDim, ", ", second, " = ", outputDim,
              ")");
      return true;
    }

    proc checkRealCplxDimMismatch(Dreal, Dcomplex, dim, fnname, cplxarrdesc="") {
      const realDim = Dreal.dim(dim).size/2+1;
      const complexDim = Dcomplex.dim(dim).size;

      if (realDim == complexDim) then
        return false;

      writeln("Error: In ", fnname, ", the ", cplxarrdesc,
              " array's leading dimension is not the proper size (expected ", realDim,
              ", got ", complexDim, ")");
      return true;
    }
  }

  pragma "no doc"
  module C_FFTW {
    extern proc fftw_plan_dft(rank: c_int,
                              n,  // BLC: having trouble being specific
                                  // This is a const int* in C
                              in1: [] complex(128),
                              out1: [] complex(128),
                              sign : c_int, c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_r2c(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                      // This is a const int* in C
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_c2r(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                      // This is a const int* in C
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_execute(const plan : fftw_plan);

    extern proc fftw_destroy_plan(plan : fftw_plan);

    extern proc fftw_cleanup();

    extern proc fftw_init_threads() : c_int;

    extern proc fftw_cleanup_threads();

    extern proc fftw_plan_with_nthreads(n : c_int);
  }
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Wrapper for FFTW version 3.

/*
  FFT computations via key routines from FFTW (version 3)

  This module defines Chapel wrappers for key 64-bit routines from
  FFTW (http://www.fftw.org), version 3.  Over time, the intention is
  to expand this module to support additional routines, prioritizing
  based on requests and feedback from users.  Also, see the
  :mod:`FFTW_MT` module which provides support for calls to the
  multi-threaded FFTW implementation.

  To use this module:

  1. Ensure that FFTW (version 3) is installed on your system and that
     the header and library files (e.g., fftw3.h, libfftw3.*) are
     either installed in a standard system location or that your C
     compiler's environment variables are set up to find them.

  2. Add ``use FFTW;`` to your Chapel code.

  3. Compile and run your Chapel program as usual.

  As in standard FFTW usage, the flow is to:

  1. Create plan(s) using the :proc:`plan_dft* <plan_dft>` routines.

  2. Execute the plan(s) one or more times using :proc:`execute`.

  3. Destroy the plan(s) using :proc:`destroy_plan`.

  4. Call :proc:`cleanup`.

  Note that each of the Chapel :proc:`plan_dft* <plan_dft>` routines
  support both `in-place` and `out-of-place` versions of the
  transforms, where the former versions use a single array for both
  input and output, and the latter use two distinct arrays.

  In future versions of this module, we anticipate improving the
  plan_dft*() interfaces to make better use of Chapel features and
  move further away from C-isms (like the overloaded role of `flags`
  and the use of C-based types).  Such features are expected to take
  advantage of Chapel's support for default argument values and
  keyword-based argument passing.  We are also thinking about changing
  the interface for the in-place routines to use array slicing rather
  than separate arguments for the array and domain.
*/

// Possible TODOs:
//
// - How do we feel about FFTW_ALLCAPS names given that the routines
//   themselves don't have fftw_ prefixes, that they're defined as
//   c_int's, and that they're all-caps?  What if we were to replace
//   these with bool/enum arguments representing the separate planning
//   aspects that are overloaded into 'flags'?  This would also allow
//   us to move away from c_ints (which could be done in other ways
//   as well, such as safe-casting and providing our own versions of
//   the C variables).
//
// - It seems that rather than passing in a domain and an array for
//   the in-place cases, we could probably pass in an array slice
//   where the array's allocating domain reflected the padding and
//   the slicing domain reflected the area over which the FFT should
//   be performed.  Would this be cleaner?  Would the overhead be
//   significant?
//

module FFTW {

  use SysCTypes;
  use "fftw3.h", "-lfftw3";

  /*
    Disable execution-time array size checks in the FFTW
    :proc:`plan_dft` routine.
   */
  config param noFFTWsizeChecks = false;

  /*
    An opaque type used to store and reuse FFTW plans across multiple
    routines.
  */
  extern type fftw_plan; // opaque type

  // Planner functions
  // Complex : 4.3.1
  // NOTE : We pass in arrays using ref

  // TODO: Can we have the plan_dft() routine below take in native
  // Chapel types without changing the external C constants from
  // type c_int?  Probably given that Chapel's int will be >=
  // C's int for the forseeable future.  See also the TODO above
  // about whether we'd want to represent those constants using
  // more native Chapel types anyway.

  /*
    Creates a plan for an out-of-place complex-to-complex DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `complex(128)`

    :arg output: The output array, whose size and shape must match the input array's
    :type output: [] `complex(128)`

    :arg sign: :const:`FFTW_FORWARD` or :const:`FFTW_BACKWARD`
    :type sign: c_int

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft(input: [?Din] complex(128), output: [?Dout] complex(128),
                 sign: c_int, flags: c_uint) : fftw_plan
  {
    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..input.rank do
        error |= Private_FFTW.checkDimMismatch(Din, Dout, i, "plan_dft()");

      if error then
        halt("Incorrect array sizes in plan_dft()");
    }

    return plan_dft_help(input, output, sign, flags);
  }

  /*
    Creates a plan for an in-place complex-to-complex DFT.

    :arg arr: The array to use as the in-place input/output array.
    :type arr: [] `complex(128)`

    :arg sign: :const:`FFTW_FORWARD` or :const:`FFTW_BACKWARD`
    :type sign: c_int

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft(arr: [] complex(128), sign: c_int, flags: c_uint): fftw_plan {
    return plan_dft_help(arr, arr, sign, flags);
  }

  //
  // Though not strictly necessary, this helper routine is to avoid
  // doing the size check for the in-place case.  This is the kind
  // of thing we'd like to mark as "private" once we have that
  // capability.  I could move it into a sub-module for the time
  // being, but want to keep the diff with the previous version of
  // the code minimal.  -BLC
  //
  pragma "no doc"
  proc plan_dft_help(input: [] complex(128), output: [] complex(128),
                 sign: c_int, flags: c_uint) : fftw_plan
  {
    param rank = input.rank;

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size.safeCast(c_int);

    return C_FFTW.fftw_plan_dft(rank.safeCast(c_int), dims, input,
                                     output, sign, flags);
  }

  // Real-to-complex and complex-to-real planning routines There are
  // two cases that we treat independently here : in-place and
  // out-of-place transforms

  /*
    Create a plan for a real-to-complex, out-of-place DFT.

    :arg input: The input array, which can be of any rank
    :type input: [] `real(64)`

    :arg output: The output array, whose size and shape must match the input array's, except for the leading dimension which should be n/2 + 1, where n is the size of the input array's leading dimension.  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type output: [] `complex(128)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft_r2c(input : [?Din] real(64), output : [?Dout] complex(128),
                    flags :c_uint) : fftw_plan
  {
    param rank = input.rank: c_int;

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(Din, Dout, i, "plan_dft_r2c()");

      {
        const inputDim = input.domain.dim(rank).size/2+1;
        const outputDim = output.domain.dim(rank).size;
        if (inputDim != outputDim) {
          writeln("Error: In plan_dft_r2c(), output array's leading dimension is not of the appropriate size (expected ", inputDim, ", got ", outputDim, ")");
          error = true;
        }
      }
      if (error) then
        halt("Incorrect array sizes in plan_dft_r2c()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = input.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, input, output, flags);
  }

  // In-place routines, note that these take in the true leading dimension
  /*
    Create a plan for a real-to-complex, in-place DFT.

    :arg realDom: Describes the indices of the 'real' view of the array
    :type realDom: `domain`

    :arg arr: The array to be used as the in-place input/output array.  If passing in an array of `real` elements, the leading dimension of the array must be padded to store 2(n/2 + 1) elements, where `n` is the size of the corresponding dimension of `realDom`.  If passing in an array of `complex` elements, the leading dimension should be (n/2 + 1).  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type arr: [] `T` where `T` is of type `real(640` or `complex(128)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
   */
  proc plan_dft_r2c(realDom : domain, arr : [?D] ?t, flags : c_uint) : fftw_plan
    where t == real || t == complex
  {
    param rank = realDom.rank: c_int;

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(realDom, D, i, "plan_dft_r2c()",
                                               inplace=true);

      if t == real {
        const arrDim = arr.domain.dim(rank).size;
        const domDim = 2*(realDom.dim(rank).size/2+1);
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_r2c(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
      } else {
        const arrDim = arr.domain.dim(rank).size;
        const domDim = realDom.dim(rank).size/2+1;
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_r2c(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
      }
      if (error) then
        halt("Incorrect array sizes in plan_dft_r2c()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_r2c(rank, dims, arr, arr, flags);
  }

  //
  // Error overload
  //
  pragma "no doc"
  proc plan_dft_r2c(realDom : domain, arr: [] ?t, flags : c_uint) : fftw_plan {
    compilerError("plan_dft_r2c() is only supported for arrays of type real(64) and complex(128)");
  }

  /*
    Create a plan for a complex-to-real, out-of-place DFT.

    :arg input: The input array, whose size and shape must match the output array's, except for the leading dimension which should be n/2 + 1, where n is the size of the output array's leading dimension.  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type input: [] `complex(128)`

    :arg output: The output array
    :type output: [] `real(64)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
  */
  proc plan_dft_c2r(input : [?Din] complex(128), output : [?Dout] real(64),
                    flags :c_uint) : fftw_plan
  {
    param rank = output.rank: c_int; // The dimensions are that of the real array

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(Din, Dout, i, "plan_dft_c2r()");

      {
        const inputDim = input.domain.dim(rank).size;
        const outputDim = output.domain.dim(rank).size/2+1;
        if (inputDim != outputDim) {
          writeln("Error: In plan_dft_c2r(), input array's leading dimension is not of the appropriate size (expected ", outputDim, ", got ", inputDim, ")");
          error = true;
        }
      }
      if (error) then
        halt("Incorrect array sizes in plan_dft_c2r()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = output.domain.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, input, output, flags);
  }

  /*
    Create a plan for a complex-to-real, in-place DFT.

    :arg realDom: Describes the indices of the 'real' view of the array
    :type realDom: `domain`

    :arg arr: The array to be used as the in-place input/output array.  If passing in an array of `real` elements, the leading dimension of the array must be padded to store 2(n/2 + 1) elements, where `n` is the size of the corresponding dimension of `realDom`.  If passing in an array of `complex` elements, the leading dimension should be (n/2 + 1).  See the `FFTW documentation <http://www.fftw.org/fftw3_doc/Multi_002dDimensional-DFTs-of-Real-Data.html#Multi_002dDimensional-DFTs-of-Real-Data>`_ for more information.
    :type arr: [] `T` where `T` is of type `real(64)` or `complex(128)`

    :arg flags: the bitwise-or of any planning-rigor or algorithm-restriction flags that should be used in creating the plan (e.g., :const:`FFTW_MEASURE` ``|`` :const:`FFTW_PRESERVE_INPUT`)
    :type flags: c_int

    :returns: The :type:`fftw_plan` representing the resulting plan
   */
  proc plan_dft_c2r(realDom : domain, arr: [?D] ?t, flags : c_uint) : fftw_plan
    where t == real || t == complex
  {
    param rank = realDom.rank: c_int;

    if !noFFTWsizeChecks {
      var error = false;

      for i in 1..rank-1 do
        error |= Private_FFTW.checkDimMismatch(realDom, D, i, "plan_dft_c2r()",
                                               inplace=true);

      if t == real {
        const arrDim = arr.domain.dim(rank).size;
        const domDim = 2*(realDom.dim(rank).size/2+1);
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_c2r(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
      } else {
        const arrDim = arr.domain.dim(rank).size;
        const domDim = realDom.dim(rank).size/2+1;
        if (arrDim != domDim) {
          writeln("Error: In plan_dft_c2r(), the array's leading dimension is not of the expected size (expected ", domDim, ", got ", arrDim, ")");
          error = true;
        }
      }
      if (error) then
        halt("Incorrect array sizes in plan_dft_c2r()");
    }

    var dims: rank*c_int;
    for param i in 1..rank do
      dims(i) = realDom.dim(i).size: c_int;

    return C_FFTW.fftw_plan_dft_c2r(rank, dims, arr, arr, flags);
  }

  pragma "no doc"
  proc plan_dft_c2r(realDom : domain, arr: [] ?t, flags : c_uint) : fftw_plan {
    compilerError("plan_dft_c2r() is only supported for arrays of type real(64) and complex(128)");
  }

  // Using plans

  /*
    Execute an FFTW plan.

    :arg plan: The plan to execute, as computed by a `plan_dft*()` routine.
    :type plan: fftw_plan
  */
  proc execute(const plan: fftw_plan) {
   C_FFTW.fftw_execute(plan);
  }

  /*
    Destroy an FFTW plan.

    :arg plan: The plan to destroy
    :type plan: fftw_plan
  */
  proc destroy_plan(plan: fftw_plan) {
    C_FFTW.fftw_destroy_plan(plan);
  }

  /*
    Clean up FFTW overall.
  */
  proc cleanup() {
    C_FFTW.fftw_cleanup();
  }

  // Define the various planner flags
  // See Sec. 4.3.2 of FFTW manual "Planner Flags"

  //
  // TODO: Is it weird that we use FFTW_ names here, but not in
  // the routines?  Should we wrap the external flags so that
  // they can be ints or enums and avoid the all-caps C scheme
  // or is the familiarity better?
  //

  // Direction flags

  /* Request a forward transform (i.e., use a negative exponent in the tranform). */
  extern const FFTW_FORWARD : c_int;
  /* Request a backward transform (i.e., use a positive exponent in the transofmr). */
  extern const FFTW_BACKWARD : c_int;

  // Planning-rigor flags

  /* Specify that a simple heuristic should be used to pick a plan
     quickly.  This will prevent the input/output arrays from being
     overwritten during planning. */
  extern const FFTW_ESTIMATE : c_uint;

  /* Specify that FFTW should try and find an optimized plan by
     computing several FFTs and measuring their execution time.
     This can consume some time.
  */
  // TODO: If/when we support defaults, might say something like: This
  // is the default planning option.
  extern const FFTW_MEASURE : c_uint;

  /* Specify that FFTW should expend a greater effort finding an
     optimized plan. */
  extern const FFTW_PATIENT : c_uint;

  /* Specify that FFTW should expend an even greater effort finding an
     optimized plan. */
  extern const FFTW_EXHAUSTIVE : c_uint;

  /* This is a special planning mode that is useful for querying
     whether wisdom is available.  When using it, the plan is only
     created when wisdom is available for the given problem; otherwise
     a `null plan` is returned.  This can be combined with other flags
     to create a plan if the wisdom available was created in that mode
     (e.g., ``FFTW_WISDOM_ONLY | FFW_PATIENT``).  For more details on
     this flag and the previous four, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual
  */
  extern const FFTW_WISDOM_ONLY : c_uint;

  // Algorithm-restriction flags

  /* Specify that an out-of-place transform is permitted to overwrite
     its input array with arbitrary data.  This permits more efficient
     algorithms to be used in some cases. */
  // TODO: When we're ready to mention defaults, add: "This is the default for
  // :proc:`plan_dft_c2r`. // NOTE: ...and hc2r once supported...
  extern const FFTW_DESTROY_INPUT : c_uint;

  /* Specify that an out-of-place transform cannot change its input
     array. */
  // TODO: When we're ready to mention defaults, add: This is the
  // default for :proc:`plan_dft` and :proc:`plan_dft_r2c`. */
  extern const FFTW_PRESERVE_INPUT : c_uint;

  /* Specify that the algorithm may not impose any unusual alignment
     requirements on the input/output arrays.  This flag should not be
     necessary for current Chapel use since the planner will
     automatically detect such cases.  For more details on this flag
     and the previous two, refer to `Section 4.3.2
     <http://www.fftw.org/doc/Planner-Flags.html>`_ of the FFTW manual.
  */  // NOTE: But it will be if/when the new-array execute interface is supported
  extern const FFTW_UNALIGNED : c_uint;

  //
  // This is a helper module to give us a "private" notion.  Once we
  // have a private keyword, these could be hoisted to module scope
  // and labeled private.
  //
  pragma "no doc"
  module Private_FFTW {
    //
    // Check that two domain dimensions match, print an error if they
    // don't, and return whether or not an error occurred.  Note that
    // the 'inplace' argument is used to customize the error message
    // for in-place and out-of-place cases.
    //
    proc checkDimMismatch(inDom, outDom, dim, fnname, inplace=false) {
      const inputDim = inDom.dim(dim).size;
      const outputDim = outDom.dim(dim).size;
      if (inputDim == outputDim) then
        return false;

      const first = if inplace then "domain" else "input";
      const second = if inplace then "array" else "output";

      writeln("Error: In ", fnname, ", ", first, " and ", second,
              if !inplace then " arrays" else "",
              " don't have same size in dimension ", dim,
              " (", first, " = ", inputDim, ", ", second, " = ", outputDim,
              ")");
      return true;
    }
  }

  pragma "no doc"
  module C_FFTW {
    extern proc fftw_plan_dft(rank: c_int,
                              n,  // BLC: having trouble being specific
                                  // This is a const int* in C
                              in1: [] complex(128),
                              out1: [] complex(128),
                              sign : c_int, c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_r2c(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                      // This is a const int* in C
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_plan_dft_c2r(rank: c_int,
                                  n,  // BLC: having trouble being specific
                                      // This is a const int* in C
                                  in1: [],
                                  out1: [],
                                  c_flags : c_uint) : fftw_plan;

    extern proc fftw_execute(const plan : fftw_plan);

    extern proc fftw_destroy_plan(plan : fftw_plan);

    extern proc fftw_cleanup();

    extern proc fftw_init_threads() : c_int;

    extern proc fftw_cleanup_threads();

    extern proc fftw_plan_with_nthreads(n : c_int);
  }
}


Remove "no doc" pragma now that chpldoc recognizes private

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  Bitwise operations implemented using C intrinsics when possible.
 */
module BitOps {

  /*
    Count leading zeros in `x`.

    :returns: the number of 0 bits before the most significant 1 bit in `x`
    :rtype: `x.type`
   */
  inline proc clz(x: integral) {
    return BitOps_internal.clz(x);
  }

  /*
    Count trailing zeros in `x`.

    :returns: the number of 0 bits after the least significant 1 bit in `x`
    :rtype: `x.type`
   */
  inline proc ctz(x: integral) {
    return BitOps_internal.ctz(x);
  }

  /*
    Find the population count of `x`.

    :returns: the number of 1 bits set in `x` as `x.type`
    :rtype: `x.type`
   */
  inline proc popcount(x: integral) {
    return BitOps_internal.popcount(x);
  }

  /*
    Find the parity of `x`.

    :returns: * 0 -- when an even number of bits are set in `x`
              * 1 -- when an odd number of bits are set in `x`
    :rtype: `x.type`
   */
  inline proc parity(x: integral) {
    return BitOps_internal.parity(x);
  }

  /*
    Rotate `x` left.

    :arg x: integral of size `bits`
    :arg n: rotation amount, must be less than `bits`

    :returns: `x` rotated left by `n` bits
    :rtype: `x.type`
   */
  inline proc rotl(x: integral, n: integral) {
    return BitOps_internal.rotl(x, n);
  }

  /*
    Rotate `x` right.

    :arg x: integral of size `bits`
    :arg n: rotation amount, must be less than `bits`

    :returns: `x` rotated right by `n` bits
    :rtype: `x.type`
   */
  inline proc rotr(x: integral, n: integral) {
    return BitOps_internal.rotr(x, n);
  }

  // Legacy operations
  // -----------------
  // kyleb: a few tests use these, but I'm not sure if there too much value

  // do a bitwise matrix-multiplication of x and y treating
  // them as 8x8 bit matrices and using | as the combinator
  // function.  There should be a similar version using xor
  // as the combinator, but I'm not sure where that would
  // go to be perfectly truthful
  pragma "no doc"
  proc bitMatMultOr(x: uint(64), y: uint(64)): uint(64) {
    // return the transpose of x, treating it as an 8x8 bit-matrix.
    proc bitMatTrans(x: uint(64))
      return ((x & 0x8040201008040201)       |
              (x & 0x0080402010080402) <<  7 |
              (x & 0x0000804020100804) << 14 |
              (x & 0x0000008040201008) << 21 |
              (x & 0x0000000080402010) << 28 |
              (x & 0x0000000000804020) << 35 |
              (x & 0x0000000000008040) << 42 |
              (x & 0x0000000000000080) << 49 |
              (x >>  7) & 0x0080402010080402 |
              (x >> 14) & 0x0000804020100804 |
              (x >> 21) & 0x0000008040201008 |
              (x >> 28) & 0x0000000080402010 |
              (x >> 35) & 0x0000000000804020 |
              (x >> 42) & 0x0000000000008040 |
              (x >> 49) & 0x0000000000000080);

    // set every eighth bit to one or zero based on the results of the
    // eight reductions of the sets of eight bits
    inline proc byteOrReduce(u: uint(64)) {
      var t1 = u | (u >> 4);
      var t2 = t1 | (t1 >> 2);
      var t3 = t2 | (t2 >> 1);
      return t3 & 0x0101010101010101;
    }

    // set every bit in sets of eight to one if the last one is one
    inline proc byteExpand(u: uint(64))
      return 0x8080808080808080 ^ (0x8080808080808080 - u);

    var result:uint(64) = 0;
    var yTranspose = bitMatTrans(y);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x8040201008040201;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x4020100804020180;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x2010080402018040;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x1008040201804020;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0804020180402010;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0402018040201008;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0201804020100804;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0180402010080402;
    return result;
  }

}

/**
 * module to hide the extern procedures
 */
private module BitOps_internal {
  extern proc chpl_bitops_popcount_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_popcount_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_clz_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_clz_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_ctz_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_ctz_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_parity_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_parity_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_rotl_8(x: uint(8), n: uint(8)) : uint(8);
  extern proc chpl_bitops_rotl_16(x: uint(16), n: uint(16)) : uint(16);
  extern proc chpl_bitops_rotl_32(x: uint(32), n: uint(32)) : uint(32);
  extern proc chpl_bitops_rotl_64(x: uint(64), n: uint(64)) : uint(64);

  extern proc chpl_bitops_rotr_8(x: uint(8), n: uint(8)) : uint(8);
  extern proc chpl_bitops_rotr_16(x: uint(16), n: uint(16)) : uint(16);
  extern proc chpl_bitops_rotr_32(x: uint(32), n: uint(32)) : uint(32);
  extern proc chpl_bitops_rotr_64(x: uint(64), n: uint(64)) : uint(64);

  inline proc clz(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_clz_64(x);
      when 32 do
        return chpl_bitops_clz_32(x);
      when 16 do
        // gets promoted to 32bit, subtract the extra leading 0s
        return (chpl_bitops_clz_32(x)-16):uint(16);
      when 8 do
        // gets promoted to 32bit, subtract the extra leading 0s
        return (chpl_bitops_clz_32(x)-24):uint(8);
      otherwise
        // NOTE: this actually cant happen with how the integer types are setup
        //       right now - leaving it here for the future when we support
        //       >64bit types
        compilerError("clz is not supported for that bit width.");
    }
  }

  inline proc clz(x: int(?bits)) {
    return BitOps_internal.clz(x:uint(bits)):int(bits);
  }

  inline proc ctz(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_ctz_64(x);
      when 32 do
        return chpl_bitops_ctz_32(x);
      when 16 do
        return chpl_bitops_ctz_32(x):uint(16);
      when 8 do
        return chpl_bitops_ctz_32(x):uint(8);
      otherwise
        // NOTE: this actually cant happen with how the integer types are setup
        //       right now - leaving it here for the future when we support
        //       >64bit types
        compilerError("ctz is not supported for that bit width.");
    }
  }

  inline proc ctz(x: int(?bits)) {
    return BitOps_internal.ctz(x:uint(bits)):int(bits);
  }

  inline proc popcount(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_popcount_64(x);
      when 32 do
        return chpl_bitops_popcount_32(x);
      when 16 do
        return chpl_bitops_popcount_32(x):uint(16);
      when 8 do
        return chpl_bitops_popcount_32(x):uint(8);
      // In the future we could also break a large int (128+) into 64bit chucks
      // and add up the results.
      otherwise
        // NOTE: this actually cant happen with how the integer types are setup
        //       right now - leaving it here for the future when we support
        //       >64bit types
        compilerError("popcount is not supported for that bit width.");
    }
  }

  inline proc popcount(x: int(?bits)) {
    return BitOps_internal.popcount(x:uint(bits)):int(bits);
  }

  inline proc parity(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_parity_64(x);
      when 32 do
        return chpl_bitops_parity_32(x);
      when 16 do
        return chpl_bitops_parity_32(x):uint(16);
      when 8 do
        return chpl_bitops_parity_32(x):uint(8);
      otherwise
        compilerError("parity is not supported for that bit width.");
    }
  }

  inline proc parity(x: int(?bits)) {
    return BitOps_internal.parity(x:uint(bits)):int(bits);
  }

  inline proc rotl(x: uint(?bits), n: integral) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_rotl_64(x, n:uint(bits));
      when 32 do
        return chpl_bitops_rotl_32(x, n:uint(bits));
      when 16 do
        return chpl_bitops_rotl_16(x, n:uint(bits));
      when 8 do
        return chpl_bitops_rotl_8(x, n:uint(bits));
      otherwise
        compilerError("rotl is not supported for that bit width.");
    }
  }

  inline proc rotl(x: int(?bits), n: integral) {
    return BitOps_internal.rotl(x:uint(bits), n):int(bits);
  }

  inline proc rotr(x: uint(?bits), n: integral) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_rotr_64(x, n:uint(bits));
      when 32 do
        return chpl_bitops_rotr_32(x, n:uint(bits));
      when 16 do
        return chpl_bitops_rotr_16(x, n:uint(bits));
      when 8 do
        return chpl_bitops_rotr_8(x, n:uint(bits));
      otherwise
        compilerError("rotr is not supported for that bit width.");
    }
  }

  inline proc rotr(x: int(?bits), n: integral) {
    return BitOps_internal.rotr(x:uint(bits), n):int(bits);
  }
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  Bitwise operations implemented using C intrinsics when possible.
 */
module BitOps {

  /*
    Count leading zeros in `x`.

    :returns: the number of 0 bits before the most significant 1 bit in `x`
    :rtype: `x.type`
   */
  inline proc clz(x: integral) {
    return BitOps_internal.clz(x);
  }

  /*
    Count trailing zeros in `x`.

    :returns: the number of 0 bits after the least significant 1 bit in `x`
    :rtype: `x.type`
   */
  inline proc ctz(x: integral) {
    return BitOps_internal.ctz(x);
  }

  /*
    Find the population count of `x`.

    :returns: the number of 1 bits set in `x` as `x.type`
    :rtype: `x.type`
   */
  inline proc popcount(x: integral) {
    return BitOps_internal.popcount(x);
  }

  /*
    Find the parity of `x`.

    :returns: * 0 -- when an even number of bits are set in `x`
              * 1 -- when an odd number of bits are set in `x`
    :rtype: `x.type`
   */
  inline proc parity(x: integral) {
    return BitOps_internal.parity(x);
  }

  /*
    Rotate `x` left.

    :arg x: integral of size `bits`
    :arg n: rotation amount, must be less than `bits`

    :returns: `x` rotated left by `n` bits
    :rtype: `x.type`
   */
  inline proc rotl(x: integral, n: integral) {
    return BitOps_internal.rotl(x, n);
  }

  /*
    Rotate `x` right.

    :arg x: integral of size `bits`
    :arg n: rotation amount, must be less than `bits`

    :returns: `x` rotated right by `n` bits
    :rtype: `x.type`
   */
  inline proc rotr(x: integral, n: integral) {
    return BitOps_internal.rotr(x, n);
  }

  // Legacy operations
  // -----------------
  // kyleb: a few tests use these, but I'm not sure if there too much value

  // do a bitwise matrix-multiplication of x and y treating
  // them as 8x8 bit matrices and using | as the combinator
  // function.  There should be a similar version using xor
  // as the combinator, but I'm not sure where that would
  // go to be perfectly truthful
  pragma "no doc"
  proc bitMatMultOr(x: uint(64), y: uint(64)): uint(64) {
    // return the transpose of x, treating it as an 8x8 bit-matrix.
    proc bitMatTrans(x: uint(64))
      return ((x & 0x8040201008040201)       |
              (x & 0x0080402010080402) <<  7 |
              (x & 0x0000804020100804) << 14 |
              (x & 0x0000008040201008) << 21 |
              (x & 0x0000000080402010) << 28 |
              (x & 0x0000000000804020) << 35 |
              (x & 0x0000000000008040) << 42 |
              (x & 0x0000000000000080) << 49 |
              (x >>  7) & 0x0080402010080402 |
              (x >> 14) & 0x0000804020100804 |
              (x >> 21) & 0x0000008040201008 |
              (x >> 28) & 0x0000000080402010 |
              (x >> 35) & 0x0000000000804020 |
              (x >> 42) & 0x0000000000008040 |
              (x >> 49) & 0x0000000000000080);

    // set every eighth bit to one or zero based on the results of the
    // eight reductions of the sets of eight bits
    inline proc byteOrReduce(u: uint(64)) {
      var t1 = u | (u >> 4);
      var t2 = t1 | (t1 >> 2);
      var t3 = t2 | (t2 >> 1);
      return t3 & 0x0101010101010101;
    }

    // set every bit in sets of eight to one if the last one is one
    inline proc byteExpand(u: uint(64))
      return 0x8080808080808080 ^ (0x8080808080808080 - u);

    var result:uint(64) = 0;
    var yTranspose = bitMatTrans(y);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x8040201008040201;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x4020100804020180;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x2010080402018040;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x1008040201804020;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0804020180402010;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0402018040201008;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0201804020100804;
    yTranspose = rotl(yTranspose, 8);
    result |= byteExpand(byteOrReduce(x & yTranspose)) & 0x0180402010080402;
    return result;
  }

}

/**
 * module to hide the extern procedures
 */
pragma "no doc"
private module BitOps_internal {
  extern proc chpl_bitops_popcount_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_popcount_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_clz_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_clz_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_ctz_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_ctz_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_parity_32(x: c_uint) : uint(32);
  extern proc chpl_bitops_parity_64(x: c_ulonglong) : uint(64);

  extern proc chpl_bitops_rotl_8(x: uint(8), n: uint(8)) : uint(8);
  extern proc chpl_bitops_rotl_16(x: uint(16), n: uint(16)) : uint(16);
  extern proc chpl_bitops_rotl_32(x: uint(32), n: uint(32)) : uint(32);
  extern proc chpl_bitops_rotl_64(x: uint(64), n: uint(64)) : uint(64);

  extern proc chpl_bitops_rotr_8(x: uint(8), n: uint(8)) : uint(8);
  extern proc chpl_bitops_rotr_16(x: uint(16), n: uint(16)) : uint(16);
  extern proc chpl_bitops_rotr_32(x: uint(32), n: uint(32)) : uint(32);
  extern proc chpl_bitops_rotr_64(x: uint(64), n: uint(64)) : uint(64);

  inline proc clz(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_clz_64(x);
      when 32 do
        return chpl_bitops_clz_32(x);
      when 16 do
        // gets promoted to 32bit, subtract the extra leading 0s
        return (chpl_bitops_clz_32(x)-16):uint(16);
      when 8 do
        // gets promoted to 32bit, subtract the extra leading 0s
        return (chpl_bitops_clz_32(x)-24):uint(8);
      otherwise
        // NOTE: this actually cant happen with how the integer types are setup
        //       right now - leaving it here for the future when we support
        //       >64bit types
        compilerError("clz is not supported for that bit width.");
    }
  }

  inline proc clz(x: int(?bits)) {
    return BitOps_internal.clz(x:uint(bits)):int(bits);
  }

  inline proc ctz(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_ctz_64(x);
      when 32 do
        return chpl_bitops_ctz_32(x);
      when 16 do
        return chpl_bitops_ctz_32(x):uint(16);
      when 8 do
        return chpl_bitops_ctz_32(x):uint(8);
      otherwise
        // NOTE: this actually cant happen with how the integer types are setup
        //       right now - leaving it here for the future when we support
        //       >64bit types
        compilerError("ctz is not supported for that bit width.");
    }
  }

  inline proc ctz(x: int(?bits)) {
    return BitOps_internal.ctz(x:uint(bits)):int(bits);
  }

  inline proc popcount(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_popcount_64(x);
      when 32 do
        return chpl_bitops_popcount_32(x);
      when 16 do
        return chpl_bitops_popcount_32(x):uint(16);
      when 8 do
        return chpl_bitops_popcount_32(x):uint(8);
      // In the future we could also break a large int (128+) into 64bit chucks
      // and add up the results.
      otherwise
        // NOTE: this actually cant happen with how the integer types are setup
        //       right now - leaving it here for the future when we support
        //       >64bit types
        compilerError("popcount is not supported for that bit width.");
    }
  }

  inline proc popcount(x: int(?bits)) {
    return BitOps_internal.popcount(x:uint(bits)):int(bits);
  }

  inline proc parity(x: uint(?bits)) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_parity_64(x);
      when 32 do
        return chpl_bitops_parity_32(x);
      when 16 do
        return chpl_bitops_parity_32(x):uint(16);
      when 8 do
        return chpl_bitops_parity_32(x):uint(8);
      otherwise
        compilerError("parity is not supported for that bit width.");
    }
  }

  inline proc parity(x: int(?bits)) {
    return BitOps_internal.parity(x:uint(bits)):int(bits);
  }

  inline proc rotl(x: uint(?bits), n: integral) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_rotl_64(x, n:uint(bits));
      when 32 do
        return chpl_bitops_rotl_32(x, n:uint(bits));
      when 16 do
        return chpl_bitops_rotl_16(x, n:uint(bits));
      when 8 do
        return chpl_bitops_rotl_8(x, n:uint(bits));
      otherwise
        compilerError("rotl is not supported for that bit width.");
    }
  }

  inline proc rotl(x: int(?bits), n: integral) {
    return BitOps_internal.rotl(x:uint(bits), n):int(bits);
  }

  inline proc rotr(x: uint(?bits), n: integral) {
    // the select will be folded out at compile time.
    select bits {
      when 64 do
        return chpl_bitops_rotr_64(x, n:uint(bits));
      when 32 do
        return chpl_bitops_rotr_32(x, n:uint(bits));
      when 16 do
        return chpl_bitops_rotr_16(x, n:uint(bits));
      when 8 do
        return chpl_bitops_rotr_8(x, n:uint(bits));
      otherwise
        compilerError("rotr is not supported for that bit width.");
    }
  }

  inline proc rotr(x: int(?bits), n: integral) {
    return BitOps_internal.rotr(x:uint(bits), n):int(bits);
  }
}