Another session's work on HPCC FFT -- this isn't a particularly
clean snapshot, but I'm unlikely to get back to it in the next
few days, so wanted to capture where I was.  Realized after this
pass that I should be using complex types, which ought to simplify
a lot of the ridiculousness in the current code.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@5834 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config var logN: int = 4;

// PLACEHOLDER for external stuff
class timer {
  fun start();
  fun stop();
  fun gettime() { return 1.0; }
}
// END PLACEHOLDER

fun main() {
  const N = 1 << logN;
  const N2 = 2 * N;
  const gflop = 5.0 * N * logN / 1000000000.0;

  const EPS = 2.0 ** -51.0;
  const THRESHOLD = 16.0;

// BLC: const domains not supported (constdomain.chpl)
//  const DN2: domain(1) = [1..N2];
// rewritten as var:
  var DN2: domain(1) = [1..N2];
// BLC: subdomains not supported (subdomain.chpl)
//  const DOdd: subdomain(DN2) = DN2 by 2;
// BLC: const domains not supported (constdomain.chpl)
//  const DOdd: domain(1) = DN2 by 2;
  var DOdd: domain(1) = DN2 by 2;
//  var DOdd: domain(1) = [1..N2 by 2];
  var DNhalf: domain(1) = [0..N/2 - 1];

  var A: [DN2] float;
  var B: [DN2] float;
  var W: [DNhalf] float;

  prand_array(A);

  B = A;  // save A for verification step

  twiddles(W);
  W = bit_reverse(W);

  forall i in DOdd do
    A(i) = -A(i);

  A = bit_reverse(A);

  forall i in DOdd do

  dfft(A, W);

  forall i in DOdd {
    A(i) = A(i) / N;
    A(i+1) = -A(i+1) / N;
  }

  var fftTimer = timer();
  fftTimer.start();

  A = bit_reverse(A);
  dfft(A, W);

  fftTimer.stop();
  var time = fftTimer.gettime();

// BLC: reductions not working yet...
  var maxerr = max reduce [i in DOdd] sqrt((B(i) - A(i))**2 +
                                           (B(i+1) - A(i+1))**2);

  maxerr = maxerr / logN / EPS;

  var status = if (maxerr < THRESHOLD) then "SUCCESS" else "FAILURE";
  writeln(status, ", error = ", maxerr);
  writeln("\n");

  writeln("N      = ", N);
  writeln("Time   = ", time);
  writeln("GFlops = ", gflop / time);
}

fun bit_reverse(W) {
//  const WD = W.domain;
  const n = WD.extent();
//  const mask: uint = 0x0102040810204080;
  const shift = (log(n) / log(2)): integer;

  for i in 1..n {
    const ndx = MTA_BIT_MAT_OR(mask, MTA_BIT_MAT_OR(i, mask));
    ndx = MTA_ROTATE_LEFT(ndx, shift);
    v(2*ndx) = w(2*i);
    v(2*ndx + 1) = w(2*i + 1);
  }

  return V;
}

fun dfft(n, logn, a, w) {
  cf1st(n, a, w);

  var l = 8;
  for i in 4..logn/2 by 2 {
    l *= 4;
    cftmd1(n, l, a, w);
  }
//  for i =
}

fun prand_array(X) {
}

//fun twiddles(W: [?DW] float) {
fun twiddles(W) {
//  const DW = W.domain();
  const n = DW.extent() / 2;
  const delta = atan(1.0) / n;

  W(0) = 1.0;
  W(1) = 0.0;
  W(n) = cos(delta * n);
  W(n+1) = W(n);
  for i in 2..n-1 by 2 {
    const x = cos(delta * i);
    const y = sin(delta * i);
    W(i)           = x;
    W(i+1)         = y;
    W(2*n - i)     = y;
    W(2*n - i + 1) = x;
  }
  writeln(W);
}

/*

#include <math.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mta_task.h>
#include <machine/runtime.h>

double timer()
{ return ((double) mta_get_clock(0) / mta_clock_freq()); }

#pragma mta inline
void btrfly(j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d)
  int j;
  double wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, *a, *b, *c, *d;
{ double x0r = a[j    ] + b[j    ];
  double x0i = a[j + 1] + b[j + 1];
  double x1r = a[j    ] - b[j    ];
  double x1i = a[j + 1] - b[j + 1];
  double x2r = c[j    ] + d[j    ];
  double x2i = c[j + 1] + d[j + 1];
  double x3r = c[j    ] - d[j    ];
  double x3i = c[j + 1] - d[j + 1];

  a[j    ] = x0r + x2r;
  a[j + 1] = x0i + x2i;
  x0r -= x2r;
  x0i -= x2i;
  c[j    ] = wk2r * x0r - wk2i * x0i;
  c[j + 1] = wk2r * x0i + wk2i * x0r;
  x0r = x1r - x3i;
  x0i = x1i + x3r;
  b[j    ] = wk1r * x0r - wk1i * x0i;
  b[j + 1] = wk1r * x0i + wk1i * x0r;
  x0r = x1r + x3i;
  x0i = x1i - x3r;
  d[j    ] = wk3r * x0r - wk3i * x0i;
  d[j + 1] = wk3r * x0i + wk3i * x0r;
}

double * bit_reverse(int n, double *w) {
  unsigned int i, mask, shift;
  double *v = new double[2 * n];

  mask  = 0x0102040810204080;
  shift = (int) (log(n) / log(2));

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (i = 0; i < n; i++) {
      int ndx = MTA_BIT_MAT_OR(mask, MTA_BIT_MAT_OR(i, mask));
      ndx = MTA_ROTATE_LEFT(ndx, shift);
      v[2 * ndx]     = w[2 * i];
      v[2 * ndx + 1] = w[2 * i + 1];
  }

  free(w);
  return(v);
}

void twiddles(int n, double *w)
{ int i;
  double delta = atan(1.0) / n;

  w[0]     = 1;
  w[1]     = 0;
  w[n]     = cos(delta * n);
  w[n + 1] = w[n];

#pragma mta assert no dependence
  for (i = 2; i < n; i += 2) {
      double x = cos(delta * i);
      double y = sin(delta * i);
      w[i]             = x;
      w[i + 1]         = y;
      w[2 * n - i]     = y;
      w[2 * n - i + 1] = x;
} }

void cft1st(int n, double *a, double *w)
{ int j, k1;

  double *v   = w + 1;
  double *b   = a + 2;
  double *c   = a + 4;
  double *d   = a + 6;
  double wk1r = w[2];

  double x0r = a[0] + a[2];
  double x0i = a[1] + a[3];
  double x1r = a[0] - a[2];
  double x1i = a[1] - a[3];
  double x2r = a[4] + a[6];
  double x2i = a[5] + a[7];
  double x3r = a[4] - a[6];
  double x3i = a[5] - a[7];

  a[0] = x0r + x2r;
  a[1] = x0i + x2i;
  a[4] = x0r - x2r;
  a[5] = x0i - x2i;
  a[2] = x1r - x3i;
  a[3] = x1i + x3r;
  a[6] = x1r + x3i;
  a[7] = x1i - x3r;

  x0r  = a[8]  + a[10];
  x0i  = a[9]  + a[11];
  x1r  = a[8]  - a[10];
  x1i  = a[9]  - a[11];
  x2r  = a[12] + a[14];
  x2i  = a[13] + a[15];
  x3r  = a[12] - a[14];
  x3i  = a[13] - a[15];
  a[8] = x0r + x2r;
  a[9] = x0i + x2i;
  a[12] = x2i - x0i;
  a[13] = x0r - x2r;
  x0r   = x1r - x3i;
  x0i   = x1i + x3r;
  a[10] = wk1r * (x0r - x0i);
  a[11] = wk1r * (x0r + x0i);
  x0r   = x3i + x1r;
  x0i   = x3r - x1i;
  a[14] = wk1r * (x0i - x0r);
  a[15] = wk1r * (x0i + x0r);

#pragma mta use 100 streams
#pragma mta no scalar expansion
#pragma mta assert no dependence
  for (j = 16, k1 = 2; j < n; j += 16, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

      btrfly (j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);

      wk1r = w[k1 + k1 + 2];
      wk1i = v[k1 + k1 + 2];
      wk3r = wk1r - 2 * wk2r * wk1i;
      wk3i = 2 * wk2r * wk1r - wk1i;

      btrfly (j + 8, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);
} }

void cftmd0(int n, int l, double *a, double *w)
{ int j, m = l << 2;

  double wk1r = w[2];

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = 0; j < l; j += 2)
      btrfly(j, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, a, b, c, d);

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = m; j < l + m; j += 2)
      btrfly(j, wk1r, wk1r, 0.0, 1.0, - wk1r, wk1r, a, b, c, d);
}

void cftmd1(int n, int l, double *a, double *w)
{ int j, k, k1;

  int m  = l << 2;
  int m2 = 2 * m;

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

  cftmd0(n, l, a, w);

#pragma mta use 100 streams
#pragma mta no scalar expansion
#pragma mta assert no dependence
  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

      for (j = k; j < l + k; j += 2)
          btrfly (j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);

      wk1r = w[k1 + k1 + 2];
      wk1i = v[k1 + k1 + 2];
      wk3r = wk1r - 2 * wk2r * wk1i;
      wk3i = 2 * wk2r * wk1r - wk1i;

      for (j = k + m; j < l + k + m; j += 2)
          btrfly (j, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);
} }

void cftmd21(int n, int l, double *a, double *w)
{ int j, k, k1;
  int m  = l << 2;
  int m2 = 2 * m;
  int m3 = 3 * m;

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = k; j < k + l; j += 2)
      btrfly (j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);

      wk1r = w[k1 + k1 + 2];
      wk1i = v[k1 + k1 + 2];
      wk3r = wk1r - 2 * wk2r * wk1i;
      wk3i = 2 * wk2r * wk1r - wk1i;

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = k + m; j < k + m + l; j += 2)
      btrfly (j, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);

} }

void cftmd2(int n, int l, double *a, double *w)
{ int j, k, k1;

  int m  = l << 2;
  int m2 = 2 * m;

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

  cftmd0(n, l, a, w);

  if (m2 >= n) return;
  if (m2 >= n / 8) {cftmd21(n, l, a, w); return;}

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = 0; j < l; j += 2)  {
#pragma mta assert no dependence
  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

      btrfly (j + k, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);
  }

#pragma mta assert no dependence
  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1 + 2];
      double wk1i = v[k1 + k1 + 2];
      double wk3r = wk1r - 2 * wk2r * wk1i;
      double wk3i = 2 * wk2r * wk1r - wk1i;

      btrfly (j + k + m, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);

} } }

void dfft(int n, int logn, double *a, double *w)
{ int i, l, j;
  double *v, *b, *c, *d;

  cft1st(n, a, w);

  i = 4; l = 8;

  for ( ; i <= logn / 2; i += 2, l *= 4) cftmd1(n, l, a, w);
  for ( ; i <= logn - 1; i += 2, l *= 4) cftmd2(n, l, a, w);

  v = w + 1;
  b = a + l;
  c = a + l + l;
  d = a + l + l + l;

  if ((l << 2) == n) {

#pragma mta use 100 streams
#pragma mta assert no dependence
     for (j = 0; j < l; j += 2)
         btrfly(j, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, a, b, c, d);

  } else {

#pragma mta use 100 streams
#pragma mta assert no dependence
     for (j = 0; j < l; j += 2) {
         double x0r = a[j    ];
         double x0i = a[j + 1];
         double x1r = b[j    ];
         double x1i = b[j + 1];
         a[j    ]   = x0r + x1r;
         a[j + 1]   = x0i + x1i;
         b[j    ]   = x0r - x1r;
         b[j + 1]   = x0i - x1i;
} }  }

*/

config var logN: int = 4;

// PLACEHOLDER for external stuff
class timer {
  fun start();
  fun stop();
  fun gettime() { return 1.0; }
}
// END PLACEHOLDER

fun main() {
// BLC: left-shift not supported (shiftops.chpl)
//  const N = 1 << logN;
// rewritten as exponentiation:
  const N = 2 ** logN;
  const N2 = 2 * N;
  const gflop = 5.0 * N * logN / 1000000000.0;

  const EPS = 2.0 ** -51.0;
  const THRESHOLD = 16.0;

// BLC: const domains not supported (constdomain.chpl)
//  const DN2: domain(1) = [1..N2];
// rewritten as var:
  var DN2: domain(1) = [1..N2];
// BLC: subdomains not supported (subdomain.chpl)
//  const DOdd: subdomain(DN2) = DN2 by 2;
// BLC: const domains not supported (constdomain.chpl)
//  const DOdd: domain(1) = DN2 by 2;
// BLC: by operator not supported (bydomain.chpl)
//  var DOdd: domain(1) = DN2 by 2;
  var DOdd: domain(1) = [1..N2 by 2];
  var DNhalf: domain(1) = [1..N/2];

  var A: [DN2] float;
  var B: [DN2] float;
  var W: [DNhalf] float;

  prand_array(A);

  B = A;  // save A for verification step

  twiddles(W);
  W = bit_reverse(W);

  forall i in DOdd do
    A(i) = -A(i);

  A = bit_reverse(A);

  forall i in DOdd do

  dfft(A, W);

  forall i in DOdd {
    A(i) = A(i) / N;
    A(i+1) = -A(i+1) / N;
  }

  var fftTimer = timer();
  fftTimer.start();

  A = bit_reverse(A);
  dfft(A, W);

  fftTimer.stop();
  var time = fftTimer.gettime();

/*
  var maxerr = max reduce [i in DOdd] sqrt((B(i) - A(i))**2 +
                                           (B(i+1) - A(i+1))**2);
*/
  var maxerr = 42.1;

  maxerr = maxerr / logN / EPS;

  var status = if (maxerr < THRESHOLD) then "SUCCESS" else "FAILURE";
  writeln(status, ", error = ", maxerr);
  writeln("\n");

  writeln("N      = ", N);
  writeln("Time   = ", time);
  writeln("GFlops = ", gflop / time);
}

fun bit_reverse(X) {
  return X;
}

fun dfft(X, Y) {
}

fun prand_array(X) {
}

fun twiddles(X) {
}

/*

#include <math.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mta_task.h>
#include <machine/runtime.h>

double timer()
{ return ((double) mta_get_clock(0) / mta_clock_freq()); }

#pragma mta inline
void btrfly(j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d)
  int j;
  double wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, *a, *b, *c, *d;
{ double x0r = a[j    ] + b[j    ];
  double x0i = a[j + 1] + b[j + 1];
  double x1r = a[j    ] - b[j    ];
  double x1i = a[j + 1] - b[j + 1];
  double x2r = c[j    ] + d[j    ];
  double x2i = c[j + 1] + d[j + 1];
  double x3r = c[j    ] - d[j    ];
  double x3i = c[j + 1] - d[j + 1];

  a[j    ] = x0r + x2r;
  a[j + 1] = x0i + x2i;
  x0r -= x2r;
  x0i -= x2i;
  c[j    ] = wk2r * x0r - wk2i * x0i;
  c[j + 1] = wk2r * x0i + wk2i * x0r;
  x0r = x1r - x3i;
  x0i = x1i + x3r;
  b[j    ] = wk1r * x0r - wk1i * x0i;
  b[j + 1] = wk1r * x0i + wk1i * x0r;
  x0r = x1r + x3i;
  x0i = x1i - x3r;
  d[j    ] = wk3r * x0r - wk3i * x0i;
  d[j + 1] = wk3r * x0i + wk3i * x0r;
}

double * bit_reverse(int n, double *w) {
  unsigned int i, mask, shift;
  double *v = new double[2 * n];

  mask  = 0x0102040810204080;
  shift = (int) (log(n) / log(2));

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (i = 0; i < n; i++) {
      int ndx = MTA_BIT_MAT_OR(mask, MTA_BIT_MAT_OR(i, mask));
      ndx = MTA_ROTATE_LEFT(ndx, shift);
      v[2 * ndx]     = w[2 * i];
      v[2 * ndx + 1] = w[2 * i + 1];
  }

  free(w);
  return(v);
}

void twiddles(int n, double *w)
{ int i;
  double delta = atan(1.0) / n;

  w[0]     = 1;
  w[1]     = 0;
  w[n]     = cos(delta * n);
  w[n + 1] = w[n];

#pragma mta assert no dependence
  for (i = 2; i < n; i += 2) {
      double x = cos(delta * i);
      double y = sin(delta * i);
      w[i]             = x;
      w[i + 1]         = y;
      w[2 * n - i]     = y;
      w[2 * n - i + 1] = x;
} }

void cft1st(int n, double *a, double *w)
{ int j, k1;

  double *v   = w + 1;
  double *b   = a + 2;
  double *c   = a + 4;
  double *d   = a + 6;
  double wk1r = w[2];

  double x0r = a[0] + a[2];
  double x0i = a[1] + a[3];
  double x1r = a[0] - a[2];
  double x1i = a[1] - a[3];
  double x2r = a[4] + a[6];
  double x2i = a[5] + a[7];
  double x3r = a[4] - a[6];
  double x3i = a[5] - a[7];

  a[0] = x0r + x2r;
  a[1] = x0i + x2i;
  a[4] = x0r - x2r;
  a[5] = x0i - x2i;
  a[2] = x1r - x3i;
  a[3] = x1i + x3r;
  a[6] = x1r + x3i;
  a[7] = x1i - x3r;

  x0r  = a[8]  + a[10];
  x0i  = a[9]  + a[11];
  x1r  = a[8]  - a[10];
  x1i  = a[9]  - a[11];
  x2r  = a[12] + a[14];
  x2i  = a[13] + a[15];
  x3r  = a[12] - a[14];
  x3i  = a[13] - a[15];
  a[8] = x0r + x2r;
  a[9] = x0i + x2i;
  a[12] = x2i - x0i;
  a[13] = x0r - x2r;
  x0r   = x1r - x3i;
  x0i   = x1i + x3r;
  a[10] = wk1r * (x0r - x0i);
  a[11] = wk1r * (x0r + x0i);
  x0r   = x3i + x1r;
  x0i   = x3r - x1i;
  a[14] = wk1r * (x0i - x0r);
  a[15] = wk1r * (x0i + x0r);

#pragma mta use 100 streams
#pragma mta no scalar expansion
#pragma mta assert no dependence
  for (j = 16, k1 = 2; j < n; j += 16, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

      btrfly (j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);

      wk1r = w[k1 + k1 + 2];
      wk1i = v[k1 + k1 + 2];
      wk3r = wk1r - 2 * wk2r * wk1i;
      wk3i = 2 * wk2r * wk1r - wk1i;

      btrfly (j + 8, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);
} }

void cftmd0(int n, int l, double *a, double *w)
{ int j, m = l << 2;

  double wk1r = w[2];

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = 0; j < l; j += 2)
      btrfly(j, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, a, b, c, d);

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = m; j < l + m; j += 2)
      btrfly(j, wk1r, wk1r, 0.0, 1.0, - wk1r, wk1r, a, b, c, d);
}

void cftmd1(int n, int l, double *a, double *w)
{ int j, k, k1;

  int m  = l << 2;
  int m2 = 2 * m;

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

  cftmd0(n, l, a, w);

#pragma mta use 100 streams
#pragma mta no scalar expansion
#pragma mta assert no dependence
  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

      for (j = k; j < l + k; j += 2)
          btrfly (j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);

      wk1r = w[k1 + k1 + 2];
      wk1i = v[k1 + k1 + 2];
      wk3r = wk1r - 2 * wk2r * wk1i;
      wk3i = 2 * wk2r * wk1r - wk1i;

      for (j = k + m; j < l + k + m; j += 2)
          btrfly (j, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);
} }

void cftmd21(int n, int l, double *a, double *w)
{ int j, k, k1;
  int m  = l << 2;
  int m2 = 2 * m;
  int m3 = 3 * m;

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = k; j < k + l; j += 2)
      btrfly (j, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);

      wk1r = w[k1 + k1 + 2];
      wk1i = v[k1 + k1 + 2];
      wk3r = wk1r - 2 * wk2r * wk1i;
      wk3i = 2 * wk2r * wk1r - wk1i;

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = k + m; j < k + m + l; j += 2)
      btrfly (j, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);

} }

void cftmd2(int n, int l, double *a, double *w)
{ int j, k, k1;

  int m  = l << 2;
  int m2 = 2 * m;

  double *v = w + 1;
  double *b = a + l;
  double *c = a + l + l;
  double *d = a + l + l + l;

  cftmd0(n, l, a, w);

  if (m2 >= n) return;
  if (m2 >= n / 8) {cftmd21(n, l, a, w); return;}

#pragma mta use 100 streams
#pragma mta assert no dependence
  for (j = 0; j < l; j += 2)  {
#pragma mta assert no dependence
  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1];
      double wk1i = v[k1 + k1];
      double wk3r = wk1r - 2 * wk2i * wk1i;
      double wk3i = 2 * wk2i * wk1r - wk1i;

      btrfly (j + k, wk1r, wk1i, wk2r, wk2i, wk3r, wk3i, a, b, c, d);
  }

#pragma mta assert no dependence
  for (k = m2, k1 = 2; k < n; k += m2, k1 += 2) {
      double wk2r = w[k1];
      double wk2i = v[k1];
      double wk1r = w[k1 + k1 + 2];
      double wk1i = v[k1 + k1 + 2];
      double wk3r = wk1r - 2 * wk2r * wk1i;
      double wk3i = 2 * wk2r * wk1r - wk1i;

      btrfly (j + k + m, wk1r, wk1i, - wk2i, wk2r, wk3r, wk3i, a, b, c, d);

} } }

void dfft(int n, int logn, double *a, double *w)
{ int i, l, j;
  double *v, *b, *c, *d;

  cft1st(n, a, w);

  i = 4; l = 8;

  for ( ; i <= logn / 2; i += 2, l *= 4) cftmd1(n, l, a, w);
  for ( ; i <= logn - 1; i += 2, l *= 4) cftmd2(n, l, a, w);

  v = w + 1;
  b = a + l;
  c = a + l + l;
  d = a + l + l + l;

  if ((l << 2) == n) {

#pragma mta use 100 streams
#pragma mta assert no dependence
     for (j = 0; j < l; j += 2)
         btrfly(j, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, a, b, c, d);

  } else {

#pragma mta use 100 streams
#pragma mta assert no dependence
     for (j = 0; j < l; j += 2) {
         double x0r = a[j    ];
         double x0i = a[j + 1];
         double x1r = b[j    ];
         double x1i = b[j + 1];
         a[j    ]   = x0r + x1r;
         a[j + 1]   = x0i + x1i;
         b[j    ]   = x0r - x1r;
         b[j + 1]   = x0i - x1i;
} }  }

*/


Getting Random --enable-parallel-iterators safe

Implemented fillRandom() to use sequential iteration if
the array being filled is >1D.  This is a short-term
measure until we have a better multidimensional parallel
iterator story worked out, and is being done to allow
--enable-parallel-iterators to be removed.

This version also inserts a delete for the local clone
of the RandomStream class that I create in a follower
iterator, to plug that memory leak.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14738 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// This random number generator was written from the NPB
// random number generator.
//
// It's the linear congruential generator with
//
// m = 2^46 (the modulus)
// a = 1220703125.0 (the multiplier)
// c = 0 (the increment)
//
// It requires an odd integer for its seed.

use Time;

config param debugParRandom: bool = false;

class SeedGeneratorClass {
  def clockMS {
    return getCurrentTime(unit=TimeUnits.microseconds):int(64);
  }
}

var SeedGenerator = new SeedGeneratorClass();

class RandomStream {
  const seed:int(64) = SeedGenerator.clockMS;
  const arand = 1220703125.0;

  // These are meant to be private; internal to class; not set by user
  const r23   = 0.5**23,
        t23   = 2.0**23,
        r46   = 0.5**46,
        t46   = 2.0**46;
  const internalSeed = computeInternalSeed();

  var lock$: sync bool = false;
  var sharedCursor = internalSeed:real;
  var sharedCount: int(64) = 1;

  def computeInternalSeed() {
    var val: int(64) = seed;
    // ensure seed is odd
    if (val % 2 == 0) then val += 1;
    return val;
  }

  def initCursorVal() {
    sharedCursor = internalSeed:real;
    sharedCount = 1;
  }

  def randlc(inout x, a = arand) {
    var t1 = r23 * a;
    const a1 = floor(t1),
          a2 = a - t23 * a1;
    t1 = r23 * x;
    const x1 = floor(t1),
          x2 = x - t23 * x1;
    t1 = a1 * x2 + a2 * x1;
    const t2 = floor(r23 * t1),
          z  = t1 - t23 * t2,
          t3 = t23 * z + a2 * x2,
          t4 = floor(r46 * t3),
          x3 = t3 - t46 * t4;
    x = x3;

    return r46 * x3;
  }

  def getNext() {
    sharedCount += 1;
    return randlc(sharedCursor);
  }

  def skipToNth(in n: integral) {
    if (n <= 0) then
      halt("RandomStream.skipToNth() can only be called with positive values");
    initCursorVal();
    sharedCount = n;
    n -= 1;
    var t = arand;
    var retval = arand;
    while (n != 0) {
      const i = n / 2;
      if (2 * i != n) then
        randlc(sharedCursor, t);
      retval = randlc(t, t);
      n = i;
    }
  }

  // n is assumed to be 1..
  def getNth(n : integral) {
    if (n <= 0) then
      halt("RandomStream.getNth() can only be called with a positive value, received: ", n);
    skipToNth(n);
    return getNext();
  }

  def fillRandom(X: [] real) {
    lock$;
    const startCount: int(64) = sharedCount;
    sharedCount = startCount + X.numElements;
    lock$ = false;
    if (X.rank == 1) {
      forall (x, r) in (X, chpl__these(startCount, X.numElements)) {
        x = r;
      }
    } else {
      for (x,r) in (X, chpl__these(startCount, X.numElements)) {
        x = r;
      }
    }
  }

  def fillRandom(x: [] complex) {
    for i in x.domain {
      x(i).re = getNext();
      x(i).im = getNext();
    }
  }

  //
  // The following are internal-only iterators used by the fillRandom
  // routine.  We would like to support general leaders/followers on
  // RandomStreams, but the update to the shared cursor (sharedCount)
  // is not possible to do in a safe way with our current iterator
  // interface.  Future work should help with this.
  //
  def chpl__these(startAt: int(64), numElements) {
    skipToNth(startAt);
    for i in 1..numElements {
      const val = getNext();
      yield val;
    }
  }

  def chpl__these(param tag: iterator, startAt: int(64), numElements)
      where tag == iterator.leader {
    var D: domain(1, int(64)) = [1..numElements:int(64)];
    halt("Someone's trying to call the leader");
    yield D;
  }

  def chpl__these(param tag: iterator, follower, startAt: int(64), numElements)
      where tag == iterator.follower {
    // make a local copy of the 'this' random stream class
    var locStream = new RandomStream(seed);
    locStream.skipToNth(follower.low + startAt);
    for i in follower {
      if (debugParRandom) then
        writeln("Doing iteration ", format("#####", i+1), " on locale ", here.id);
      const val = locStream.getNext();
      yield val;
    }
    delete locStream;
  }

  /*  BLC: Would like to add something like this, but should
      really add a warning if the range is greater than max(int(64))
      or a real's precision something to deal with the possible
      accuracy issues...
  def fillRandom(x: [],
		 minval: x.eltType = min(x.eltType),
		 maxval: x.eltType = max(x.eltType))
               where _isIntegralType(x.eltType) {
    const numVals = maxval:real - minval:real + 1;
    for i in x.domain {
      x(i) = (minval + (getNext() * numVals)): x.eltType;
    }
  }
  */
}

def fillRandom(x:[], initseed: int(64) = SeedGenerator.clockMS) {
  use Random; // This is ridiculous, but only required for modules
              // that call Random.fillRandom without use-ing Random
              // like test/modules/standard/stonea/fillRandom.chpl
  var randNums = new RandomStream(initseed);

  randNums.fillRandom(x);
}

// This random number generator was written from the NPB
// random number generator.
//
// It's the linear congruential generator with
//
// m = 2^46 (the modulus)
// a = 1220703125.0 (the multiplier)
// c = 0 (the increment)
//
// It requires an odd integer for its seed.

use Time;

config param debugParRandom: bool = false;

class SeedGeneratorClass {
  def clockMS {
    return getCurrentTime(unit=TimeUnits.microseconds):int(64);
  }
}

var SeedGenerator = new SeedGeneratorClass();

class RandomStream {
  const seed:int(64) = SeedGenerator.clockMS;
  const arand = 1220703125.0;

  // These are meant to be private; internal to class; not set by user
  const r23   = 0.5**23,
        t23   = 2.0**23,
        r46   = 0.5**46,
        t46   = 2.0**46;
  const internalSeed = computeInternalSeed();

  var lock$: sync bool = false;
  var sharedCursor = internalSeed:real;
  var sharedCount: int(64) = 1;

  def computeInternalSeed() {
    var val: int(64) = seed;
    // ensure seed is odd
    if (val % 2 == 0) then val += 1;
    return val;
  }

  def initCursorVal() {
    sharedCursor = internalSeed:real;
    sharedCount = 1;
  }

  def randlc(inout x, a = arand) {
    var t1 = r23 * a;
    const a1 = floor(t1),
          a2 = a - t23 * a1;
    t1 = r23 * x;
    const x1 = floor(t1),
          x2 = x - t23 * x1;
    t1 = a1 * x2 + a2 * x1;
    const t2 = floor(r23 * t1),
          z  = t1 - t23 * t2,
          t3 = t23 * z + a2 * x2,
          t4 = floor(r46 * t3),
          x3 = t3 - t46 * t4;
    x = x3;

    return r46 * x3;
  }

  def getNext() {
    sharedCount += 1;
    return randlc(sharedCursor);
  }

  def skipToNth(in n: integral) {
    if (n <= 0) then
      halt("RandomStream.skipToNth() can only be called with positive values");
    initCursorVal();
    sharedCount = n;
    n -= 1;
    var t = arand;
    var retval = arand;
    while (n != 0) {
      const i = n / 2;
      if (2 * i != n) then
        randlc(sharedCursor, t);
      retval = randlc(t, t);
      n = i;
    }
  }

  // n is assumed to be 1..
  def getNth(n : integral) {
    if (n <= 0) then
      halt("RandomStream.getNth() can only be called with a positive value, received: ", n);
    skipToNth(n);
    return getNext();
  }

  def fillRandom(X: [] real) {
    lock$;
    const startCount: int(64) = sharedCount;
    sharedCount = startCount + X.numElements;
    lock$ = false;
    forall (x, r) in (X, chpl__these(startCount, X.numElements)) {
      x = r;
    }
  }

  def fillRandom(x: [] complex) {
    for i in x.domain {
      x(i).re = getNext();
      x(i).im = getNext();
    }
  }

  //
  // The following are internal-only iterators used by the fillRandom
  // routine.  We would like to support general leaders/followers on
  // RandomStreams, but the update to the shared cursor (sharedCount)
  // is not possible to do in a safe way with our current iterator
  // interface.  Future work should help with this.
  //
  def chpl__these(startAt: int(64), numElements) {
    skipToNth(startAt);
    for i in 1..numElements {
      const val = getNext();
      yield val;
    }
  }

  def chpl__these(param tag: iterator, startAt: int(64), numElements)
      where tag == iterator.leader {
    var D: domain(1, int(64)) = [1..numElements:int(64)];
    halt("Someone's trying to call the leader");
    yield D;
  }

  def chpl__these(param tag: iterator, follower, startAt: int(64), numElements)
      where tag == iterator.follower {
    // make a local copy of the 'this' random stream class
    var locStream = new RandomStream(seed);
    locStream.skipToNth(follower.low + startAt);
    for i in follower {
      if (debugParRandom) then
        writeln("Doing iteration ", format("#####", i+1), " on locale ", here.id);
      const val = locStream.getNext();
      yield val;
    }
  }

  /*  BLC: Would like to add something like this, but should
      really add a warning if the range is greater than max(int(64))
      or a real's precision something to deal with the possible
      accuracy issues...
  def fillRandom(x: [],
		 minval: x.eltType = min(x.eltType),
		 maxval: x.eltType = max(x.eltType))
               where _isIntegralType(x.eltType) {
    const numVals = maxval:real - minval:real + 1;
    for i in x.domain {
      x(i) = (minval + (getNext() * numVals)): x.eltType;
    }
  }
  */
}

def fillRandom(x:[], initseed: int(64) = SeedGenerator.clockMS) {
  use Random; // This is ridiculous, but only required for modules
              // that call Random.fillRandom without use-ing Random
              // like test/modules/standard/stonea/fillRandom.chpl
  var randNums = new RandomStream(initseed);

  randNums.fillRandom(x);
}


Set up svn:ignore on a directory

Set up svn:ignore on a directory

Set the svn:ignore property on this directory like it is on many other test
directories.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13934 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

def foo() { }

def bar() { }

var done: single bool;

def timeout(n: uint) {
  use Time;
  begin { sleep(n); writeln("Timeout"); exit(1); }
  begin { done; exit(0); }
}

timeout(30); // exit after 30 seconds or when done is set.

for i in 1..100000 {
  cobegin {
    cobegin { bar(); foo(); }
    cobegin { bar(); foo(); }
    cobegin { bar(); foo(); }
    cobegin { bar(); foo(); }
  }
  if i % 10000 == 0 then
    writeln("iteration ", i, " done.");
}

done = true;

def foo() { }

def bar() { }

var done: single bool;

def timeout(n: uint) {
  use Time;
  begin { sleep(n); exit(1); }
  begin { done; exit(0); }
}

timeout(30); // exit after 30 seconds or when done is set.

for i in 1..100000 {
  cobegin {
    cobegin { bar(); foo(); }
    cobegin { bar(); foo(); }
    cobegin { bar(); foo(); }
    cobegin { bar(); foo(); }
  }
  if i % 10000 == 0 then
    writeln("iteration ", i, " done.");
}

done = true;


Fixes compilation problem with --baseline

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@19513 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

module BufferInternals {
  use SysBasic;
  use Error;

  extern type qbytes_ptr_t;
  extern type qbuffer_ptr_t;
  extern type qbuffer_iter_t;
  extern const QBYTES_PTR_NULL:qbytes_ptr_t;
  extern const QBUFFER_PTR_NULL:qbuffer_ptr_t;

  extern proc qbytes_retain(qb:qbytes_ptr_t);
  extern proc qbytes_release(qb:qbytes_ptr_t);
  extern proc qbytes_len(qb:qbytes_ptr_t):int(64);
  extern proc qbytes_data(qb:qbytes_ptr_t):c_ptr;

  extern proc qbytes_create_iobuf(inout ret:qbytes_ptr_t):err_t;
  extern proc qbytes_create_calloc(inout ret:qbytes_ptr_t, len:int(64)):err_t;

  extern proc qbuffer_iter_null():qbuffer_iter_t;

  /*
  extern proc qbuffer_init(inout buf:qbuffer_t):err_t;
  extern proc qbuffer_destroy(inout buf:qbuffer_t);
  */
  extern proc qbuffer_create(inout buf:qbuffer_ptr_t):err_t;
  extern proc qbuffer_retain(buf:qbuffer_ptr_t);
  extern proc qbuffer_release(buf:qbuffer_ptr_t);

  extern proc qbuffer_append(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):err_t;
  extern proc qbuffer_append_buffer(buf:qbuffer_ptr_t, src:qbuffer_ptr_t, src_start:qbuffer_iter_t, src_end:qbuffer_iter_t):err_t;
  extern proc qbuffer_prepend(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):err_t;
  extern proc qbuffer_flatten(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout bytes_out):err_t;
  //extern proc qbuffer_clone(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout buf_out:qbuffer_t):err_t;
  extern proc qbuffer_copyout(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):err_t;
  extern proc qbuffer_copyin(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):err_t;

  extern proc qbuffer_begin(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_end(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_iter_next_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_prev_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_advance(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t, amt:int(64));

  extern proc qbuffer_iter_get(it: qbuffer_iter_t, end:qbuffer_iter_t,
                                inout bytes_out:qbytes_ptr_t,
                                inout skip_out:int(64),
                                inout len_out:int(64));
  extern proc qbuffer_iter_num_bytes(start:qbuffer_iter_t, end:qbuffer_iter_t):int(64);

  extern proc debug_print_qbuffer_iter(inout it:qbuffer_iter_t);

  extern proc qbuffer_start_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_end_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_reposition(buf:qbuffer_ptr_t, new_offset_start:int(64));

  extern proc bulk_get_bytes(src_locale:int, src_addr:qbytes_ptr_t):qbytes_ptr_t;

  extern proc bulk_put_buffer(dst_locale:int, dst_addr:c_ptr, dst_len:int(64), buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t):err_t;

}

module Buffers {
  use BufferInternals; // TODO -- non-exporting use

  // Now define the Chapel types using the originals..
  record bytes {
    var home_uid:int;
    var _bytes_internal:qbytes_ptr_t;
  }

  record buffer_iterator {
    var home_uid:int;
    var _bufit_internal:qbuffer_iter_t;
  }
  record buffer_range {
    var start:buffer_iterator;
    var end:buffer_iterator;
  }

  proc buffer_range.len:int(64) {
    var ret:int(64);
    on __primitive("chpl_on_locale_num", this.start.home_uid) {
      ret = qbuffer_iter_num_bytes(this.start._bufit_internal,
                                   this.end._bufit_internal);
    }
    return ret;
  }

  // FUTURE -- we could create const buffers
  // with a boolean type param
  record buffer {
    var home_uid:int;
    var _buf_internal:qbuffer_ptr_t;
  }

  // bytes methods.
  proc bytes.bytes() {
    var tmp:int;
    this._internal = c_nil;
    this.home_uid = __primitive("_get_locale", tmp);
  }
  proc bytes.bytes(len:int(64), inout error:err_t) {
    var tmp:int;
    error = qbytes_create_calloc(this._bytes_internal, len);
    this.home_uid = __primitive("_get_locale", tmp);
  }
  proc bytes.bytes(len:int(64)) {
    var tmp:int;
    var error:err_t = ENOERR;
    error = qbytes_create_calloc(this._bytes_internal, len);
    if error then ioerror(error, "in bytes constructor");
    this.home_uid = __primitive("_get_locale", tmp);
  }

  proc create_iobuf(inout error:err_t):bytes {
    var ret:bytes;
    var tmp:int;
    error = qbytes_create_iobuf(ret._bytes_internal);
    ret.home_uid = __primitive("_get_locale", tmp);
    return ret;
  }
  proc create_iobuf():bytes {
    var err:err_t = ENOERR;
    var ret = create_iobuf(err);
    if err then ioerror(err, "in create_iobuf");
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: bytes) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    if x.home_uid == here_uid {
      qbytes_retain(x._bytes_internal);
      return x;
    } else {
      var ret:bytes;
      ret.home_uid = here_uid;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home_uid, x._bytes_internal);
      return ret;
    }
  }

  proc =(ret:bytes, x:bytes) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);

    // retain -- release
    if( x.home_uid == here_uid ) {
      on __primitive("chpl_on_locale_num", x.home_uid) {
        qio_bytes_retain(x._bytes_internal);
      }

      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qio_bytes_release(ret._bytes_internal);
      }

      ret.home = x.home;
      ret._bytes_internal = x._bytes_internal;
    } else {
      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qio_bytes_release(ret._bytes_internal);
      }
      ret.home = here;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home_uid, x._bytes_internal);
    }
    return ret;
  }

  proc bytes.~bytes() {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      qbytes_release(this._bytes_internal);
      this._bytes_internal = QBYTES_PTR_NULL;
    }
  }
  proc bytes.len:int(64) {
    var ret:int(64);
    on __primitive("chpl_on_locale_num", this.home_uid) {
     ret = qbytes_len(this._bytes_internal);
    }
    return ret;
  }

  // buffer iterator
  proc buffer_iterator.buffer_iterator() {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    this.home_uid = here_uid;
    this._bufit_internal = qbuffer_iter_null();
  }
  /* don't believe this is necessary...
  proc chpl__initCopy(x: buffer_iterator) {
    return x;
  }
  */

  // buffer methods.
  proc buffer.buffer(inout error:err_t) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    this.home_uid = here_uid;
    error = qbuffer_create(this._buf_internal);
  }
  proc buffer.buffer() {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    var error:err_t = ENOERR;
    this.home_uid = here_uid;
    error = qbuffer_create(this._buf_internal);
    if error then ioerror(error, "in buffer constructor");
  }

  // buffer.flatten is special in that it creates a
  // bytes object wherever it is called (and communicates it
  // intelligently). It is use in buffer's initCopy.
  proc buffer.flatten(range:buffer_range, inout error:err_t):bytes {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    var ret:bytes;
    if this.home_uid == here_uid {
      error = qbuffer_flatten(this._buf_internal, range.start._bufit_internal, range.end._bufit_internal, ret);
    } else {
      var dst_locale = here;
      var dst_len:int(64) = range.len;
      ret = new bytes(dst_len);
      var dst_addr = qbytes_data(ret._bytes_internal);
      on __primitive("chpl_on_locale_num", this.home_uid) {
        // Copy the buffer to the bytes...
        error = bulk_put_buffer(dst_locale.uid, dst_addr, dst_len,
                                this._buf_internal,
                                range.start._bufit_internal,
                                range.end._bufit_internal);
      }
    }
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: buffer) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    if x.home_uid == here_uid {
      qbuffer_retain(x._buf_internal);
      return x;
    } else {
      var ret:buffer;
      var start_offset:int(64);
      var end_offset:int(64);

      on __primitive("chpl_on_locale_num", x.home_uid) {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here_uid;

      on __primitive("chpl_on_locale_num", x.home_uid) {
        var err:err_t = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer init copy");
      }

      ret.append(b);
      return ret;
    }
  }

  proc =(ret:buffer, x:buffer) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    ret.home_uid = here_uid;
    // retain -- release
    if( x.home_uid == ret.home_uid ) {
      on __primitive("chpl_on_locale_num", x.home_uid) {
        qbuffer_retain(x._buf_internal);
      }

      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qbuffer_release(ret._buf_internal);
      }

      ret.home = x.home;
      ret._buf_internal = x._buf_internal;
    } else {
      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qbuffer_release(ret._buf_internal);
      }

      var start_offset:int(64);
      var end_offset:int(64);

      on __primitive("chpl_on_locale_num", x.home_uid) {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here_uid;

      on __primitive("chpl_on_locale_num", x.home_uid) {
        var err:err_t = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer assignment");
      }

      ret.append(b);
    }
  }

  proc buffer.~buffer() {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      qbuffer_release(this._buf_internal);
      this._buf_internal = QBUFFER_PTR_NULL;
    }
  }

  proc buffer.len {
    var ret:int(64);
    on __primitive("chpl_on_locale_num", this.home_uid) {
      ret = qbuffer_len(this._buf_internal);
    }
    return ret;
  }

  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, inout error:err_t) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      error = qbuffer_append(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:err_t = ENOERR;
    this.append(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.append(buf:buffer, part:buffer_range = buf.all(), inout error:err_t) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      error = qbuffer_append_buffer(this._buf_internal, buf._buf_internal, part.start._bufit_internal, part.end._bufit_internal);
    }
  }
  proc buffer.append(buf:buffer, part:buffer_range = buf.all()) {
    var err:err_t = ENOERR;
    this.append(buf, part, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, inout error:err_t) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      qbuffer_prepend(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:err_t = ENOERR;
    this.prepend(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.prepend");
  }

  proc buffer.start():buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      ret._bufit_internal = qbuffer_begin(this._buf_internal);
    }
    return ret;
  }
  proc buffer.end():buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      ret._bufit_internal = qbuffer_end(this._buf_internal);
    }
    return ret;
  }
  proc buffer.all():buffer_range {
    return new buffer_range(this.start(), this.end());
  }

  proc buffer.next_part(inout it:buffer_iterator) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var x = it;
      qbuffer_iter_next_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.prev_part(inout it:buffer_iterator) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var x = it;
      qbuffer_iter_prev_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.advance(inout it:buffer_iterator, amount:int(64)) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var x = it;
      qbuffer_iter_advance(this._buf_internal, x._bufit_internal, amount);
      it = x;
    }
  }

  proc buffer_iterator.debug_print()
  {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      debug_print_qbuffer_iter(this._bufit_internal);
    }
  }

  /*
  proc buffer.clone(range:buffer_range, err:ErrorHandler = nil):buffer {
    on this.home {
      var ret:buffer;
      seterr(qbuffer_clone(this._buf_internal, range.start, range.end, ret._buf_internal));
      return ret;
    }
  }*/

  // methods to read/write basic types.
  proc buffer.copyout(it:buffer_iterator, out value, inout error:err_t):buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var end:buffer_iterator = it;
      var tmp:value.type;
      var sz = numBytes(value.type);
      this.advance(end, sz);
      error = qbuffer_copyout(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      value = tmp;
      ret = end;
    }
    return ret;
  }
  proc buffer.copyout(it:buffer_iterator, out value):buffer_iterator {
    var err:err_t = ENOERR;
    this.copyout(it, value, err);
    if err then ioerror(err, "in buffer.copyout");
  }

  proc buffer.copyin( it:buffer_iterator, value, inout error:err_t):buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      //writeln("iterator on way in");
      //debug_print_qbuffer_iter(it._bufit_internal);
      var end = it;
      var tmp = value;
      var sz = numBytes(value.type);
      //writeln("iterator copied to end");
      //debug_print_qbuffer_iter(end._bufit_internal);

      //writeln("sz is ", sz);
      this.advance(end, sz);
      //debug_print_qbuffer_iter(end._bufit_internal);
      error = qbuffer_copyin(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      ret = end;
    }
    return ret;
  }
  proc buffer.copyin( it:buffer_iterator, value):buffer_iterator {
    var err:err_t = ENOERR;
    this.copyin(it, value, err);
    if err then ioerror(err, "in buffer.copyin");
  }
}

module BufferInternals {
  use SysBasic;
  use Error;

  extern type qbytes_ptr_t;
  extern type qbuffer_ptr_t;
  extern type qbuffer_iter_t;
  extern const QBYTES_PTR_NULL:qbytes_ptr_t;
  extern const QBUFFER_PTR_NULL:qbuffer_ptr_t;

  extern proc qbytes_retain(qb:qbytes_ptr_t);
  extern proc qbytes_release(qb:qbytes_ptr_t):err_t;
  extern proc qbytes_len(qb:qbytes_ptr_t):int(64);
  extern proc qbytes_data(qb:qbytes_ptr_t):c_ptr;

  extern proc qbytes_create_iobuf(inout ret:qbytes_ptr_t):err_t;
  extern proc qbytes_create_calloc(inout ret:qbytes_ptr_t, len:int(64)):err_t;

  extern proc qbuffer_iter_null():qbuffer_iter_t;

  /*
  extern proc qbuffer_init(inout buf:qbuffer_t):err_t;
  extern proc qbuffer_destroy(inout buf:qbuffer_t);
  */
  extern proc qbuffer_create(inout buf:qbuffer_ptr_t):err_t;
  extern proc qbuffer_retain(buf:qbuffer_ptr_t);
  extern proc qbuffer_release(buf:qbuffer_ptr_t):err_t;

  extern proc qbuffer_append(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):err_t;
  extern proc qbuffer_append_buffer(buf:qbuffer_ptr_t, src:qbuffer_ptr_t, src_start:qbuffer_iter_t, src_end:qbuffer_iter_t):err_t;
  extern proc qbuffer_prepend(buf:qbuffer_ptr_t, bytes:qbytes_ptr_t, skip_bytes:int(64), len_bytes:int(64)):err_t;
  extern proc qbuffer_flatten(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout bytes_out):err_t;
  //extern proc qbuffer_clone(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout buf_out:qbuffer_t):err_t;
  extern proc qbuffer_copyout(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):err_t;
  extern proc qbuffer_copyin(buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t, inout x, size):err_t;

  extern proc qbuffer_begin(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_end(buf:qbuffer_ptr_t):qbuffer_iter_t;
  extern proc qbuffer_iter_next_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_prev_part(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t);
  extern proc qbuffer_iter_advance(buf:qbuffer_ptr_t, inout it:qbuffer_iter_t, amt:int(64));

  extern proc qbuffer_iter_get(it: qbuffer_iter_t, end:qbuffer_iter_t,
                                inout bytes_out:qbytes_ptr_t,
                                inout skip_out:int(64),
                                inout len_out:int(64));
  extern proc qbuffer_iter_num_bytes(start:qbuffer_iter_t, end:qbuffer_iter_t):int(64);

  extern proc debug_print_qbuffer_iter(inout it:qbuffer_iter_t);

  extern proc qbuffer_start_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_end_offset(buf:qbuffer_ptr_t):int(64);
  extern proc qbuffer_reposition(buf:qbuffer_ptr_t, new_offset_start:int(64));

  extern proc bulk_get_bytes(src_locale:int, src_addr:qbytes_ptr_t):qbytes_ptr_t;

  extern proc bulk_put_buffer(dst_locale:int, dst_addr:c_ptr, dst_len:int(64), buf:qbuffer_ptr_t, start:qbuffer_iter_t, end:qbuffer_iter_t):err_t;

}

module Buffers {
  use BufferInternals; // TODO -- non-exporting use

  // Now define the Chapel types using the originals..
  record bytes {
    var home_uid:int;
    var _bytes_internal:qbytes_ptr_t;
  }

  record buffer_iterator {
    var home_uid:int;
    var _bufit_internal:qbuffer_iter_t;
  }
  record buffer_range {
    var start:buffer_iterator;
    var end:buffer_iterator;
  }

  proc buffer_range.len:int(64) {
    var ret:int(64);
    on __primitive("chpl_on_locale_num", this.start.home_uid) {
      ret = qbuffer_iter_num_bytes(this.start._bufit_internal,
                                   this.end._bufit_internal);
    }
    return ret;
  }

  // FUTURE -- we could create const buffers
  // with a boolean type param
  record buffer {
    var home_uid:int;
    var _buf_internal:qbuffer_ptr_t;
  }

  // bytes methods.
  proc bytes.bytes() {
    var tmp:int;
    this._internal = c_nil;
    this.home_uid = __primitive("_get_locale", tmp);
  }
  proc bytes.bytes(len:int(64), inout error:err_t) {
    var tmp:int;
    error = qbytes_create_calloc(this._bytes_internal, len);
    this.home_uid = __primitive("_get_locale", tmp);
  }
  proc bytes.bytes(len:int(64)) {
    var tmp:int;
    var error:err_t = ENOERR;
    error = qbytes_create_calloc(this._bytes_internal, len);
    if error then ioerror(error, "in bytes constructor");
    this.home_uid = __primitive("_get_locale", tmp);
  }

  proc create_iobuf(inout error:err_t):bytes {
    var ret:bytes;
    var tmp:int;
    error = qbytes_create_iobuf(ret._bytes_internal);
    ret.home_uid = __primitive("_get_locale", tmp);
    return ret;
  }
  proc create_iobuf():bytes {
    var err:err_t = ENOERR;
    var ret = create_iobuf(err);
    if err then ioerror(err, "in create_iobuf");
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: bytes) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    if x.home_uid == here_uid {
      qbytes_retain(x._bytes_internal);
      return x;
    } else {
      var ret:bytes;
      ret.home_uid = here_uid;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home_uid, x._bytes_internal);
      return ret;
    }
  }

  proc =(ret:bytes, x:bytes) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);

    // retain -- release
    if( x.home_uid == here_uid ) {
      on __primitive("chpl_on_locale_num", x.home_uid) {
        qio_bytes_retain(x._bytes_internal);
      }

      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qio_bytes_release(ret._bytes_internal);
      }

      ret.home = x.home;
      ret._bytes_internal = x._bytes_internal;
    } else {
      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qio_bytes_release(ret._bytes_internal);
      }
      ret.home = here;
      writeln("Bulk moving bytes");
      ret._bytes_internal = bulk_get_bytes(x.home_uid, x._bytes_internal);
    }
    return ret;
  }

  proc bytes.~bytes() {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      qbytes_release(this._bytes_internal);
      this._bytes_internal = QBYTES_PTR_NULL;
    }
  }
  proc bytes.len:int(64) {
    var ret:int(64);
    on __primitive("chpl_on_locale_num", this.home_uid) {
     ret = qbytes_len(this._bytes_internal);
    }
    return ret;
  }

  // buffer iterator
  proc buffer_iterator.buffer_iterator() {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    this.home_uid = here_uid;
    this._bufit_internal = qbuffer_iter_null();
  }
  /* don't believe this is necessary...
  proc chpl__initCopy(x: buffer_iterator) {
    return x;
  }
  */

  // buffer methods.
  proc buffer.buffer(inout error:err_t) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    this.home_uid = here_uid;
    error = qbuffer_create(this._buf_internal);
  }
  proc buffer.buffer() {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    var error:err_t = ENOERR;
    this.home_uid = here_uid;
    error = qbuffer_create(this._buf_internal);
    if error then ioerror(error, "in buffer constructor");
  }

  // buffer.flatten is special in that it creates a
  // bytes object wherever it is called (and communicates it
  // intelligently). It is use in buffer's initCopy.
  proc buffer.flatten(range:buffer_range, inout error:err_t):bytes {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    var ret:bytes;
    if this.home_uid == here_uid {
      error = qbuffer_flatten(this._buf_internal, range.start._bufit_internal, range.end._bufit_internal, ret);
    } else {
      var dst_locale = here;
      var dst_len:int(64) = range.len;
      ret = new bytes(dst_len);
      var dst_addr = qbytes_data(ret._bytes_internal);
      on __primitive("chpl_on_locale_num", this.home_uid) {
        // Copy the buffer to the bytes...
        error = bulk_put_buffer(dst_locale.uid, dst_addr, dst_len,
                                this._buf_internal,
                                range.start._bufit_internal,
                                range.end._bufit_internal);
      }
    }
    return ret;
  }

  // TODO -- shouldn't have to write this this way!
  proc chpl__initCopy(x: buffer) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    if x.home_uid == here_uid {
      qbuffer_retain(x._buf_internal);
      return x;
    } else {
      var ret:buffer;
      var start_offset:int(64);
      var end_offset:int(64);

      on __primitive("chpl_on_locale_num", x.home_uid) {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here_uid;

      on __primitive("chpl_on_locale_num", x.home_uid) {
        var err:err_t = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer init copy");
      }

      ret.append(b);
      return ret;
    }
  }

  proc =(ret:buffer, x:buffer) {
    var tmp:int;
    var here_uid = __primitive("_get_locale", tmp);
    ret.home_uid = here_uid;
    // retain -- release
    if( x.home_uid == ret.home_uid ) {
      on __primitive("chpl_on_locale_num", x.home_uid) {
        qbuffer_retain(x._buf_internal);
      }

      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qbuffer_release(ret._buf_internal);
      }

      ret.home = x.home;
      ret._buf_internal = x._buf_internal;
    } else {
      on __primitive("chpl_on_locale_num", ret.home_uid) {
        qbuffer_release(ret._buf_internal);
      }

      var start_offset:int(64);
      var end_offset:int(64);

      on __primitive("chpl_on_locale_num", x.home_uid) {
        start_offset = qbuffer_start_offset(x._buf_internal);
        end_offset = qbuffer_end_offset(x._buf_internal);
      }

      var b = new bytes(end_offset - start_offset);
      var len = qbytes_len(b._bytes_internal);
      var ptr = qbytes_data(b._bytes_internal);

      // Now create a local buffer with the right data
      // in it starting at the right position.
      qbuffer_reposition(ret._buf_internal, start_offset);

      var there_uid = here_uid;

      on __primitive("chpl_on_locale_num", x.home_uid) {
        var err:err_t = ENOERR;
        err = bulk_put_buffer(there_uid, ptr, len, x._buf_internal,
                              qbuffer_begin(x._buf_internal),
                              qbuffer_end(x._buf_internal));
        if err then ioerror(err, "in buffer assignment");
      }

      ret.append(b);
    }
  }

  proc buffer.~buffer() {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      qbuffer_release(this._buf_internal);
      this._buf_internal = QBUFFER_PTR_NULL;
    }
  }

  proc buffer.len {
    var ret:int(64);
    on __primitive("chpl_on_locale_num", this.home_uid) {
      ret = qbuffer_len(this._buf_internal);
    }
    return ret;
  }

  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, inout error:err_t) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      error = qbuffer_append(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.append(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:err_t = ENOERR;
    this.append(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.append(buf:buffer, part:buffer_range = buf.all(), inout error:err_t) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      error = qbuffer_append_buffer(this._buf_internal, buf._buf_internal, part.start._bufit_internal, part.end._bufit_internal);
    }
  }
  proc buffer.append(buf:buffer, part:buffer_range = buf.all()) {
    var err:err_t = ENOERR;
    this.append(buf, part, err);
    if err then ioerror(err, "in buffer.append");
  }

  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len, inout error:err_t) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      qbuffer_prepend(this._buf_internal, b._bytes_internal, skip_bytes, len_bytes);
    }
  }
  proc buffer.prepend(b:bytes, skip_bytes:int(64) = 0, len_bytes:int(64) = b.len) {
    var err:err_t = ENOERR;
    this.prepend(b, skip_bytes, len_bytes, err);
    if err then ioerror(err, "in buffer.prepend");
  }

  proc buffer.start():buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      ret._bufit_internal = qbuffer_begin(this._buf_internal);
    }
    return ret;
  }
  proc buffer.end():buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      ret._bufit_internal = qbuffer_end(this._buf_internal);
    }
    return ret;
  }
  proc buffer.all():buffer_range {
    return new buffer_range(this.start(), this.end());
  }

  proc buffer.next_part(inout it:buffer_iterator) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var x = it;
      qbuffer_iter_next_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.prev_part(inout it:buffer_iterator) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var x = it;
      qbuffer_iter_prev_part(this._buf_internal, x._bufit_internal);
      it = x;
    }
  }
  proc buffer.advance(inout it:buffer_iterator, amount:int(64)) {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var x = it;
      qbuffer_iter_advance(this._buf_internal, x._bufit_internal, amount);
      it = x;
    }
  }

  proc buffer_iterator.debug_print()
  {
    on __primitive("chpl_on_locale_num", this.home_uid) {
      debug_print_qbuffer_iter(this._bufit_internal);
    }
  }

  /*
  proc buffer.clone(range:buffer_range, err:ErrorHandler = nil):buffer {
    on this.home {
      var ret:buffer;
      seterr(qbuffer_clone(this._buf_internal, range.start, range.end, ret._buf_internal));
      return ret;
    }
  }*/

  // methods to read/write basic types.
  proc buffer.copyout(it:buffer_iterator, out value, inout error:err_t):buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      var end:buffer_iterator = it;
      var tmp:value.type;
      var sz = numBytes(value.type);
      this.advance(end, sz);
      error = qbuffer_copyout(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      value = tmp;
      ret = end;
    }
    return ret;
  }
  proc buffer.copyout(it:buffer_iterator, out value):buffer_iterator {
    var err:err_t = ENOERR;
    this.copyout(it, value, err);
    if err then ioerror(err, "in buffer.copyout");
  }

  proc buffer.copyin( it:buffer_iterator, value, inout error:err_t):buffer_iterator {
    var ret:buffer_iterator;
    ret.home_uid = this.home_uid;
    on __primitive("chpl_on_locale_num", this.home_uid) {
      //writeln("iterator on way in");
      //debug_print_qbuffer_iter(it._bufit_internal);
      var end = it;
      var tmp = value;
      var sz = numBytes(value.type);
      //writeln("iterator copied to end");
      //debug_print_qbuffer_iter(end._bufit_internal);

      //writeln("sz is ", sz);
      this.advance(end, sz);
      //debug_print_qbuffer_iter(end._bufit_internal);
      error = qbuffer_copyin(this._buf_internal, it._bufit_internal, end._bufit_internal, tmp, sz);
      ret = end;
    }
    return ret;
  }
  proc buffer.copyin( it:buffer_iterator, value):buffer_iterator {
    var err:err_t = ENOERR;
    this.copyin(it, value, err);
    if err then ioerror(err, "in buffer.copyin");
  }
}


Improve chpldoc Sys module

Improve chpldoc Sys module

I didn't go to the extreme here, but wrote a decent introduction
and cleaned up the chpldoc output (which now mostly shows all of
these constants and functions without comments).

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
   This module is for low-level programming. It provides Chapel versions of
   many POSIX/Linux C library or system calls. For documentation on these
   system calls and constants, please see your system's manual pages.

   Each of the functions in this file provides the same functionality
   as the corresponding function without the ``sys_`` prefix, except that
   the ``sys_`` versions all return an error code (of type :type:`~SysBasic.err_t`)
   and return any other values (such as how much was read) through an out
   argument.

   For example, you can find more about the ``mmap`` call with:

   .. code-block:: sh

     man mmap

   The call available here, :proc:`sys_mmap`, always returns an error
   code (or 0 for no error). The pointer value normally returned by ``mmap``
   will be returned through the final ``ret_out`` argument.

 */
module Sys {
  // get basic types from SysBasic
  use SysBasic;

  // CONSTANTS

  // note that if a constant is not defined in the C environment,
  // the compile will fail when it is used (and not here!)
  // That means that we can just list all of the constants we know of,
  // for different OSs, and if one is missing errors will occur in
  // C code generation.
  // mferguson -- TODO -- is that still true? Even for LLVM?

  // basic file flags
  extern const O_RDONLY:c_int;
  extern const O_WRONLY:c_int;
  extern const O_RDWR:c_int;

  // POSIX flags
  extern const O_APPEND:c_int;
  extern const O_CREAT:c_int;
  extern const O_DSYNC:c_int;
  extern const O_EXCL:c_int;
  extern const O_NOCTTY:c_int;
  extern const O_NONBLOCK:c_int;
  extern const O_RSYNC:c_int;
  extern const O_SYNC:c_int;
  extern const O_TRUNC:c_int;

  // Linux flags (make them 0 if not on linux, please)
  //extern const O_ASYNC:c_int;
  //extern const O_CLOEXEC:c_int;
  //extern const O_DIRECT:c_int;
  //extern const O_DIRECTORY:c_int;
  //extern const O_LARGEFILE:c_int;
  //extern const O_NOATIME:c_int;
  //extern const O_NOFOLLOW:c_int;
  //extern const O_NDELAY:c_int;

  // open permission flags.
  // These are too complicated for the LLVM backend!
  //extern const S_IRWXU:c_int;
  //extern const S_IRUSR:c_int;
  //extern const S_IWUSR:c_int;
  //extern const S_IXUSR:c_int;
  //extern const S_IRWXG:c_int;
  //extern const S_IRGRP:c_int;
  //extern const S_IWGRP:c_int;
  //extern const S_IXGRP:c_int;
  //extern const S_IRWXO:c_int;
  //extern const S_IROTH:c_int;
  //extern const S_IWOTH:c_int;
  //extern const S_IXOTH:c_int;

  // socket domains
  extern const AF_UNIX:c_int;
  extern const AF_LOCAL:c_int;
  extern const AF_INET:c_int;
  extern const AF_INET6:c_int;
  extern const AF_IPX:c_int;
  extern const AF_NETLINK:c_int;
  extern const AF_X25:c_int;
  extern const AF_AX25:c_int;
  extern const AF_ATMPVC:c_int;
  extern const AF_APPLETALK:c_int;
  extern const AF_PACKET:c_int;

  // socket types
  extern const SOCK_STREAM:c_int;
  extern const SOCK_DGRAM:c_int;
  extern const SOCK_SEQPACKET:c_int;
  extern const SOCK_RAW:c_int;
  extern const SOCK_RDM:c_int;

  extern const SOCK_NONBLOCK:c_int;
  extern const SOCK_CLOEXEC:c_int;

  // sendmsg flags.
  extern const MSG_CONFIRM:c_int;
  extern const MSG_DONTROUTE:c_int;
  extern const MSG_DONTWAIT:c_int;
  extern const MSG_EOR:c_int;
  extern const MSG_MORE:c_int;
  extern const MSG_NOSIGNAL:c_int;
  extern const MSG_OOB:c_int;

  // recvmsg flags
  extern const MSG_CMSG_CLOEXEC:c_int;
  extern const MSG_ERRQUEUE:c_int;
  extern const MSG_PEEK:c_int;
  extern const MSG_TRUNC:c_int;
  extern const MSG_WAITALL:c_int;

  //extern const MSG_EOR:c_int; sendmsg flag
  extern const MSG_CTRUNC:c_int;

  // shutdown how.
  extern const SHUT_RD:c_int;
  extern const SHUT_WR:c_int;
  extern const SHUT_RDWR:c_int;

  // socket option 'levels'
  extern const IPPROTO_IP:c_int;
  extern const IPPROTO_IPV6:c_int;
  extern const IPPROTO_TCP:c_int;
  extern const IPPROTO_UDP:c_int;

  // IP socket options
  extern const IP_ADD_MEMBERSHIP:c_int;
  extern const IP_DROP_MEMBERSHIP:c_int;
  extern const IP_HDRINCL:c_int;
  extern const IP_MTU:c_int;
  extern const IP_MTU_DISCOVER:c_int;
  extern const IP_MULTICAST_IF:c_int;
  extern const IP_MULTICAST_LOOP:c_int;
  extern const IP_MULTICAST_TTL:c_int;
  extern const IP_OPTIONS:c_int;
  extern const IP_PKTINFO:c_int;
  extern const IP_RECVERR:c_int;
  extern const IP_RECVOPTS:c_int;
  extern const IP_RECVTOS:c_int;
  extern const IP_RECVTTL:c_int;
  extern const IP_RETOPTS:c_int;
  extern const IP_ROUTER_ALERT:c_int;
  extern const IP_TOS:c_int;
  extern const IP_TTL:c_int;

  // IP 6 socket options
  extern const IPV6_ADDRFORM:c_int;
  extern const IPV6_ADD_MEMBERSHIP:c_int;
  extern const IPV6_DROP_MEMBERSHIP:c_int;
  extern const IPV6_MTU:c_int;
  extern const IPV6_MTU_DISCOVER:c_int;
  extern const IPV6_MULTICAST_HOPS:c_int;
  extern const IPV6_MULTICAST_IF:c_int;
  extern const IPV6_MULTICAST_LOOP:c_int;
  extern const IPV6_PKTINFO:c_int;
  extern const IPV6_RTHDR:c_int;
  extern const IPV6_AUTHHDR:c_int;
  extern const IPV6_DSTOPTS:c_int;
  extern const IPV6_HOPOPTS:c_int;
  //extern const IPV6_FLOWINFO:c_int;
  extern const IPV6_HOPLIMIT:c_int;
  extern const IPV6_RECVERR:c_int;
  extern const IPV6_ROUTER_ALERT:c_int;
  extern const IPV6_UNICAST_HOPS:c_int;
  extern const IPV6_V6ONLY:c_int;

  // TCP socket options
  extern const TCP_CORK:c_int;
  extern const TCP_DEFER_ACCEPT:c_int;
  extern const TCP_INFO:c_int;
  extern const TCP_KEEPCNT:c_int;
  extern const TCP_KEEPIDLE:c_int;
  extern const TCP_KEEPINTVL:c_int;
  extern const TCP_LINGER2:c_int;
  extern const TCP_MAXSEG:c_int;
  extern const TCP_NODELAY:c_int;
  extern const TCP_QUICKACK:c_int;
  extern const TCP_SYNCNT:c_int;
  extern const TCP_WINDOW_CLAMP:c_int;

  // UDP socket options
  //extern const UDP_CORK:c_int;

  /* SOCKET STRUCTURE TYPES */

  extern type sys_sockaddr_storage_t;
  extern record sys_sockaddr_t {
    var addr:sys_sockaddr_storage_t;
    var len:socklen_t;
    proc sys_sockaddr_t() {
      sys_init_sys_sockaddr_t(this);
    }
  }

  extern record sys_addrinfo_t {
    var ai_flags: c_int;
    var ai_family: c_int;
    var ai_socktype: c_int;
    var ai_protocol: c_int;
  }
  extern type sys_addrinfo_ptr_t; // opaque

  proc sys_addrinfo_ptr_t.flags:c_int { return sys_getaddrinfo_flags(this); }
  proc sys_addrinfo_ptr_t.family:c_int { return sys_getaddrinfo_family(this); }
  proc sys_addrinfo_ptr_t.socktype:c_int { return sys_getaddrinfo_socktype(this); }
  proc sys_addrinfo_ptr_t.socktype:c_int { return sys_getaddrinfo_socktype(this); }
  proc sys_addrinfo_ptr_t.addr:sys_sockaddr_t { return sys_getaddrinfo_addr(this); }
  // Not supported yet
  // proc sys_addrinfo_ptr_t.canonname:c_string { return sys_getaddrinfo_canonname(this); }
  proc sys_addrinfo_ptr_t.next:sys_addrinfo_ptr_t { return sys_getaddrinfo_next(this); }

  extern proc sys_init_sys_sockaddr(ref addr:sys_sockaddr_t);
  extern proc sys_strerror(error:err_t, ref string_out:c_string):err_t;

  extern proc sys_readlink(path:c_string, ref string_out:c_string):err_t;
  extern proc sys_readlink(path:c_string, ref string_out:c_string):err_t;
  extern proc sys_getenv(name:c_string, ref string_out:c_string):c_int;
  extern proc sys_open(pathname:c_string, flags:c_int, mode:mode_t, ref fd_out:fd_t):err_t;
  extern proc sys_close(fd:fd_t):err_t;

  extern proc sys_mmap(addr:c_void_ptr, length:size_t, prot:c_int, flags:c_int, fd:fd_t, offset:off_t, ref ret_out:c_void_ptr):err_t;
  extern proc sys_munmap(addr:c_void_ptr, length:size_t):err_t;

  // readv, writev, preadv, pwritev -- can't (yet) pass array.

  extern proc sys_fcntl(fd:fd_t, cmd:c_int, ref ret_out:c_int):err_t;
  extern proc sys_fcntl_long(fd:fd_t, cmd:c_int, arg:c_long, ref ret_out:c_int):err_t;
  extern proc sys_fcntl_ptr(fd:fd_t, cmd:c_int, arg:c_void_ptr, ref ret_out:c_int):err_t;
  extern proc sys_dup(oldfd:fd_t, ref fd_out:fd_t):err_t;
  extern proc sys_dup2(oldfd:fd_t, newfd:fd_t, ref fd_out:fd_t):err_t;
  extern proc sys_pipe(ref read_fd_out:fd_t, ref write_fd_out:fd_t):err_t;
  extern proc sys_accept(sockfd:fd_t, ref add_out:sys_sockaddr_t, ref fd_out:fd_t):err_t;
  extern proc sys_bind(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;
  extern proc sys_connect(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;
  extern proc sys_getaddrinfo(node:c_string, service:c_string, ref hints:sys_addrinfo_t, ref res_out:sys_addrinfo_ptr_t):err_t;
  extern proc sys_getaddrinfo_flags(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_family(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_socktype(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_protocol(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_addr(res:sys_addrinfo_ptr_t):sys_sockaddr_t;
  extern proc sys_getaddrinfo_next(res:sys_addrinfo_ptr_t):sys_addrinfo_ptr_t;
  extern proc sys_freeaddrinfo(res:sys_addrinfo_ptr_t);

  extern proc sys_getnameinfo(ref addr:sys_sockaddr_t, ref host_out:c_string, ref serv_outc_:c_string, flags:c_int):err_t;
  extern proc sys_getpeername(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;
  extern proc sys_getsockname(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;

  // TODO -- these should be generic, assuming caller knows what they
  // are doing.
  extern proc sys_getsockopt(sockfd:fd_t, level:c_int, optname:c_int, optval:c_void_ptr, ref optlen:socklen_t):err_t;
  extern proc sys_setsockopt(sockfd:fd_t, level:c_int, optname:c_int, optval:c_void_ptr, optlen:socklen_t):err_t;

  extern proc sys_listen(sockfd:fd_t, backlog:c_int):err_t;
  extern proc sys_shutdown(sockfd:fd_t, how:c_int):err_t;
  extern proc sys_socket(_domain:c_int, _type:c_int, protocol:c_int, ref sockfd_out:fd_t):err_t;
  extern proc sys_socketpair(_domain:c_int, _type:c_int, protocol:c_int, ref sockfd_out_a:fd_t, ref sockfd_out_b:fd_t):err_t;

  // recv, recvfrom, recvmsg, send, sendto, sendmsg are in io
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* Sys module. For low level programming and system call wrapping */
module Sys {
  /* BASIC TYPES */
  use SysBasic;

  /* CONSTANTS */

  // note that if a constant is not defined in the C environment,
  // the compile will fail when it is used (and not here!)
  // That means that we can just list all of the constants we know of,
  // for different OSs, and if one is missing errors will occur in
  // C code generation.

  // basic file flags
  extern const O_RDONLY:c_int;
  extern const O_WRONLY:c_int;
  extern const O_RDWR:c_int;

  // POSIX flags
  extern const O_APPEND:c_int;
  extern const O_CREAT:c_int;
  extern const O_DSYNC:c_int;
  extern const O_EXCL:c_int;
  extern const O_NOCTTY:c_int;
  extern const O_NONBLOCK:c_int;
  extern const O_RSYNC:c_int;
  extern const O_SYNC:c_int;
  extern const O_TRUNC:c_int;

  // Linux flags (make them 0 if not on linux, please)
  //extern const O_ASYNC:c_int;
  //extern const O_CLOEXEC:c_int;
  //extern const O_DIRECT:c_int;
  //extern const O_DIRECTORY:c_int;
  //extern const O_LARGEFILE:c_int;
  //extern const O_NOATIME:c_int;
  //extern const O_NOFOLLOW:c_int;
  //extern const O_NDELAY:c_int;

  // open permission flags.
  // These are too complicated for the LLVM backend!
  //extern const S_IRWXU:c_int;
  //extern const S_IRUSR:c_int;
  //extern const S_IWUSR:c_int;
  //extern const S_IXUSR:c_int;
  //extern const S_IRWXG:c_int;
  //extern const S_IRGRP:c_int;
  //extern const S_IWGRP:c_int;
  //extern const S_IXGRP:c_int;
  //extern const S_IRWXO:c_int;
  //extern const S_IROTH:c_int;
  //extern const S_IWOTH:c_int;
  //extern const S_IXOTH:c_int;

  // socket domains
  extern const AF_UNIX:c_int;
  extern const AF_LOCAL:c_int;
  extern const AF_INET:c_int;
  extern const AF_INET6:c_int;
  extern const AF_IPX:c_int;
  extern const AF_NETLINK:c_int;
  extern const AF_X25:c_int;
  extern const AF_AX25:c_int;
  extern const AF_ATMPVC:c_int;
  extern const AF_APPLETALK:c_int;
  extern const AF_PACKET:c_int;

  // socket types
  extern const SOCK_STREAM:c_int;
  extern const SOCK_DGRAM:c_int;
  extern const SOCK_SEQPACKET:c_int;
  extern const SOCK_RAW:c_int;
  extern const SOCK_RDM:c_int;

  extern const SOCK_NONBLOCK:c_int;
  extern const SOCK_CLOEXEC:c_int;

  // sendmsg flags.
  extern const MSG_CONFIRM:c_int;
  extern const MSG_DONTROUTE:c_int;
  extern const MSG_DONTWAIT:c_int;
  extern const MSG_EOR:c_int;
  extern const MSG_MORE:c_int;
  extern const MSG_NOSIGNAL:c_int;
  extern const MSG_OOB:c_int;

  // recvmsg flags
  extern const MSG_CMSG_CLOEXEC:c_int;
  extern const MSG_ERRQUEUE:c_int;
  extern const MSG_PEEK:c_int;
  extern const MSG_TRUNC:c_int;
  extern const MSG_WAITALL:c_int;

  //extern const MSG_EOR:c_int; sendmsg flag
  extern const MSG_CTRUNC:c_int;

  // shutdown how.
  extern const SHUT_RD:c_int;
  extern const SHUT_WR:c_int;
  extern const SHUT_RDWR:c_int;

  // socket option 'levels'
  extern const IPPROTO_IP:c_int;
  extern const IPPROTO_IPV6:c_int;
  extern const IPPROTO_TCP:c_int;
  extern const IPPROTO_UDP:c_int;

  // IP socket options
  extern const IP_ADD_MEMBERSHIP:c_int;
  extern const IP_DROP_MEMBERSHIP:c_int;
  extern const IP_HDRINCL:c_int;
  extern const IP_MTU:c_int;
  extern const IP_MTU_DISCOVER:c_int;
  extern const IP_MULTICAST_IF:c_int;
  extern const IP_MULTICAST_LOOP:c_int;
  extern const IP_MULTICAST_TTL:c_int;
  extern const IP_OPTIONS:c_int;
  extern const IP_PKTINFO:c_int;
  extern const IP_RECVERR:c_int;
  extern const IP_RECVOPTS:c_int;
  extern const IP_RECVTOS:c_int;
  extern const IP_RECVTTL:c_int;
  extern const IP_RETOPTS:c_int;
  extern const IP_ROUTER_ALERT:c_int;
  extern const IP_TOS:c_int;
  extern const IP_TTL:c_int;

  // IP 6 socket options
  extern const IPV6_ADDRFORM:c_int;
  extern const IPV6_ADD_MEMBERSHIP:c_int;
  extern const IPV6_DROP_MEMBERSHIP:c_int;
  extern const IPV6_MTU:c_int;
  extern const IPV6_MTU_DISCOVER:c_int;
  extern const IPV6_MULTICAST_HOPS:c_int;
  extern const IPV6_MULTICAST_IF:c_int;
  extern const IPV6_MULTICAST_LOOP:c_int;
  extern const IPV6_PKTINFO:c_int;
  extern const IPV6_RTHDR:c_int;
  extern const IPV6_AUTHHDR:c_int;
  extern const IPV6_DSTOPTS:c_int;
  extern const IPV6_HOPOPTS:c_int;
  //extern const IPV6_FLOWINFO:c_int;
  extern const IPV6_HOPLIMIT:c_int;
  extern const IPV6_RECVERR:c_int;
  extern const IPV6_ROUTER_ALERT:c_int;
  extern const IPV6_UNICAST_HOPS:c_int;
  extern const IPV6_V6ONLY:c_int;

  // TCP socket options
  extern const TCP_CORK:c_int;
  extern const TCP_DEFER_ACCEPT:c_int;
  extern const TCP_INFO:c_int;
  extern const TCP_KEEPCNT:c_int;
  extern const TCP_KEEPIDLE:c_int;
  extern const TCP_KEEPINTVL:c_int;
  extern const TCP_LINGER2:c_int;
  extern const TCP_MAXSEG:c_int;
  extern const TCP_NODELAY:c_int;
  extern const TCP_QUICKACK:c_int;
  extern const TCP_SYNCNT:c_int;
  extern const TCP_WINDOW_CLAMP:c_int;

  // UDP socket options
  //extern const UDP_CORK:c_int;

  /* SOCKET STRUCTURE TYPES */

  extern type sys_sockaddr_storage_t;
  extern record sys_sockaddr_t {
    var addr:sys_sockaddr_storage_t;
    var len:socklen_t;
    proc sys_sockaddr_t() {
      sys_init_sys_sockaddr_t(this);
    }
  }

  extern record sys_addrinfo_t {
    var ai_flags: c_int;
    var ai_family: c_int;
    var ai_socktype: c_int;
    var ai_protocol: c_int;
  }
  extern type sys_addrinfo_ptr_t; // opaque

  proc sys_addrinfo_ptr_t.flags:c_int { return sys_getaddrinfo_flags(this); }
  proc sys_addrinfo_ptr_t.family:c_int { return sys_getaddrinfo_family(this); }
  proc sys_addrinfo_ptr_t.socktype:c_int { return sys_getaddrinfo_socktype(this); }
  proc sys_addrinfo_ptr_t.socktype:c_int { return sys_getaddrinfo_socktype(this); }
  proc sys_addrinfo_ptr_t.addr:sys_sockaddr_t { return sys_getaddrinfo_addr(this); }
  // Not supported yet
  // proc sys_addrinfo_ptr_t.canonname:c_string { return sys_getaddrinfo_canonname(this); }
  proc sys_addrinfo_ptr_t.next:sys_addrinfo_ptr_t { return sys_getaddrinfo_next(this); }

  extern proc sys_init_sys_sockaddr(ref addr:sys_sockaddr_t);
  extern proc sys_strerror(error:err_t, ref string_out:c_string):err_t;

  extern proc sys_readlink(path:c_string, ref string_out:c_string):err_t;
  extern proc sys_readlink(path:c_string, ref string_out:c_string):err_t;
  extern proc sys_getenv(name:c_string, ref string_out:c_string):c_int;
  extern proc sys_open(pathname:c_string, flags:c_int, mode:mode_t, ref fd_out:fd_t):err_t;
  extern proc sys_close(fd:fd_t):err_t;

  extern proc sys_mmap(addr:c_void_ptr, length:size_t, prot:c_int, flags:c_int, fd:fd_t, offset:off_t, ref ret_out:c_void_ptr):err_t;
  extern proc sys_munmap(addr:c_void_ptr, length:size_t):err_t;

  // readv, writev, preadv, pwritev -- can't (yet) pass array.

  extern proc sys_fcntl(fd:fd_t, cmd:c_int, ref ret_out:c_int):err_t;
  extern proc sys_fcntl_long(fd:fd_t, cmd:c_int, arg:c_long, ref ret_out:c_int):err_t;
  extern proc sys_fcntl_ptr(fd:fd_t, cmd:c_int, arg:c_void_ptr, ref ret_out:c_int):err_t;
  extern proc sys_dup(oldfd:fd_t, ref fd_out:fd_t):err_t;
  extern proc sys_dup2(oldfd:fd_t, newfd:fd_t, ref fd_out:fd_t):err_t;
  extern proc sys_pipe(ref read_fd_out:fd_t, ref write_fd_out:fd_t):err_t;
  extern proc sys_accept(sockfd:fd_t, ref add_out:sys_sockaddr_t, ref fd_out:fd_t):err_t;
  extern proc sys_bind(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;
  extern proc sys_connect(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;
  extern proc sys_getaddrinfo(node:c_string, service:c_string, ref hints:sys_addrinfo_t, ref res_out:sys_addrinfo_ptr_t):err_t;
  extern proc sys_getaddrinfo_flags(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_family(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_socktype(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_protocol(res:sys_addrinfo_ptr_t):c_int;
  extern proc sys_getaddrinfo_addr(res:sys_addrinfo_ptr_t):sys_sockaddr_t;
  extern proc sys_getaddrinfo_next(res:sys_addrinfo_ptr_t):sys_addrinfo_ptr_t;
  extern proc sys_freeaddrinfo(res:sys_addrinfo_ptr_t);

  extern proc sys_getnameinfo(ref addr:sys_sockaddr_t, ref host_out:c_string, ref serv_outc_:c_string, flags:c_int):err_t;
  extern proc sys_getpeername(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;
  extern proc sys_getsockname(sockfd:fd_t, ref addr:sys_sockaddr_t):err_t;

  // TODO -- these should be generic, assuming caller knows what they
  // are doing.
  extern proc sys_getsockopt(sockfd:fd_t, level:c_int, optname:c_int, optval:c_void_ptr, ref optlen:socklen_t):err_t;
  extern proc sys_setsockopt(sockfd:fd_t, level:c_int, optname:c_int, optval:c_void_ptr, optlen:socklen_t):err_t;

  extern proc sys_listen(sockfd:fd_t, backlog:c_int):err_t;
  extern proc sys_shutdown(sockfd:fd_t, how:c_int):err_t;
  extern proc sys_socket(_domain:c_int, _type:c_int, protocol:c_int, ref sockfd_out:fd_t):err_t;
  extern proc sys_socketpair(_domain:c_int, _type:c_int, protocol:c_int, ref sockfd_out_a:fd_t, ref sockfd_out_b:fd_t):err_t;

  // recv, recvfrom, recvmsg, send, sendto, sendmsg are in io
}


disable writer sync by default

disable writer sync by default

// BUG: declaring use IO, Memory here causes chpl compiler segv

/*

determine number of locales
create map from hash to locale

read a list of words
for each word:
  find hash
  map hash to a locale
  send word to mapped locale

via repl:
  ask for a word
  find hash
  map hash to a locale
  get word info (e.g. count) from locale
  print word info

*/

module Search {

  use IO, Memory;

  config const verbose = false;
  config const sync_writers = false;

  type DocId = int(64);

  // for scoring and compactness purposes, consider making this use more contiguous memory,
  //  or an associative array(s) that use the score or docid as index.
  class Entry {
    var word: string;
    var score: real; // TODO: may not be needed if we use docCount as frequency
    var documentCount: atomic_int64;
    var documents: [0..8192-1] DocId;
  }

  class PartitionIndex {
    var count: atomic_int64;
    var entries: [0..(1024*1024/2)-1] Entry; // TODO: use a hashtable once the general approach is validated
    var writerLock: atomicflag;

    inline proc lockIndexWriter() {
      // writeln("attempting to get lock");
      if (sync_writers) then while writerLock.testAndSet() do chpl_task_yield();
      // writeln("have lock");
    }

    inline proc unlockIndexWriter() {
      // writeln("releasing lock");
      if (sync_writers) then writerLock.clear();
      // writeln("released lock");
    }
  }

  // number of dimensions in the partition space
  config var partitionDimensions = 16;
  var Partitions: [0..partitionDimensions-1] locale;

  var Indices: [0..Partitions.size-1] PartitionIndex;

  proc initPartitions() {
    // project the partitions down to the locales
    for i in 0..Partitions.size-1 {
      Partitions[i] = Locales[i % numLocales];
      on Partitions[i] {
        writeln("partition[", i, "] is mapped to locale ", here.id);

        // allocate the partition index on the partition locale
        Indices[i] = new PartitionIndex();
      }
    }

    writeln();
  }

  proc partitionForWord(word: string): int {
    return 0;
  }

  proc localeForPartition(partition: DocId) {
    if (verbose) then writeln("index: ", partition % Partitions.size);
    return Partitions[partition % Partitions.size];
  }

  proc localeForWord(word: string): locale {
    return localeForPatition(hashFromWord(word));
  }

  proc indexWord(word: string, docid: DocId) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      partitionIndex.lockIndexWriter();

      var entry = entryForWord(word, partitionIndex);
      if (entry != nil) {
        if (verbose) then writeln("adding ", word, " to existing entries on partition ", partition);
        var docCount = entry.documentCount.read();
        if (docCount < entry.documents.size) {
          entry.documents[docCount] = docid;
          entry.documentCount.add(1);
        } else {
          if (verbose) then writeln("TODO: realloc documents");
        }
      } else {
        if (verbose) then writeln("adding new entry ", word , " on partition ", partition);

        var entriesCount = partitionIndex.count.read();
        if (entriesCount < partitionIndex.entries.size) {
          entry = new Entry();
          entry.word = word;
          entry.score = 0;
          entry.documents[0] = docid;
          entry.documentCount.add(1);

          partitionIndex.entries[entriesCount] = entry;
          partitionIndex.count.add(1);
        }
      }

      partitionIndex.unlockIndexWriter();
    }
  }

  proc indexContainsWord(word: string, partitionIndex: PartitionIndex): bool {
    return entryIndexForWord(word, partitionIndex) != -1;
  }

  proc entryForWord(word: string, partitionIndex: PartitionIndex): Entry {
    var entryIndex = entryIndexForWord(word, partitionIndex);
    if (entryIndex >= 0) {
      return partitionIndex.entries[entryIndex];
    }
    return nil;
  }

  proc entryIndexForWord(word: string, partitionIndex: PartitionIndex): int {
    for i in 0..partitionIndex.count.read()-1 {
      if (partitionIndex.entries[i].word == word) {
        return i;
      }
    }
    return -1;
  }

  proc dumpEntry(entry: Entry) {
    on entry {
      writeln("word: ", entry.word, " score: ", entry.score);
      for i in 0..entry.documentCount.read()-1 {
        writeln("\t", entry.documents[i]);
      }
    }
  }

  proc dumpPartition(partition: DocId) {
    var partitionIndex = Indices[partition];
    on partitionIndex {
      writeln("entries on partition (", partition, ") locale (", here.id, ") ", partitionIndex.entries);

      var word: string;
      for i in 0..partitionIndex.count.read()-1 {
        var entry = partitionIndex.entries[i];
        writeln("word: ", entry.word);
        dumpPostingTableForWord(entry.word);
      }
    }
  }

  proc dumpPostingTableForWord(word: string) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      var entryIndex = entryIndexForWord(word, partitionIndex);
      if (entryIndex >= 0) {
        dumpEntry(partitionIndex.entries[entryIndex]);
      } else {
        writeln("word (", word, ") is not in the index");
      }
    }
  }
}

// BUG: declaring use IO, Memory here causes chpl compiler segv

/*

determine number of locales
create map from hash to locale

read a list of words
for each word:
  find hash
  map hash to a locale
  send word to mapped locale

via repl:
  ask for a word
  find hash
  map hash to a locale
  get word info (e.g. count) from locale
  print word info

*/

module Search {

  use IO, Memory;

  config const verbose = false;

  type DocId = int(64);

  // for scoring and compactness purposes, consider making this use more contiguous memory,
  //  or an associative array(s) that use the score or docid as index.
  class Entry {
    var word: string;
    var score: real; // TODO: may not be needed if we use docCount as frequency
    var documentCount: atomic_int64;
    var documents: [0..8192-1] DocId;
  }

  class PartitionIndex {
    var count: atomic_int64;
    var entries: [0..(1024*1024/2)-1] Entry; // TODO: use a hashtable once the general approach is validated
    var writerLock: atomicflag;

    inline proc lockIndexWriter() {
      // writeln("attempting to get lock");
      while writerLock.testAndSet() do chpl_task_yield();
      // writeln("have lock");
    }

    inline proc unlockIndexWriter() {
      // writeln("releasing lock");
      writerLock.clear();
      // writeln("released lock");
    }
  }

  // number of dimensions in the partition space
  config var partitionDimensions = 16;
  var Partitions: [0..partitionDimensions-1] locale;

  var Indices: [0..Partitions.size-1] PartitionIndex;

  proc initPartitions() {
    // project the partitions down to the locales
    for i in 0..Partitions.size-1 {
      Partitions[i] = Locales[i % numLocales];
      on Partitions[i] {
        writeln("partition[", i, "] is mapped to locale ", here.id);

        // allocate the partition index on the partition locale
        Indices[i] = new PartitionIndex();
      }
    }

    writeln();
  }

  proc partitionForWord(word: string): int {
    return 0;
  }

  proc localeForPartition(partition: DocId) {
    if (verbose) then writeln("index: ", partition % Partitions.size);
    return Partitions[partition % Partitions.size];
  }

  proc localeForWord(word: string): locale {
    return localeForPatition(hashFromWord(word));
  }

  proc indexWord(word: string, docid: DocId) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      partitionIndex.lockIndexWriter();

      var entry = entryForWord(word, partitionIndex);
      if (entry != nil) {
        if (verbose) then writeln("adding ", word, " to existing entries on partition ", partition);
        var docCount = entry.documentCount.read();
        if (docCount < entry.documents.size) {
          entry.documents[docCount] = docid;
          entry.documentCount.add(1);
        } else {
          if (verbose) then writeln("TODO: realloc documents");
        }
      } else {
        if (verbose) then writeln("adding new entry ", word , " on partition ", partition);

        var entriesCount = partitionIndex.count.read();
        if (entriesCount < partitionIndex.entries.size) {
          entry = new Entry();
          entry.word = word;
          entry.score = 0;
          entry.documents[0] = docid;
          entry.documentCount.add(1);

          partitionIndex.entries[entriesCount] = entry;
          partitionIndex.count.add(1);
        }
      }

      partitionIndex.unlockIndexWriter();
    }
  }

  proc indexContainsWord(word: string, partitionIndex: PartitionIndex): bool {
    return entryIndexForWord(word, partitionIndex) != -1;
  }

  proc entryForWord(word: string, partitionIndex: PartitionIndex): Entry {
    var entryIndex = entryIndexForWord(word, partitionIndex);
    if (entryIndex >= 0) {
      return partitionIndex.entries[entryIndex];
    }
    return nil;
  }

  proc entryIndexForWord(word: string, partitionIndex: PartitionIndex): int {
    for i in 0..partitionIndex.count.read()-1 {
      if (partitionIndex.entries[i].word == word) {
        return i;
      }
    }
    return -1;
  }

  proc dumpEntry(entry: Entry) {
    on entry {
      writeln("word: ", entry.word, " score: ", entry.score);
      for i in 0..entry.documentCount.read()-1 {
        writeln("\t", entry.documents[i]);
      }
    }
  }

  proc dumpPartition(partition: DocId) {
    var partitionIndex = Indices[partition];
    on partitionIndex {
      writeln("entries on partition (", partition, ") locale (", here.id, ") ", partitionIndex.entries);

      var word: string;
      for i in 0..partitionIndex.count.read()-1 {
        var entry = partitionIndex.entries[i];
        writeln("word: ", entry.word);
        dumpPostingTableForWord(entry.word);
      }
    }
  }

  proc dumpPostingTableForWord(word: string) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      var entryIndex = entryIndexForWord(word, partitionIndex);
      if (entryIndex >= 0) {
        dumpEntry(partitionIndex.entries[entryIndex]);
      } else {
        writeln("word (", word, ") is not in the index");
      }
    }
  }
}