Fixed index expressions (which now must be parenthesized) that I missed.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@7989 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// NAS FT - Initial port to Chapel based on ZPL

use BitOps;

config const
  verbose         = true,
  timers_enabled  = false,
  timers_disabled = false;

const
  arand = 1220703125.0,
  seed  = 314159265.0,
  r23   = 0.5**23,
  t23   = 2.0**23,
  r46   = 0.5**46,
  t46   = 2.0**46;

def nextrandlc(x : real, a : real) {
  var t1 = r23 * a;
  var a1 = floor(t1);
  var a2 = a - t23 * a1;
  t1 = r23 * x;
  var x1 = floor(t1);
  var x2 = x - t23 * x1;
  t1 = a1 * x2 + a2 * x1;
  var t2 = floor(r23 * t1);
  var z  = t1 - t23 * t2;
  var t3 = t23 * z + a2 * x2;
  var t4 = floor(r46 * t3);
  var x3 = t3 - t46 * t4;
  return (x3, r46 * x3);
}

def initrandlc(seed, a : real, in n : int) : real {
  var i : int, t : real, g : real;
  var x : real = seed;
  t = a;
  while n != 0 do {
    i = n / 2;
    if 2 * i != n then
      (x, g) = nextrandlc(x, t);
    (t, g) = nextrandlc(t, t);
    n = i;
  }
  return x;
}

var randlc_last_n : int = -2,
    randlc_last_x : real;
def randlc(n : int) : real {
  var result : real;
  if n != randlc_last_n + 1 then
    randlc_last_x = initrandlc(seed, arand, n);
  randlc_last_n = n;
  (randlc_last_x, result) = nextrandlc(randlc_last_x, arand);
  return result;
}

enum classes {S = 1, W, A, B, C, D};

/*
var class_defaults : [S..D] (string, int, int, int, int) =
  (/ ('S',   64,   64,   64,  6),
     ('W',   32,  128,  128,  6),
     ('A',  128,  256,  512,  6),
     ('B',  256,  256,  512, 20),
     ('C',  512,  512,  512, 20),
     ('D', 1024, 1024, 2048, 25) /);
*/

config const problem_class = 1;

config var
  nx : int = 64,
  ny : int = 64,
  nz : int = 64,
  niter : int = 6;

const
  alpha = 1.0e-6,
  pi = 3.141592653589793238,
  epsilon = 1.0e-12;

var
  DXYZ : domain(3) = [0..nx-1, 0..ny-1, 0..nz-1],
  U0 : [DXYZ] complex,
  U1 : [DXYZ] complex,
  U2 : [DXYZ] complex,
  Twiddle : [DXYZ] real;

def compute_initial_conditions(X1) {
  forall (i,j,k) in DXYZ {
    X1(i,j,k).re = randlc(((i*ny+j)*nz+k)*2);
    X1(i,j,k).im = randlc(((i*ny+j)*nz+k)*2+1);
  }
}

def compute_index_map(Twiddle) {
  const ap = -4.0 * alpha * pi * pi;
  forall (i,j,k) in DXYZ do
    Twiddle(i,j,k) = exp(ap*(((i+nx/2) % nx - nx/2)**2 +
                             ((j+ny/2) % ny - ny/2)**2 +
                             ((k+nz/2) % nz - nz/2)**2));
}

var fftblock : int = 16, fftblockpad : int = 18;

var u : [0..nz-1] complex;

def fft_init() {
  var t : real, ti : real;
  var m = bpop(nz-1);
  var ku = 2;
  var ln = 1;
  u(0) = m+0i;
  for j in 1..m {
    for i in 0..ln-1 {
      u(i+ku-1).re = cos(i*pi/ln);
      u(i+ku-1).im = sin(i*pi/ln);
    }
    ku += ln;
    ln *= 2;
  }
}

def fftz2(dir, l, m, n, ny, ny1 : int, u, x, y) {
  var lk = 1<<(l-1), li = 1<<(m-l), lj = 2*lk;
  for i in 0..li-1 {
    var i11 = i * lk, i12 = i11 + n/2, i21 = i * lj, i22 = i21 + lk;
    var u1 = if dir == 1 then u(li+i) else conjg(u(li+i));
    for k in 0..lk-1 {
      for j in 0..ny-1 {
        var x11 = x(i11+k, j);
        var x21 = x(i12+k, j);
        y(i21+k, j) = x11+x21;
        y(i22+k, j) = u1*(x11-x21);
      }
    }
  }
}

def cfftz(dir, m, n, ny, ny1 : int, x, y) {
  for l in 1..m by 2 {
    fftz2(dir, l, m, n, ny, ny1, u, x, y);
    if l != m then
      fftz2(dir, l+1, m, n, ny, ny1, u, y, x);
    else
      forall ij in [0..n-1, 0..ny-1] do
        x(ij) = y(ij);
  }
}

def cffts1(dir, n, X1, X2, ny, ny1, x, y) {
  for j in DXYZ(2) {
    for kk in DXYZ(3) by ny {
      [(i1,_,i3) in [0..n-1,j..j,kk..kk+ny-1]] x(i1,i3-kk) = X1(i1,j,i3);
      cfftz(dir, bpop(n-1), n, ny, ny1, x, y);
      [(i1,_,i3) in [0..n-1,j..j,kk..kk+ny-1]] X2(i1,j,i3) = x(i1,i3-kk);
    }
  }
}

def cffts2(dir, n, X1, X2, ny, ny1, x, y) {
  for i in DXYZ(1) {
    for kk in DXYZ(3) by ny {
      [(_,i2,i3) in [i..i,0..n-1,kk..kk+ny-1]] x(i2,i3-kk) = X1(i,i2,i3);
      cfftz(dir, bpop(n-1), n, ny, ny1, x, y);
      [(_,i2,i3) in [i..i,0..n-1,kk..kk+ny-1]] X2(i,i2,i3) = x(i2,i3-kk);
    }
  }
}

def cffts3(dir, n, X1, X2, ny, ny1, x, y) {
  for i in DXYZ(1) {
    for jj in DXYZ(2) by ny {
      [(_,i2,i3) in [i..i,jj..jj+ny-1,0..n-1]] x(i3,i2-jj) = X1(i,i2,i3);
      cfftz(dir, bpop(n-1), n, ny, ny1, x, y);
      [(_,i2,i3) in [i..i,jj..jj+ny-1,0..n-1]] X2(i,i2,i3) = x(i3,i2-jj);
    }
  }
}

def fft(dir : int, X1, X2) {
  var x : [0..nz-1, 0..fftblockpad-1] complex;
  var y : [0..nz-1, 0..fftblockpad-1] complex;
  if dir == 1 {
    cffts3(dir, nz, X1, X1, fftblock, fftblockpad, x, y);
    cffts2(dir, ny, X1, X1, fftblock, fftblockpad, x, y);
    cffts1(dir, nx, X1, X2, fftblock, fftblockpad, x, y);
  } else {
    cffts1(dir, nx, X1, X1, fftblock, fftblockpad, x, y);
    cffts2(dir, ny, X1, X1, fftblock, fftblockpad, x, y);
    cffts3(dir, nz, X1, X2, fftblock, fftblockpad, x, y);
  }
}

def evolve(X1, X2, Twiddle) {
  forall ijk in DXYZ {
    X1(ijk) *= (Twiddle(ijk) + 0i);
    X2(ijk) = X1(ijk);
  }
}

var sums : [1..niter] complex; -- verification checksums

const
  vdata_s : [1..6] complex =
    (/ 5.546087004964e+02+4.845363331978e+02i,
       5.546385409189e+02+4.865304269511e+02i,
       5.546148406171e+02+4.883910722336e+02i,
       5.545423607415e+02+4.901273169046e+02i,
       5.544255039624e+02+4.917475857993e+02i,
       5.542683411902e+02+4.932597244941e+02i /),

  vdata_w : [1..6] complex =
    (/ 5.673612178944e+02+5.293246849175e+02i,
       5.631436885271e+02+5.282149986629e+02i,
       5.594024089970e+02+5.270996558037e+02i,
       5.560698047020e+02+5.260027904925e+02i,
       5.530898991250e+02+5.249400845633e+02i,
       5.504159734538e+02+5.239212247086e+02i /),

  vdata_a : [1..6] complex =
    (/ 5.046735008193e+02+5.114047905510e+02i,
       5.059412319734e+02+5.098809666433e+02i,
       5.069376896287e+02+5.098144042213e+02i,
       5.077892868474e+02+5.101336130759e+02i,
       5.085233095391e+02+5.104914655194e+02i,
       5.091487099959e+02+5.107917842803e+02i /),

  vdata_b : [1..20] complex =
    (/ 5.177643571579e+02+5.077803458597e+02i,
       5.154521291263e+02+5.088249431599e+02i,
       5.146409228649e+02+5.096208912659e+02i,
       5.142378756213e+02+5.101023387619e+02i,
       5.139626667737e+02+5.103976610617e+02i,
       5.137423460082e+02+5.105948019802e+02i,
       5.135547056878e+02+5.107404165783e+02i,
       5.133910925466e+02+5.108576573661e+02i,
       5.132470705390e+02+5.109577278523e+02i,
       5.131197729984e+02+5.110460304483e+02i,
       5.130070319283e+02+5.111252433800e+02i,
       5.129070537032e+02+5.111968077718e+02i,
       5.128182883502e+02+5.112616233064e+02i,
       5.127393733383e+02+5.113203605551e+02i,
       5.126691062020e+02+5.113735928093e+02i,
       5.126064276004e+02+5.114218460548e+02i,
       5.125504076570e+02+5.114656139760e+02i,
       5.125002331720e+02+5.115053595966e+02i,
       5.124551951846e+02+5.115415130407e+02i,
       5.124146770029e+02+5.115744692211e+02i /),

  vdata_c : [1..20] complex =
    (/ 5.195078707457e+02+5.149019699238e+02i,
       5.155422171134e+02+5.127578201997e+02i,
       5.144678022222e+02+5.122251847514e+02i,
       5.140150594328e+02+5.121090289018e+02i,
       5.137550426810e+02+5.121143685824e+02i,
       5.135811056728e+02+5.121496764568e+02i,
       5.134569343165e+02+5.121870921893e+02i,
       5.133651975661e+02+5.122193250322e+02i,
       5.132955192805e+02+5.122454735794e+02i,
       5.132410471738e+02+5.122663649603e+02i,
       5.131971141679e+02+5.122830879827e+02i,
       5.131605205716e+02+5.122965869718e+02i,
       5.131290734194e+02+5.123075927445e+02i,
       5.131012720314e+02+5.123166486553e+02i,
       5.130760908195e+02+5.123241541685e+02i,
       5.130528295923e+02+5.123304037599e+02i,
       5.130310107773e+02+5.123356167976e+02i,
       5.130103090133e+02+5.123399592211e+02i,
       5.129905029333e+02+5.123435588985e+02i,
       5.129714421109e+02+5.123465164008e+02i /),

  vdata_d : [1..25] complex =
    (/ 5.122230065252e+02+5.118534037109e+02i,
       5.120463975765e+02+5.117061181082e+02i,
       5.119865766760e+02+5.117096364601e+02i,
       5.119518799488e+02+5.117373863950e+02i,
       5.119269088223e+02+5.117680347632e+02i,
       5.119082416858e+02+5.117967875532e+02i,
       5.118943814638e+02+5.118225281841e+02i,
       5.118842385057e+02+5.118451629348e+02i,
       5.118769435632e+02+5.118649119387e+02i,
       5.118718203448e+02+5.118820803844e+02i,
       5.118683569061e+02+5.118969781011e+02i,
       5.118661708593e+02+5.119098918835e+02i,
       5.118649768950e+02+5.119210777066e+02i,
       5.118645605626e+02+5.119307604484e+02i,
       5.118647586618e+02+5.119391362671e+02i,
       5.118654451572e+02+5.119463757241e+02i,
       5.118665212451e+02+5.119526269238e+02i,
       5.118679083821e+02+5.119580184108e+02i,
       5.118695433664e+02+5.119626617538e+02i,
       5.118713748264e+02+5.119666538138e+02i,
       5.118733606701e+02+5.119700787219e+02i,
       5.118754661974e+02+5.119730095953e+02i,
       5.118776626738e+02+5.119755100241e+02i,
       5.118799262314e+02+5.119776353561e+02i,
       5.118822370068e+02+5.119794338060e+02i /);

def verify() {
  var rerr : real, ierr : real;
  for iter in 1..niter {
    select problem_class {
      when S {
        rerr = (sums(iter).re - vdata_s(iter).re) / vdata_s(iter).re;
        ierr = (sums(iter).im - vdata_s(iter).im) / vdata_s(iter).im;
      }
      when W {
        rerr = (sums(iter).re - vdata_w(iter).re) / vdata_w(iter).re;
        ierr = (sums(iter).im - vdata_w(iter).im) / vdata_w(iter).im;
      }
      when A {
        rerr = (sums(iter).re - vdata_a(iter).re) / vdata_a(iter).re;
        ierr = (sums(iter).im - vdata_a(iter).im) / vdata_a(iter).im;
      }
      when B {
        rerr = (sums(iter).re - vdata_b(iter).re) / vdata_b(iter).re;
        ierr = (sums(iter).im - vdata_b(iter).im) / vdata_b(iter).im;
      }
      when C {
        rerr = (sums(iter).re - vdata_c(iter).re) / vdata_c(iter).re;
        ierr = (sums(iter).im - vdata_c(iter).im) / vdata_c(iter).im;
      }
      when D {
        rerr = (sums(iter).re - vdata_d(iter).re) / vdata_d(iter).re;
        ierr = (sums(iter).im - vdata_d(iter).im) / vdata_d(iter).im;
      }
    }
    if !(abs(rerr) <= epsilon && abs(ierr) <= epsilon) then
      return false;
  }
  return true;
}

def checksum(i, X1) {
  var chk : complex;
  [j in 1..1024] chk += X1((5*j) % nx, (3*j) % ny, j % nz);
  chk = chk / ((nx * ny * nz) + 0i);
  sums(i) = chk;
  if verbose then
    writeln("T = ", i, "    Checksum = ", sums(i).re:"%15.12e", " ", sums(i).im:"%15.12e");
}

--
-- Run problem once to ensure all data is touched
--
compute_index_map(Twiddle);
compute_initial_conditions(U1);
fft_init();
fft(1, U1, U0);

--
-- Restart benchmark
--

compute_index_map(Twiddle);
compute_initial_conditions(U1);
fft_init();
fft(1, U1, U0);

for iter in 1..niter {
  evolve(U0, U1, Twiddle);
  fft(-1, U1, U2);
  checksum(iter, U2);
}

if verify() {
  writeln("Result verification successful.");
} else {
  writeln("Result verification failed.");
}

// NAS FT - Initial port to Chapel based on ZPL

use BitOps;

config const
  verbose         = true,
  timers_enabled  = false,
  timers_disabled = false;

const
  arand = 1220703125.0,
  seed  = 314159265.0,
  r23   = 0.5**23,
  t23   = 2.0**23,
  r46   = 0.5**46,
  t46   = 2.0**46;

def nextrandlc(x : real, a : real) {
  var t1 = r23 * a;
  var a1 = floor(t1);
  var a2 = a - t23 * a1;
  t1 = r23 * x;
  var x1 = floor(t1);
  var x2 = x - t23 * x1;
  t1 = a1 * x2 + a2 * x1;
  var t2 = floor(r23 * t1);
  var z  = t1 - t23 * t2;
  var t3 = t23 * z + a2 * x2;
  var t4 = floor(r46 * t3);
  var x3 = t3 - t46 * t4;
  return (x3, r46 * x3);
}

def initrandlc(seed, a : real, in n : int) : real {
  var i : int, t : real, g : real;
  var x : real = seed;
  t = a;
  while n != 0 do {
    i = n / 2;
    if 2 * i != n then
      (x, g) = nextrandlc(x, t);
    (t, g) = nextrandlc(t, t);
    n = i;
  }
  return x;
}

var randlc_last_n : int = -2,
    randlc_last_x : real;
def randlc(n : int) : real {
  var result : real;
  if n != randlc_last_n + 1 then
    randlc_last_x = initrandlc(seed, arand, n);
  randlc_last_n = n;
  (randlc_last_x, result) = nextrandlc(randlc_last_x, arand);
  return result;
}

enum classes {S = 1, W, A, B, C, D};

/*
var class_defaults : [S..D] (string, int, int, int, int) =
  (/ ('S',   64,   64,   64,  6),
     ('W',   32,  128,  128,  6),
     ('A',  128,  256,  512,  6),
     ('B',  256,  256,  512, 20),
     ('C',  512,  512,  512, 20),
     ('D', 1024, 1024, 2048, 25) /);
*/

config const problem_class = 1;

config var
  nx : int = 64,
  ny : int = 64,
  nz : int = 64,
  niter : int = 6;

const
  alpha = 1.0e-6,
  pi = 3.141592653589793238,
  epsilon = 1.0e-12;

var
  DXYZ : domain(3) = [0..nx-1, 0..ny-1, 0..nz-1],
  U0 : [DXYZ] complex,
  U1 : [DXYZ] complex,
  U2 : [DXYZ] complex,
  Twiddle : [DXYZ] real;

def compute_initial_conditions(X1) {
  forall i,j,k in DXYZ {
    X1(i,j,k).re = randlc(((i*ny+j)*nz+k)*2);
    X1(i,j,k).im = randlc(((i*ny+j)*nz+k)*2+1);
  }
}

def compute_index_map(Twiddle) {
  const ap = -4.0 * alpha * pi * pi;
  forall i,j,k in DXYZ do
    Twiddle(i,j,k) = exp(ap*(((i+nx/2) % nx - nx/2)**2 +
                             ((j+ny/2) % ny - ny/2)**2 +
                             ((k+nz/2) % nz - nz/2)**2));
}

var fftblock : int = 16, fftblockpad : int = 18;

var u : [0..nz-1] complex;

def fft_init() {
  var t : real, ti : real;
  var m = bpop(nz-1);
  var ku = 2;
  var ln = 1;
  u(0) = m+0i;
  for j in 1..m {
    for i in 0..ln-1 {
      u(i+ku-1).re = cos(i*pi/ln);
      u(i+ku-1).im = sin(i*pi/ln);
    }
    ku += ln;
    ln *= 2;
  }
}

def fftz2(dir, l, m, n, ny, ny1 : int, u, x, y) {
  var lk = 1<<(l-1), li = 1<<(m-l), lj = 2*lk;
  for i in 0..li-1 {
    var i11 = i * lk, i12 = i11 + n/2, i21 = i * lj, i22 = i21 + lk;
    var u1 = if dir == 1 then u(li+i) else conjg(u(li+i));
    for k in 0..lk-1 {
      for j in 0..ny-1 {
        var x11 = x(i11+k, j);
        var x21 = x(i12+k, j);
        y(i21+k, j) = x11+x21;
        y(i22+k, j) = u1*(x11-x21);
      }
    }
  }
}

def cfftz(dir, m, n, ny, ny1 : int, x, y) {
  for l in 1..m by 2 {
    fftz2(dir, l, m, n, ny, ny1, u, x, y);
    if l != m then
      fftz2(dir, l+1, m, n, ny, ny1, u, y, x);
    else
      forall ij in [0..n-1, 0..ny-1] do
        x(ij) = y(ij);
  }
}

def cffts1(dir, n, X1, X2, ny, ny1, x, y) {
  for j in DXYZ(2) {
    for kk in DXYZ(3) by ny {
      [i1,_,i3 in [0..n-1,j..j,kk..kk+ny-1]] x(i1,i3-kk) = X1(i1,j,i3);
      cfftz(dir, bpop(n-1), n, ny, ny1, x, y);
      [i1,_,i3 in [0..n-1,j..j,kk..kk+ny-1]] X2(i1,j,i3) = x(i1,i3-kk);
    }
  }
}

def cffts2(dir, n, X1, X2, ny, ny1, x, y) {
  for i in DXYZ(1) {
    for kk in DXYZ(3) by ny {
      [_,i2,i3 in [i..i,0..n-1,kk..kk+ny-1]] x(i2,i3-kk) = X1(i,i2,i3);
      cfftz(dir, bpop(n-1), n, ny, ny1, x, y);
      [_,i2,i3 in [i..i,0..n-1,kk..kk+ny-1]] X2(i,i2,i3) = x(i2,i3-kk);
    }
  }
}

def cffts3(dir, n, X1, X2, ny, ny1, x, y) {
  for i in DXYZ(1) {
    for jj in DXYZ(2) by ny {
      [_,i2,i3 in [i..i,jj..jj+ny-1,0..n-1]] x(i3,i2-jj) = X1(i,i2,i3);
      cfftz(dir, bpop(n-1), n, ny, ny1, x, y);
      [_,i2,i3 in [i..i,jj..jj+ny-1,0..n-1]] X2(i,i2,i3) = x(i3,i2-jj);
    }
  }
}

def fft(dir : int, X1, X2) {
  var x : [0..nz-1, 0..fftblockpad-1] complex;
  var y : [0..nz-1, 0..fftblockpad-1] complex;
  if dir == 1 {
    cffts3(dir, nz, X1, X1, fftblock, fftblockpad, x, y);
    cffts2(dir, ny, X1, X1, fftblock, fftblockpad, x, y);
    cffts1(dir, nx, X1, X2, fftblock, fftblockpad, x, y);
  } else {
    cffts1(dir, nx, X1, X1, fftblock, fftblockpad, x, y);
    cffts2(dir, ny, X1, X1, fftblock, fftblockpad, x, y);
    cffts3(dir, nz, X1, X2, fftblock, fftblockpad, x, y);
  }
}

def evolve(X1, X2, Twiddle) {
  forall ijk in DXYZ {
    X1(ijk) *= (Twiddle(ijk) + 0i);
    X2(ijk) = X1(ijk);
  }
}

var sums : [1..niter] complex; -- verification checksums

const
  vdata_s : [1..6] complex =
    (/ 5.546087004964e+02+4.845363331978e+02i,
       5.546385409189e+02+4.865304269511e+02i,
       5.546148406171e+02+4.883910722336e+02i,
       5.545423607415e+02+4.901273169046e+02i,
       5.544255039624e+02+4.917475857993e+02i,
       5.542683411902e+02+4.932597244941e+02i /),

  vdata_w : [1..6] complex =
    (/ 5.673612178944e+02+5.293246849175e+02i,
       5.631436885271e+02+5.282149986629e+02i,
       5.594024089970e+02+5.270996558037e+02i,
       5.560698047020e+02+5.260027904925e+02i,
       5.530898991250e+02+5.249400845633e+02i,
       5.504159734538e+02+5.239212247086e+02i /),

  vdata_a : [1..6] complex =
    (/ 5.046735008193e+02+5.114047905510e+02i,
       5.059412319734e+02+5.098809666433e+02i,
       5.069376896287e+02+5.098144042213e+02i,
       5.077892868474e+02+5.101336130759e+02i,
       5.085233095391e+02+5.104914655194e+02i,
       5.091487099959e+02+5.107917842803e+02i /),

  vdata_b : [1..20] complex =
    (/ 5.177643571579e+02+5.077803458597e+02i,
       5.154521291263e+02+5.088249431599e+02i,
       5.146409228649e+02+5.096208912659e+02i,
       5.142378756213e+02+5.101023387619e+02i,
       5.139626667737e+02+5.103976610617e+02i,
       5.137423460082e+02+5.105948019802e+02i,
       5.135547056878e+02+5.107404165783e+02i,
       5.133910925466e+02+5.108576573661e+02i,
       5.132470705390e+02+5.109577278523e+02i,
       5.131197729984e+02+5.110460304483e+02i,
       5.130070319283e+02+5.111252433800e+02i,
       5.129070537032e+02+5.111968077718e+02i,
       5.128182883502e+02+5.112616233064e+02i,
       5.127393733383e+02+5.113203605551e+02i,
       5.126691062020e+02+5.113735928093e+02i,
       5.126064276004e+02+5.114218460548e+02i,
       5.125504076570e+02+5.114656139760e+02i,
       5.125002331720e+02+5.115053595966e+02i,
       5.124551951846e+02+5.115415130407e+02i,
       5.124146770029e+02+5.115744692211e+02i /),

  vdata_c : [1..20] complex =
    (/ 5.195078707457e+02+5.149019699238e+02i,
       5.155422171134e+02+5.127578201997e+02i,
       5.144678022222e+02+5.122251847514e+02i,
       5.140150594328e+02+5.121090289018e+02i,
       5.137550426810e+02+5.121143685824e+02i,
       5.135811056728e+02+5.121496764568e+02i,
       5.134569343165e+02+5.121870921893e+02i,
       5.133651975661e+02+5.122193250322e+02i,
       5.132955192805e+02+5.122454735794e+02i,
       5.132410471738e+02+5.122663649603e+02i,
       5.131971141679e+02+5.122830879827e+02i,
       5.131605205716e+02+5.122965869718e+02i,
       5.131290734194e+02+5.123075927445e+02i,
       5.131012720314e+02+5.123166486553e+02i,
       5.130760908195e+02+5.123241541685e+02i,
       5.130528295923e+02+5.123304037599e+02i,
       5.130310107773e+02+5.123356167976e+02i,
       5.130103090133e+02+5.123399592211e+02i,
       5.129905029333e+02+5.123435588985e+02i,
       5.129714421109e+02+5.123465164008e+02i /),

  vdata_d : [1..25] complex =
    (/ 5.122230065252e+02+5.118534037109e+02i,
       5.120463975765e+02+5.117061181082e+02i,
       5.119865766760e+02+5.117096364601e+02i,
       5.119518799488e+02+5.117373863950e+02i,
       5.119269088223e+02+5.117680347632e+02i,
       5.119082416858e+02+5.117967875532e+02i,
       5.118943814638e+02+5.118225281841e+02i,
       5.118842385057e+02+5.118451629348e+02i,
       5.118769435632e+02+5.118649119387e+02i,
       5.118718203448e+02+5.118820803844e+02i,
       5.118683569061e+02+5.118969781011e+02i,
       5.118661708593e+02+5.119098918835e+02i,
       5.118649768950e+02+5.119210777066e+02i,
       5.118645605626e+02+5.119307604484e+02i,
       5.118647586618e+02+5.119391362671e+02i,
       5.118654451572e+02+5.119463757241e+02i,
       5.118665212451e+02+5.119526269238e+02i,
       5.118679083821e+02+5.119580184108e+02i,
       5.118695433664e+02+5.119626617538e+02i,
       5.118713748264e+02+5.119666538138e+02i,
       5.118733606701e+02+5.119700787219e+02i,
       5.118754661974e+02+5.119730095953e+02i,
       5.118776626738e+02+5.119755100241e+02i,
       5.118799262314e+02+5.119776353561e+02i,
       5.118822370068e+02+5.119794338060e+02i /);

def verify() {
  var rerr : real, ierr : real;
  for iter in 1..niter {
    select problem_class {
      when S {
        rerr = (sums(iter).re - vdata_s(iter).re) / vdata_s(iter).re;
        ierr = (sums(iter).im - vdata_s(iter).im) / vdata_s(iter).im;
      }
      when W {
        rerr = (sums(iter).re - vdata_w(iter).re) / vdata_w(iter).re;
        ierr = (sums(iter).im - vdata_w(iter).im) / vdata_w(iter).im;
      }
      when A {
        rerr = (sums(iter).re - vdata_a(iter).re) / vdata_a(iter).re;
        ierr = (sums(iter).im - vdata_a(iter).im) / vdata_a(iter).im;
      }
      when B {
        rerr = (sums(iter).re - vdata_b(iter).re) / vdata_b(iter).re;
        ierr = (sums(iter).im - vdata_b(iter).im) / vdata_b(iter).im;
      }
      when C {
        rerr = (sums(iter).re - vdata_c(iter).re) / vdata_c(iter).re;
        ierr = (sums(iter).im - vdata_c(iter).im) / vdata_c(iter).im;
      }
      when D {
        rerr = (sums(iter).re - vdata_d(iter).re) / vdata_d(iter).re;
        ierr = (sums(iter).im - vdata_d(iter).im) / vdata_d(iter).im;
      }
    }
    if !(abs(rerr) <= epsilon && abs(ierr) <= epsilon) then
      return false;
  }
  return true;
}

def checksum(i, X1) {
  var chk : complex;
  [j in 1..1024] chk += X1((5*j) % nx, (3*j) % ny, j % nz);
  chk = chk / ((nx * ny * nz) + 0i);
  sums(i) = chk;
  if verbose then
    writeln("T = ", i, "    Checksum = ", sums(i).re:"%15.12e", " ", sums(i).im:"%15.12e");
}

--
-- Run problem once to ensure all data is touched
--
compute_index_map(Twiddle);
compute_initial_conditions(U1);
fft_init();
fft(1, U1, U0);

--
-- Restart benchmark
--

compute_index_map(Twiddle);
compute_initial_conditions(U1);
fft_init();
fft(1, U1, U0);

for iter in 1..niter {
  evolve(U0, U1, Twiddle);
  fft(-1, U1, U2);
  checksum(iter, U2);
}

if verify() {
  writeln("Result verification successful.");
} else {
  writeln("Result verification failed.");
}


Moved all the extern runtime function decls in FileSystem inside the functions

Brad was saying we didn't want extern functions declared at the module level
scope, so I moved all the ones in the FileSystem module into the individual
functions and methods.

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use Error;

/* Change the current working directory of the current locale to the specified
   name. Returns any errors that occurred via an out parameter.
   err: a syserr used to indicate if an error occurred
   name: a string indicating a new directory

   Note: this is not safe within a parallel context.  A chdir call in one task
   will affect the current working directory of all tasks for that locale.
*/
proc locale.chdir(out err: syserr, name: string) {
  extern proc chpl_fs_chdir(name: c_string):syserr;

  on this {
    err = chpl_fs_chdir(name.c_str());
  }
}

/* Change the current working directory of the current locale to the specified
   name. Generates an error message if one occurred.
   name: a string indicating a new directory

   Note: this is not safe within a parallel context.  A chdir call in one task
   will affect the current working directory of all tasks for that locale.
*/
proc locale.chdir(name: string) {
  var err: syserr = ENOERR;
  chdir(err, name);
  if err != ENOERR then ioerror(err, "in chdir", name);
}

/* Set the permissions of the file or directory specified by name to that
   indicated by settings.  Returns any errors that occurred via an out
   parameter.

   :arg err: a syserr used to indicate if an error occurred
   :arg name: the name of the file/directory which should have its permissions
              alterred.
   :arg mode: an integer representing the permissions desired for the file in
              question.  See description of the provided constants for
              potential values.
*/
proc chmod(out err: syserr, name: string, mode: int) {
  extern proc chpl_fs_chmod(name: c_string, mode: int): syserr;

  err = chpl_fs_chmod(name.c_str(), mode);
}

/* Set the permissions of the file or directory specified by name to that
   indicated by settings, and may generate an error message

   :arg name: the name of the file/directory which should have its permissions
              alterred.
   :arg mode: an integer representing the permissions desired for the file
              in question.  See description of the provided constants for potential
              values.
*/
proc chmod(name: string, mode: int){
  var err: syserr = ENOERR;
  chmod(err, name, mode);
  if err != ENOERR then ioerror(err, "in chmod", name);
}

/* Changes one or both of the owner and group id of the named file to the
   specified values.  If uid or gid are -1, the value in question will remain
   unchanged.
   err: a syserr used to indicate if an error occurred
   name: the name of the file to be changed.
   uid: user id to use as new owner, or -1 if it should remain the same.
   gid: group id to use as the new group owner, or -1 if it should remain the same.
*/
proc chown(out err: syserr, name: string, uid: int, gid: int) {
  extern proc chpl_fs_chown(name: c_string, uid: c_int, gid: c_int):syserr;

  err = chpl_fs_chown(name.c_str(), uid:c_int, gid:c_int);
}

/* Changes one or both of the owner and group id of the named file to the
   specified values.  If uid or gid are -1, the value in question will remain
   unchanged. Generates an error message if one occurred.
   name: the name of the file to be changed.
   uid: user id to use as new owner, or -1 if it should remain the same.
   gid: group id to use as the new group owner, or -1 if it should remain the same.
*/
proc chown(name: string, uid: int, gid: int) {
  var err: syserr = ENOERR;
  chown(err, name, uid, gid);
  if err != ENOERR then ioerror(err, "in chown", name);
}

/* Returns the current working directory for the current locale.
   err: a syserr used to indicate if an error occurred

   Note: another task on this locale can change the current working
   directory from underneath this task, so use caution when making use
   of this function in a parallel environment.
*/
proc locale.cwd(out err: syserr): string {
  extern proc chpl_fs_cwd(ref working_dir:c_string_copy):syserr;

  var ret:string;
  on this {
    var tmp:c_string_copy;
    // c_strings and c_string_copy's can't cross on statements.
    err = chpl_fs_cwd(tmp);
    if (err != ENOERR) {
      ret = "";
    } else {
      // This version of toString steals its operand.  No need to free.
      ret = toString(tmp);
    }
  }
  return ret;
}

/* Returns the current working directory for the current locale. Generates an
   error message if one occurred.

   Note: another task on this locale can change the current working
   directory from underneath this task, so use caution when making use
   of this function in a parallel environment.
*/
proc locale.cwd(): string {
  var err: syserr = ENOERR;
  var ret = cwd(err);
  if err != ENOERR then ioerror(err, "in cwd");
  return ret;
}

/* Returns true if the provided filename corresponds to an existing file, false
   otherwise.  Returns false for broken symbolic links.  Returns any errors that
   occurred via an out parameter
   name: a string used to attempt to find the file specified.
*/
proc exists(out err: syserr, name: string): bool {
  extern proc chpl_fs_exists(ref result:c_int, name: c_string): syserr;

  var ret:c_int;
  err = chpl_fs_exists(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the provided filename corresponds to an existing file, false
   otherwise.  Returns false for broken symbolic links.  Generates an error
   message if one occurred.
   name: a string used to attempt to find the file specified.
*/
proc exists(name: string): bool {
  var err: syserr = ENOERR;
  var ret = exists(err, name);
  if err != ENOERR then ioerror(err, "in exists");
  return ret;
}

/* Returns true if the name corresponds to a directory, false otherwise.
   err: a syserr used to indicate if an error occurred
   name: a string that could be the name of a directory.
*/
proc isDir(out err:syserr, name:string):bool {
  extern proc chpl_fs_is_dir(ref result:c_int, name: c_string):syserr;

  var ret:c_int;
  err = chpl_fs_is_dir(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the name corresponds to a directory, false otherwise.
   Generates an error message if one occurs.
   name: a string that could be the name of a directory.
*/
proc isDir(name:string):bool {
  var err:syserr;
  var ret = isDir(err, name);
  if err != ENOERR then ioerror(err, "in isDir", name);
  return ret;
}

/* Returns true if the name corresponds to a file, false otherwise.
   err: a syserr used to indicate if an error occurred
   name: a string that could be the name of a file.
*/
proc isFile(out err:syserr, name:string):bool {
  extern proc chpl_fs_is_file(ref result:c_int, name: c_string):syserr;

  var ret:c_int;
  err = chpl_fs_is_file(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the name corresponds to a file, false otherwise.
   Generates an error message if one occurs.
   name: a string that could be the name of a file.
*/
proc isFile(name:string):bool {
  var err:syserr;
  var ret = isFile(err, name);
  if err != ENOERR then ioerror(err, "in isFile", name);
  return ret;
}

/* Returns true if the name corresponds to a symbolic link, false if not
   or if symbolic links are not supported.
   name: a string that could be the name of a symbolic link.
*/
proc isLink(out err:syserr, name: string): bool {
  extern proc chpl_fs_is_link(ref result:c_int, name: c_string): syserr;

  var ret:c_int;
  err = chpl_fs_is_link(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the name corresponds to a symbolic link, false if not
   or if symbolic links are not supported.  Generates an error message if one
   occurs
   name: a string that could be the name of a symbolic link.
*/
proc isLink(name: string): bool {
  var err:syserr;
  var ret = isLink(err, name);
  if err != ENOERR then ioerror(err, "in isLink", name);
  return ret;
}

/* These are constant values of the form S_I[R | W | X][USR | GRP | OTH],
   S_IRWX[U | G | O], S_ISUID, S_ISGID, or S_ISVTX, where R corresponds to
   readable, W corresponds to writable, X corresponds to executable, USR and
   U correspond to user, GRP and G correspond to group, OTH and O correspond
   to other, directly tied to the C idea of these constants.  They are intended
   for use with functions that alter the permissions of files or directories.
*/
extern const S_IRUSR: int;
extern const S_IWUSR: int;
extern const S_IXUSR: int;
extern const S_IRWXU: int;

extern const S_IRGRP: int;
extern const S_IWGRP: int;
extern const S_IXGRP: int;
extern const S_IRWXG: int;

extern const S_IROTH: int;
extern const S_IWOTH: int;
extern const S_IXOTH: int;
extern const S_IRWXO: int;

extern const S_ISUID: int;
extern const S_ISGID: int;
extern const S_ISVTX: int;

/* Attempt to create a directory with the given path.  If parents is true,
   will attempt to create any directory in the path that did not previously
   exist.  Returns any errors that occurred via an out parameter

   :arg err: a syserr used to indicate if an error occurred
   :arg name: the name of the directory to be created, fully specified.
   :arg mode: an integer representing the permissions desired for the file in
              question. See description of the provided constants for
              potential values.
   :arg parents: a boolean indicating if parent directories should be created.
                 If set to false, any nonexistent parent will cause an error to
                 occur.

   Important note: In the case where parents is true, there is a potential
   security vulnerability.  Checking whether parent directories exist and
   creating them if not are separate events. So even if parents==true and a
   parent directory didn't exist before this function is called but does exist
   afterward, it's still not necessarily true that this function created that
   parent. Some other concurrent operation could have done so.
*/
proc mkdir(out err: syserr, name: string, mode: int = 0o777,
           parents: bool=false) {
  extern proc chpl_fs_mkdir(name: c_string, mode: int, parents: bool):syserr;

  err = chpl_fs_mkdir(name.c_str(), mode, parents);
}

/* Attempt to create a directory with the given path.  If parents is true,
   will attempt to create any directory in the path that did not previously
   exist.  Generates an error message if one occurred.

   :arg name: the name of the directory to be created, fully specified.
   :arg mode: an integer representing the permissions desired for the file in
              question. See description of the provided constants for
              potential values.
   :arg parents: a boolean indicating if parent directories should be created.
                 If set to false, any nonexistent parent will cause an error to
                 occur.

   Important note: In the case where parents is true, there is a potential
   security vulnerability.  Checking whether parent directories exist and
   creating them if not are separate events. So even if parents==true and a
   parent directory didn't exist before this function is called but does exist
   afterward, it's still not necessarily true that this function created that
   parent. Some other concurrent operation could have done so.
*/
proc mkdir(name: string, mode: int = 0o777, parents: bool=false) {
  var err: syserr = ENOERR;
  mkdir(err, name, mode, parents);
  if err != ENOERR then ioerror(err, "in mkdir", name);
}

/* Renames the file specified by oldname to newname, returning an error
   if one occurred.  The file is not opened during this operation.
   error: a syserr used to indicate if an error occurred during renaming.
   oldname: current name of the file
   newname: name which should refer to the file in the future.*/
proc rename(out error: syserr, oldname, newname: string) {
  extern proc chpl_fs_rename(oldname: c_string, newname: c_string):syserr;

  error = chpl_fs_rename(oldname.c_str(), newname.c_str());
}

/* Renames the file specified by oldname to newname, generating an error message
   if one occurred.  The file is not opened during this operation.
   oldname: current name of the file
   newname: name which should refer to the file in the future.*/
proc rename(oldname, newname: string) {
  var err:syserr = ENOERR;
  rename(err, oldname, newname);
  if err != ENOERR then ioerror(err, "in rename", oldname);
}

/* Removes the file or directory specified by name, returning an error
   if one occurred via an out parameter.
   err: a syserr used to indicate if an error occurred during removal
   name: the name of the file/directory to remove */
proc remove(out err: syserr, name: string) {
  extern proc chpl_fs_remove(name: c_string):syserr;

  err = chpl_fs_remove(name.c_str());
}

/* Removes the file or directory specified by name, generating an error message
   if one occurred.
   name: the name of the file/directory to remove */
proc remove(name: string) {
  var err:syserr = ENOERR;
  remove(err, name);
  if err != ENOERR then ioerror(err, "in remove", name);
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use Error;

extern proc chpl_fs_chdir(name: c_string):syserr;
extern proc chpl_fs_chmod(name: c_string, mode: int): syserr;
extern proc chpl_fs_chown(name: c_string, uid: c_int, gid: c_int):syserr;
extern proc chpl_fs_cwd(ref working_dir:c_string_copy):syserr;
extern proc chpl_fs_exists(ref result:c_int, name: c_string): syserr;
extern proc chpl_fs_is_dir(ref result:c_int, name: c_string):syserr;
extern proc chpl_fs_is_file(ref result:c_int, name: c_string):syserr;
extern proc chpl_fs_is_link(ref result:c_int, name: c_string): syserr;
extern proc chpl_fs_mkdir(name: c_string, mode: int, parents: bool):syserr;
extern proc chpl_fs_rename(oldname: c_string, newname: c_string):syserr;
extern proc chpl_fs_remove(name: c_string):syserr;

/* Change the current working directory of the current locale to the specified
   name. Returns any errors that occurred via an out parameter.
   err: a syserr used to indicate if an error occurred
   name: a string indicating a new directory

   Note: this is not safe within a parallel context.  A chdir call in one task
   will affect the current working directory of all tasks for that locale.
*/
proc locale.chdir(out err: syserr, name: string) {
  on this {
    err = chpl_fs_chdir(name.c_str());
  }
}

/* Change the current working directory of the current locale to the specified
   name. Generates an error message if one occurred.
   name: a string indicating a new directory

   Note: this is not safe within a parallel context.  A chdir call in one task
   will affect the current working directory of all tasks for that locale.
*/
proc locale.chdir(name: string) {
  var err: syserr = ENOERR;
  chdir(err, name);
  if err != ENOERR then ioerror(err, "in chdir", name);
}

/* Set the permissions of the file or directory specified by name to that
   indicated by settings.  Returns any errors that occurred via an out
   parameter.

   :arg err: a syserr used to indicate if an error occurred
   :arg name: the name of the file/directory which should have its permissions
              alterred.
   :arg mode: an integer representing the permissions desired for the file in
              question.  See description of the provided constants for
              potential values.
*/
proc chmod(out err: syserr, name: string, mode: int) {
  err = chpl_fs_chmod(name.c_str(), mode);
}

/* Set the permissions of the file or directory specified by name to that
   indicated by settings, and may generate an error message

   :arg name: the name of the file/directory which should have its permissions
              alterred.
   :arg mode: an integer representing the permissions desired for the file
              in question.  See description of the provided constants for potential
              values.
*/
proc chmod(name: string, mode: int){
  var err: syserr = ENOERR;
  chmod(err, name, mode);
  if err != ENOERR then ioerror(err, "in chmod", name);
}

/* Changes one or both of the owner and group id of the named file to the
   specified values.  If uid or gid are -1, the value in question will remain
   unchanged.
   err: a syserr used to indicate if an error occurred
   name: the name of the file to be changed.
   uid: user id to use as new owner, or -1 if it should remain the same.
   gid: group id to use as the new group owner, or -1 if it should remain the same.
*/
proc chown(out err: syserr, name: string, uid: int, gid: int) {
  err = chpl_fs_chown(name.c_str(), uid:c_int, gid:c_int);
}

/* Changes one or both of the owner and group id of the named file to the
   specified values.  If uid or gid are -1, the value in question will remain
   unchanged. Generates an error message if one occurred.
   name: the name of the file to be changed.
   uid: user id to use as new owner, or -1 if it should remain the same.
   gid: group id to use as the new group owner, or -1 if it should remain the same.
*/
proc chown(name: string, uid: int, gid: int) {
  var err: syserr = ENOERR;
  chown(err, name, uid, gid);
  if err != ENOERR then ioerror(err, "in chown", name);
}

/* Returns the current working directory for the current locale.
   err: a syserr used to indicate if an error occurred

   Note: another task on this locale can change the current working
   directory from underneath this task, so use caution when making use
   of this function in a parallel environment.
*/
proc locale.cwd(out err: syserr): string {
  var ret:string;
  on this {
    var tmp:c_string_copy;
    // c_strings and c_string_copy's can't cross on statements.
    err = chpl_fs_cwd(tmp);
    if (err != ENOERR) {
      ret = "";
    } else {
      // This version of toString steals its operand.  No need to free.
      ret = toString(tmp);
    }
  }
  return ret;
}

/* Returns the current working directory for the current locale. Generates an
   error message if one occurred.

   Note: another task on this locale can change the current working
   directory from underneath this task, so use caution when making use
   of this function in a parallel environment.
*/
proc locale.cwd(): string {
  var err: syserr = ENOERR;
  var ret = cwd(err);
  if err != ENOERR then ioerror(err, "in cwd");
  return ret;
}

/* Returns true if the provided filename corresponds to an existing file, false
   otherwise.  Returns false for broken symbolic links.  Returns any errors that
   occurred via an out parameter
   name: a string used to attempt to find the file specified.
*/
proc exists(out err: syserr, name: string): bool {
  var ret:c_int;
  err = chpl_fs_exists(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the provided filename corresponds to an existing file, false
   otherwise.  Returns false for broken symbolic links.  Generates an error
   message if one occurred.
   name: a string used to attempt to find the file specified.
*/
proc exists(name: string): bool {
  var err: syserr = ENOERR;
  var ret = exists(err, name);
  if err != ENOERR then ioerror(err, "in exists");
  return ret;
}

/* Returns true if the name corresponds to a directory, false otherwise.
   err: a syserr used to indicate if an error occurred
   name: a string that could be the name of a directory.
*/
proc isDir(out err:syserr, name:string):bool {
  var ret:c_int;
  err = chpl_fs_is_dir(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the name corresponds to a directory, false otherwise.
   Generates an error message if one occurs.
   name: a string that could be the name of a directory.
*/
proc isDir(name:string):bool {
  var err:syserr;
  var ret = isDir(err, name);
  if err != ENOERR then ioerror(err, "in isDir", name);
  return ret;
}

/* Returns true if the name corresponds to a file, false otherwise.
   err: a syserr used to indicate if an error occurred
   name: a string that could be the name of a file.
*/
proc isFile(out err:syserr, name:string):bool {
  var ret:c_int;
  err = chpl_fs_is_file(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the name corresponds to a file, false otherwise.
   Generates an error message if one occurs.
   name: a string that could be the name of a file.
*/
proc isFile(name:string):bool {
  var err:syserr;
  var ret = isFile(err, name);
  if err != ENOERR then ioerror(err, "in isFile", name);
  return ret;
}

/* Returns true if the name corresponds to a symbolic link, false if not
   or if symbolic links are not supported.
   name: a string that could be the name of a symbolic link.
*/
proc isLink(out err:syserr, name: string): bool {
  var ret:c_int;
  err = chpl_fs_is_link(ret, name.c_str());
  return ret != 0;
}

/* Returns true if the name corresponds to a symbolic link, false if not
   or if symbolic links are not supported.  Generates an error message if one
   occurs
   name: a string that could be the name of a symbolic link.
*/
proc isLink(name: string): bool {
  var err:syserr;
  var ret = isLink(err, name);
  if err != ENOERR then ioerror(err, "in isLink", name);
  return ret;
}

/* These are constant values of the form S_I[R | W | X][USR | GRP | OTH],
   S_IRWX[U | G | O], S_ISUID, S_ISGID, or S_ISVTX, where R corresponds to
   readable, W corresponds to writable, X corresponds to executable, USR and
   U correspond to user, GRP and G correspond to group, OTH and O correspond
   to other, directly tied to the C idea of these constants.  They are intended
   for use with functions that alter the permissions of files or directories.
*/
extern const S_IRUSR: int;
extern const S_IWUSR: int;
extern const S_IXUSR: int;
extern const S_IRWXU: int;

extern const S_IRGRP: int;
extern const S_IWGRP: int;
extern const S_IXGRP: int;
extern const S_IRWXG: int;

extern const S_IROTH: int;
extern const S_IWOTH: int;
extern const S_IXOTH: int;
extern const S_IRWXO: int;

extern const S_ISUID: int;
extern const S_ISGID: int;
extern const S_ISVTX: int;

/* Attempt to create a directory with the given path.  If parents is true,
   will attempt to create any directory in the path that did not previously
   exist.  Returns any errors that occurred via an out parameter

   :arg err: a syserr used to indicate if an error occurred
   :arg name: the name of the directory to be created, fully specified.
   :arg mode: an integer representing the permissions desired for the file in
              question. See description of the provided constants for
              potential values.
   :arg parents: a boolean indicating if parent directories should be created.
                 If set to false, any nonexistent parent will cause an error to
                 occur.

   Important note: In the case where parents is true, there is a potential
   security vulnerability.  Checking whether parent directories exist and
   creating them if not are separate events. So even if parents==true and a
   parent directory didn't exist before this function is called but does exist
   afterward, it's still not necessarily true that this function created that
   parent. Some other concurrent operation could have done so.
*/
proc mkdir(out err: syserr, name: string, mode: int = 0o777,
           parents: bool=false) {
  err = chpl_fs_mkdir(name.c_str(), mode, parents);
}

/* Attempt to create a directory with the given path.  If parents is true,
   will attempt to create any directory in the path that did not previously
   exist.  Generates an error message if one occurred.

   :arg name: the name of the directory to be created, fully specified.
   :arg mode: an integer representing the permissions desired for the file in
              question. See description of the provided constants for
              potential values.
   :arg parents: a boolean indicating if parent directories should be created.
                 If set to false, any nonexistent parent will cause an error to
                 occur.

   Important note: In the case where parents is true, there is a potential
   security vulnerability.  Checking whether parent directories exist and
   creating them if not are separate events. So even if parents==true and a
   parent directory didn't exist before this function is called but does exist
   afterward, it's still not necessarily true that this function created that
   parent. Some other concurrent operation could have done so.
*/
proc mkdir(name: string, mode: int = 0o777, parents: bool=false) {
  var err: syserr = ENOERR;
  mkdir(err, name, mode, parents);
  if err != ENOERR then ioerror(err, "in mkdir", name);
}

/* Renames the file specified by oldname to newname, returning an error
   if one occurred.  The file is not opened during this operation.
   error: a syserr used to indicate if an error occurred during renaming.
   oldname: current name of the file
   newname: name which should refer to the file in the future.*/
proc rename(out error: syserr, oldname, newname: string) {
  error = chpl_fs_rename(oldname.c_str(), newname.c_str());
}

/* Renames the file specified by oldname to newname, generating an error message
   if one occurred.  The file is not opened during this operation.
   oldname: current name of the file
   newname: name which should refer to the file in the future.*/
proc rename(oldname, newname: string) {
  var err:syserr = ENOERR;
  rename(err, oldname, newname);
  if err != ENOERR then ioerror(err, "in rename", oldname);
}

/* Removes the file or directory specified by name, returning an error
   if one occurred via an out parameter.
   err: a syserr used to indicate if an error occurred during removal
   name: the name of the file/directory to remove */
proc remove(out err: syserr, name: string) {
  err = chpl_fs_remove(name.c_str());
}

/* Removes the file or directory specified by name, generating an error message
   if one occurred.
   name: the name of the file/directory to remove */
proc remove(name: string) {
  var err:syserr = ENOERR;
  remove(err, name);
  if err != ENOERR then ioerror(err, "in remove", name);
}


Changed Sort module to use unbounded sequences, just
for fun.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@9897 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

def InsertionSort(Data: [?Dom], doublecheck = false) where Dom.rank == 1 {
  const lo = Dom.low;
  for i in Dom {
    const ithVal = Data(i);
    var inserted = false;
    for j in [lo..i) by -1 {
      if (ithVal < Data(j)) {
        Data(j+1) = Data(j);
      } else {
        Data(j+1) = ithVal;
        inserted = true;
        break;
      }
    }
    if (!inserted) {
      Data(lo) = ithVal;
    }
  }

  if (doublecheck) then
    for i in [Dom) do
      if (Data(i+1) < Data(i)) then
        halt("InsertionSort did not sort properly: ", Data);
}

def QuickSort(Data: [?Dom], doublecheck = false) where Dom.rank == 1 {
  // grab obvious indices
  const lo = Dom.low,
        hi = Dom.high,
        mid = lo + (hi-lo+1)/2;

  // base case -- use insertion sort
  if (hi - lo < 10) {
    InsertionSort(Data);
    return;
  }

  // find pivot using median-of-3 method
  if (Data(mid) < Data(lo)) then Data(mid) <=> Data(lo);
  if (Data(hi) < Data(lo)) then Data(hi) <=> Data(lo);
  if (Data(hi) < Data(mid)) then Data(hi) <=> Data(mid);
  const pivotVal = Data(mid);
  Data(mid) = Data(hi-1);
  Data(hi-1) = pivotVal;
  // end median-of-3 partitioning

  var loptr = lo,
      hiptr = hi-1;
  while (loptr < hiptr) {
    do { loptr += 1; } while (Data(loptr) < pivotVal);
    do { hiptr -= 1; } while (pivotVal < Data(hiptr));
    if (loptr < hiptr) {
      Data(loptr) <=> Data(hiptr);
    }
  }

  Data(hi-1) = Data(loptr);
  Data(loptr) = pivotVal;

  //  cobegin {
    QuickSort(Data[..loptr-1]);  // could use unbounded ranges here
    QuickSort(Data[loptr+1..]);
    //  }

  if (doublecheck) then
    for i in [Dom) do
      if (Data(i+1) < Data(i)) then
        halt("QuickSort did not sort properly: ", Data);
}

def InsertionSort(Data: [?Dom], doublecheck = false) where Dom.rank == 1 {
  const lo = Dom.low;
  for i in Dom {
    const ithVal = Data(i);
    var inserted = false;
    for j in [lo..i) by -1 {
      if (ithVal < Data(j)) {
        Data(j+1) = Data(j);
      } else {
        Data(j+1) = ithVal;
        inserted = true;
        break;
      }
    }
    if (!inserted) {
      Data(lo) = ithVal;
    }
  }

  if (doublecheck) then
    for i in [Dom) do
      if (Data(i+1) < Data(i)) then
        halt("InsertionSort did not sort properly: ", Data);
}

def QuickSort(Data: [?Dom], doublecheck = false) where Dom.rank == 1 {
  // grab obvious indices
  const lo = Dom.low,
        hi = Dom.high,
        mid = lo + (hi-lo+1)/2;

  // base case -- use insertion sort
  if (hi - lo < 10) {
    InsertionSort(Data);
    return;
  }

  // find pivot using median-of-3 method
  if (Data(mid) < Data(lo)) then Data(mid) <=> Data(lo);
  if (Data(hi) < Data(lo)) then Data(hi) <=> Data(lo);
  if (Data(hi) < Data(mid)) then Data(hi) <=> Data(mid);
  const pivotVal = Data(mid);
  Data(mid) = Data(hi-1);
  Data(hi-1) = pivotVal;
  // end median-of-3 partitioning

  var loptr = lo,
      hiptr = hi-1;
  while (loptr < hiptr) {
    do { loptr += 1; } while (Data(loptr) < pivotVal);
    do { hiptr -= 1; } while (pivotVal < Data(hiptr));
    if (loptr < hiptr) {
      Data(loptr) <=> Data(hiptr);
    }
  }

  Data(hi-1) = Data(loptr);
  Data(loptr) = pivotVal;

  //  cobegin {
    QuickSort(Data[lo..loptr-1]);  // could use unbounded ranges here
    QuickSort(Data[loptr+1..hi]);
    //  }

  if (doublecheck) then
    for i in [Dom) do
      if (Data(i+1) < Data(i)) then
        halt("QuickSort did not sort properly: ", Data);
}


Optimized the global array class's access function for Block1D arrays
if the elements are local.  The remaining problem is that we still go
through the single global array class that lives on locale 0.  This
change improves verification on one locale significantly.  From 0.1979
to 0.0513 on jaguar at --m=100000.  However, there is little or no
improvement on multiple locales.  Privatization seems like it might be
the key here too.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14825 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

//
// The distribution class
//
class Block1DDist : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    //    writeln("Created a Block1DDist:\n", this);
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDomain(param rank: int, type idxType, param stridable: bool) {
    return new Block1DDom(idxType=idxType, dist=this, stridable=stridable);
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    //    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real): lo.type);

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseArithmeticDomain {

  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  // CONSTRUCTORS:

  //
  // TODO: Would like to create the following constructor, but this
  //       is problematic because the class' default constructor
  //       sets dist to nil and then tries to derefernce it.  :P
  //
  /*
  def Block1DDom(type idxType, dist: Block1DDist(idxType)) {
    writeln("in my constructor");
    this.dist = dist;
    setupLocalDoms();
  }
  */
  //
  // so doing this instead:
  //
  def initialize() {
    setupLocalDoms();
  }

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    coforall locDom in locDoms do
      on locDom {
        yield locDom.myBlock - whole.low;
      }
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in follower {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    return new Block1DArr(idxType, elemType, stridable, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setupLocalDoms();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setupLocalDoms();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist(): Block1DDist(idxType) {
    return dist;
  }

  def setupLocalDoms() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, stridable, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1, stridable);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    halt("This is bogus");
    yield 2;
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1, stridable);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType, stridable);

  // CONSTRUCTORS:

  def initialize() {
    coforall localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, stridable, dom.locDoms(localeIdx));
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    const myLocArr = locArr(here.id);
    local {
      if myLocArr.locDom.myBlock.member(i) then
        return myLocArr.this(i);
    }
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: Rewrite this to reuse more of the global domain iterator
    // logic?  (e.g., can we forward the forall to the global domain
    // somehow?
    //
    coforall locDom in dom.locDoms do
      on locDom do
        yield locDom.myBlock - dom.whole.low;
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
    //
    // TODO: Would like to write this as follower += dom.low;
    //
    const followThis = follower + dom.low;

    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      writeln("Warning: doing expensive these iteration");
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      for i in followThis {
        yield this(i);
      }
    }

    // SJD: This was here before I added aligned:
    /*
    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if (myLocArr.owns(followThis)) {
      // we own all the elements we're following
      //

      //
      // TODO: Want this, but local does not currently work within
      // follower iterators:
      //
      //      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
        //      }
      //
    } else {
      writeln("Warning: doing expensive these iteration");
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      for i in followThis {
        yield this(i);
      }
    }
    */
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) var where tag == iterator.follower {
    yield myElems(0);
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }

  // INTERNAL INTERFACE:

  def owns(x) {
    //
    // TODO: When this is multidimensional need to do a reduction or something:
    //
    return locDom.myBlock.dim(1).boundsCheck(x.dim(1));
  }
}

// TODO: Convert to Steve's leader/follower scheme (implemented in compiler)

// TODO: Make multidimensional

// TODO: Make this work for a strided domain of locales; for a strided
// domain implemented using this distribution.

// TODO: Have the global class leader/follower iterators defer to the
// local class leader/followers once we're within a locale?

// TODO: Make these into an official distribution?

// TODO: implement the slicing interface?

config param debugBradsBlock1D = false;

//
// The distribution class
//
class Block1DDist : Distribution {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType = int(64);

  // STATE:

  //
  // The bounding box that defines the block distribution
  //
  const boundingBox: domain(1, idxType);

  //
  // a domain and array describing the set of target locales to which
  // the indices are mapped
  //
  const targetLocDom: domain(1);
  const targetLocs: [targetLocDom] locale;

  // LINKAGE:

  //
  // DOWN: an array of local distribution class descriptors -- set up
  // in the class constructor
  //
  const locDist: [targetLocDom] LocBlock1DDist(idxType);

  // CONSTRUCTORS:

  def Block1DDist(type idxType = int(64), bbox: domain(1, idxType),
                  targetLocales: [] locale = Locales) {
    boundingBox = bbox;
    //
    // 0-base the local capture of the targetLocDom for simplicity
    // later on
    //
    // TODO: Create a helper function to create a domain like this for
    // arbitrary dimensions (since the k-D case is a bit harder?)
    //
    targetLocDom = [0..#targetLocales.numElements];
    targetLocs = targetLocales;

    for locid in targetLocDom do
      on targetLocs(locid) do
        locDist(locid) = new LocBlock1DDist(idxType, locid, this);

    //    writeln("Created a Block1DDist:\n", this);
  }

  // DISTRIBUTION INTERFACE:

  //
  // Create a new domain over this distribution with the given index
  // set (inds) and global index type (idxType, idxType)
  //
  // TODO: What should we do if domIdxType did not match idxType?
  //
  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where domIdxType != idxType {
    compilerError("Trying to create a domain whose index type does not match the distribution's");
  }

  def newArithmeticDomain(param rank: int, type domIdxType, param stridable: bool)
        where rank != 1 {
    compilerError("Block1D only supports 1D domains currently");
  }

  //
  // INTERFACE NOTES: Should we support a form that passes in an
  // initial index set if one exists?  If not, we should rewrite the
  // global domain construction to not do anything with whole...
  //
  def newArithmeticDomain(param rank: int, type idxType, param stridable: bool) {
    return new Block1DDom(idxType=idxType, dist=this, stridable=stridable);
  }

  //
  // print out the distribution
  //
  def writeThis(x:Writer) {
    x.writeln("BradsBlock1DPar");
    x.writeln("---------------");
    x.writeln("distributes: ", boundingBox);
    x.writeln("across locales: ", targetLocs);
    x.writeln("indexed via: ", targetLocDom);
    x.writeln("resulting in: ");
    for locid in targetLocDom do
      x.writeln("  [", locid, "] ", locDist(locid));
  }

  //
  // convert an index into a locale value
  //
  def ind2loc(ind: idxType) {
    return targetLocs(ind2locInd(ind));
  }

  // INTERNAL INTERFACE:

  //
  // compute what chunk of inds is owned by a given locale -- assumes
  // it's being called on the locale in question
  //
  def getChunk(inds, locid) {
    // use domain slicing to get the intersection between what the
    // locale owns and the domain's index set
    //
    // TODO: Should this be able to be written as myChunk[inds] ???
    //
    // TODO: Does using David's detupling trick work here?
    //
    return locDist(locid).myChunk(inds.dim(1));
  }

  //
  // Determine which locale owns a particular index
  //
  // TODO: I jotted down a note during the code review asking whether
  // targetLocs.numElements and boundingbox.numIndices should be
  // captured locally, or captured in the default dom/array implementation
  // or inlined.  Not sure what that point was anymore, though.  Maybe
  // someone else can help me remember it (since it was probably someone
  // else's suggestion).
  //
  def ind2locInd(ind: idxType) {
    //    writeln("distribution = ", this);
    const ind0 = ind - boundingBox.low;
    const locInd = (ind0 * targetLocs.numElements:idxType) / boundingBox.numIndices;
    return locInd: index(targetLocDom);
  }
}

//
// A per-locale local distribution class
//
class LocBlock1DDist {

  // GENERICS:

  //
  // The distribution's index type and domain's global index type
  //
  type idxType;

  // STATE:

  //
  // This stores the piece of the global bounding box owned by
  // the locale.  Note that my original guess that we'd want
  // to use lclIdxType here is wrong since we're talking about
  // the section of the global index space owned by the locale.
  //
  const myChunk: domain(1, idxType);

  //
  // The locale owning this class
  //
  // TODO: This is only used in the writeThis() class -- can we remove it?
  //
  const loc: locale;

  // CONSTRUCTORS:

  //
  // Constructor computes what chunk of index(1) is owned by the
  // current locale
  //
  def LocBlock1DDist(type idxType,
                     _localeIdx: int, // the locale index from the target domain
                     dist: Block1DDist(idxType) // reference to glob dist
                     ) {
    const localeIdx = _localeIdx;
    loc = dist.targetLocs(localeIdx);
    //
    // TODO: Create these assertions for other local classes as well
    //
    if (loc != here) {
      halt("Creating a local distribution class on the wrong locale");
    }
    const lo = dist.boundingBox.low;
    const hi = dist.boundingBox.high;
    const numelems = hi - lo + 1;
    const numlocs = dist.targetLocDom.numIndices;
    const blo = if (localeIdx == 0) then min(idxType)
                else procToData((numelems: real * localeIdx) / numlocs, lo);
    const bhi = if (localeIdx == numlocs - 1) then max(idxType)
                else procToData((numelems: real * (localeIdx+1)) / numlocs, lo) - 1;
    myChunk = [blo..bhi];
    if debugBradsBlock1D then
      writeln(this);
  }

  // INTERNAL INTERFACE:

  //
  // a helper function for mapping processors to indices
  //
  def procToData(x, lo)
    return (lo + (x: lo.type) + (x:real != x:int:real): lo.type);

  //
  // print out the local distribution class
  //
  def writeThis(x:Writer) {
    x.write("locale ", loc.id, " owns chunk: ", myChunk);
  }
}

//
// The global domain class
//
class Block1DDom: BaseArithmeticDomain {

  // GENERICS:

  //
  // The index types of the domain
  //
  type idxType;

  //
  // The rank of the domain
  //
  // TODO: remove the default value; make this work for rank != 1
  //
  param rank = 1;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a pointer to the parent distribution
  //
  const dist: Block1DDist(idxType);

  //
  // DOWN: an array of local domain class descriptors -- set up in
  // initialize() below
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locDoms: [dist.targetLocDom] LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // a domain describing the complete domain
  //
  const whole: domain(1, idxType);

  // CONSTRUCTORS:

  //
  // TODO: Would like to create the following constructor, but this
  //       is problematic because the class' default constructor
  //       sets dist to nil and then tries to derefernce it.  :P
  //
  /*
  def Block1DDom(type idxType, dist: Block1DDist(idxType)) {
    writeln("in my constructor");
    this.dist = dist;
    setupLocalDoms();
  }
  */
  //
  // so doing this instead:
  //
  def initialize() {
    setupLocalDoms();
  }

  // GLOBAL DOMAIN INTERFACE:

  //
  // the iterator for the domain -- currently sequential
  //
  //
  // TODO: This really should go over the elements in row-major order,
  // not the block orders.
  //
  def these() {
    for blk in locDoms do
      // TODO: Would want to do something like:
      // on blk do
      // But can't currently have yields in on clauses
        for ind in blk do
          yield ind;
  }

  //
  // this is the parallel iterator for the global domain, following
  // a variation on Steve and David's proposals -- I've split the
  // single iterator into two iterators, distinguished by where
  // clauses on their parameter values.  This permits each to only
  // take the parameters it cares about; permits the leader to be
  // defined in an inlineable way; and permits the follower for an
  // array iterator to be a var iterator
  //
  // I've named these methods newThese() for the time being
  // to avoid conflicting with the above which is currently targeted
  // by the compiler.  My current assumption is that we would want to
  // overload these() to serve this purpose in the final language
  // definition.
  //
  // TODO: Steve's question: how would a leader containing an on
  // clause interact with a loop that used an on clause explicitly
  // within its body?  How would it be done efficiently?
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: This currently only results in a single level of
    // per-locale parallelism -- no per-core parallelism; maybe
    // approach this by deferring to the local leader/follower
    // iterators at some point?
    //
    // TODO: Really want the parallelism across the target locales
    // to be expressed more independently of the distribution by
    // pushing it into the leader iterator.  Need to get an on clause
    // into here somehow.  That is:
    //
    //   coforall locDom in locDoms do
    //     on locDom do
    //       yield locDom.myBlock - whole.low;
    //
    // TODO: And really want to split the inter- and intra-locale
    // parallelism into two separate stages for communication
    // optimization and the like
    //
    // TODO: Abstract this subtraction of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    coforall locDom in locDoms do
      on locDom {
        yield locDom.myBlock - whole.low;
      }
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    //
    // TODO: Abstract this addition of low into a function?
    // Note relationship between this operation and the
    // order/position functions -- any chance for creating similar
    // support? (esp. given how frequent this seems likely to be?)
    //
    // TODO: Is there some clever way to invoke the leader/follower
    // iterator on the local blocks in here such that the per-core
    // parallelism is expressed at that level?  Seems like a nice
    // natural composition and might help with my fears about how
    // stencil communication will be done on a per-locale basis.
    for i in follower {
      yield i + whole.low;
    }
  }

  //
  // the print method for the domain
  //
  def writeThis(x:Writer) {
    x.write(whole);
  }

  //
  // how to allocate a new array over this domain
  //
  def buildArray(type elemType) {
    return new Block1DArr(idxType, elemType, stridable, this);
  }

  //
  // queries for the number of indices, low, and high bounds
  //
  def numIndices {
    return whole.numIndices;
  }

  def low {
    return whole.low;
  }

  def high {
    return whole.high;
  }

  //
  // INTERFACE NOTES: Could we make setIndices() for an arithmetic
  // domain take a domain rather than something else?
  //
  def setIndices(x: domain) {
    if x.rank != 1 then
      compilerError("rank mismatch in domain assignment");
    if x._value.idxType != idxType then
      compilerError("index type mismatch in domain assignment");
    whole = x;
    setupLocalDoms();
  }

  def setIndices(x) {
    if x.size != 1 then
      compilerError("rank mismatch in domain assignment");
    if x(1).eltType != idxType then
      compilerError("index type mismatch in domain assignment");
    //
    // TODO: This seems weird:
    //
    whole = x(1);
    setupLocalDoms();
  }

  def getIndices() {
    return whole;
  }

  //
  // INTERNAL INTERFACE
  //
  def getDist(): Block1DDist(idxType) {
    return dist;
  }

  def setupLocalDoms() {
    for localeIdx in dist.targetLocDom do
      on dist.targetLocs(localeIdx) do
        if (locDoms(localeIdx) == nil) then
          locDoms(localeIdx) = new LocBlock1DDom(idxType, stridable, this,
                                                 dist.getChunk(whole, localeIdx));
        else
          locDoms(localeIdx).myBlock = dist.getChunk(whole, localeIdx);
    if debugBradsBlock1D then
      [loc in dist.targetLocDom] writeln(loc, " owns ", locDoms(loc));

  }
}

//
// the local domain class
//
class LocBlock1DDom {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // UP: a reference to the parent global domain class
  //
  const wholeDom: Block1DDom(idxType, 1, stridable);

  // STATE:

  //
  // a local domain describing the indices owned by this locale
  //
  // TODO: Steve would like to see the strided parameter pushed into
  // the global and local domain classes.
  //
  // NOTE: I used to use a local index type for this, but that would
  // require a glbIdxType offset in order to get from the global
  // indices back to the local index type.
  //
  var myBlock: domain(1, idxType);

  // LOCAL DOMAIN INTERFACE:

  //
  // iterator over this locale's indices
  //
  def these() {
    // May want to do something like:
    // on this do
    // But can't currently have yields in on clauses
    for ind in myBlock do
      yield ind;
  }

  //
  // this is the parallel iterator for the local domain, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) where tag == iterator.follower {
    halt("This is bogus");
    yield 2;
  }

  //
  // how to write out this locale's indices
  //
  def writeThis(x:Writer) {
    x.write(myBlock);
  }

  // INTERNAL INTERFACE:

  //
  // queries for this locale's number of indices, low, and high bounds
  //
  // TODO: I believe these are only used by the random number generator
  // in stream -- will they always be required once that is rewritten?
  //
  def numIndices {
    return myBlock.numIndices;
  }

  def low {
    return myBlock.low;
  }

  def high {
    return myBlock.high;
  }
}

//
// the global array class
//
class Block1DArr: BaseArray {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the array's element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: the global domain descriptor for this array
  //
  var dom: Block1DDom(idxType, 1, stridable);

  //
  // DOWN: an array of local array classes
  //
  // TODO: would like this to be const and initialize in-place,
  // removing the initialize method; would want to be able to use
  // an on-clause at the expression list to make this work.
  // Otherwise, would have to move the allocation into a function
  // just to get it at the statement level.
  //
  var locArr: [dom.dist.targetLocDom] LocBlock1DArr(idxType, eltType, stridable);

  // CONSTRUCTORS:

  def initialize() {
    coforall localeIdx in dom.dist.targetLocDom do
      on dom.dist.targetLocs(localeIdx) do
        locArr(localeIdx) = new LocBlock1DArr(idxType, eltType, stridable, dom.locDoms(localeIdx));
  }

  // GLOBAL ARRAY INTERFACE:

  //
  // the global accessor for the array
  //
  // TODO: Do we need a global bounds check here or in ind2locind?
  //
  def this(i: idxType) var {
    return locArr(dom.dist.ind2locInd(i))(i);
  }

  //
  // the iterator over the array's elements, currently sequential
  //
  def these() var {
    for loc in dom.dist.targetLocDom {
      // TODO: May want to do something like:
      // on this do
      // But can't currently have yields in on clauses
      for elem in locArr(loc) {
        yield elem;
      }
    }
  }

  //
  // this is the parallel iterator for the global array, see the
  // example for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    //
    // TODO: Rewrite this to reuse more of the global domain iterator
    // logic?  (e.g., can we forward the forall to the global domain
    // somehow?
    //
    coforall locDom in dom.locDoms do
      on locDom do
        yield locDom.myBlock - dom.whole.low;
  }

  def supportsAlignedFollower() param return true;

  def these(param tag: iterator, follower, param aligned: bool = false) var where tag == iterator.follower {
    //
    // TODO: Would like to write this as follower += dom.low;
    //
    const followThis = follower + dom.low;

    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if aligned {
      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
      }
    } else {
      writeln("Warning: doing expensive these iteration");
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      for i in followThis {
        yield this(i);
      }
    }

    // SJD: This was here before I added aligned:
    /*
    //
    // TODO: The following is a buggy hack that will only work when we're
    // distributing across the entire Locales array.  I still think the
    // locArr/locDoms arrays should be associative over locale values.
    //
    const myLocArr = locArr(here.id);
    if (myLocArr.owns(followThis)) {
      // we own all the elements we're following
      //

      //
      // TODO: Want this, but local does not currently work within
      // follower iterators:
      //
      //      local {
        for i in followThis {
          yield myLocArr.this(i);
        }
        //      }
      //
    } else {
      writeln("Warning: doing expensive these iteration");
      //
      // we don't own all the elements we're following
      //
      // TODO: could do something smarter to only bring the non-local
      // elements over.
      for i in followThis {
        yield this(i);
      }
    }
    */
  }

  //
  // how to print out the whole array, sequentially
  //
  def writeThis(x: Writer) {

    var first = true;
    for loc in dom.dist.targetLocDom {
      // May want to do something like the following:
      //      on loc {
      // but it causes deadlock -- see writeThisUsingOn.chpl
        if (locArr(loc).numElements >= 1) {
          if (first) {
            first = false;
          } else {
            x.write(" ");
          }
          x.write(locArr(loc));
        }
        //    }
      stdout.flush();
    }
  }

  //
  // a query for the number of elements in the array
  //
  def numElements {
    return dom.numIndices;
  }
}

//
// the local array class
//
class LocBlock1DArr {

  // GENERICS:

  //
  // The index type of the domain
  //
  type idxType;

  //
  // the element type
  //
  type eltType;

  param stridable: bool; // sjd: added stridable parameter

  // LINKAGE:

  //
  // LEFT: a reference to the local domain class for this array and locale
  //
  const locDom: LocBlock1DDom(idxType, stridable);

  // STATE:

  //
  // the block of local array data
  //
  var myElems: [locDom.myBlock] eltType;

  // LOCAL ARRAY INTERFACE:

  //
  // the accessor for the local array -- assumes the index is local
  //
  def this(i: idxType) var {
    return myElems(i);
  }

  //
  // iterator over the elements owned by this locale
  //
  def these() var {
    for elem in myElems {
      yield elem;
    }
  }

  //
  // this is the parallel iterator for the local array, see global
  // domain parallel iterators for general notes on the approach
  //
  def these(param tag: iterator) where tag == iterator.leader {
    halt("This is bogus");
    yield [1..100];
  }

  def these(param tag: iterator, follower) var where tag == iterator.follower {
    yield myElems(0);
  }

  //
  // prints out this locale's piece of the array
  //
  def writeThis(x: Writer) {
    // May want to do something like the following:
    //      on loc {
    // but it causes deadlock -- see writeThisUsingOn.chpl
    x.write(myElems);
  }

  //
  // query for the number of local array elements
  //
  def numElements {
    return myElems.numElements;
  }

  // INTERNAL INTERFACE:

  def owns(x) {
    //
    // TODO: When this is multidimensional need to do a reduction or something:
    //
    return locDom.myBlock.dim(1).boundsCheck(x.dim(1));
  }
}


Fix rename_extern_record bug by using a cstring for the external name

extern c"struct stat" record chpl_stat {
  var st_size: off_t;
}

proc getFileSize(path:c_string) : int {
  extern proc stat(x: c_string, ref chpl_stat): c_int;
  var buf: chpl_stat;

  if (stat(path, buf) == 0) {
    return buf.st_size;
  }
  return -1;
}

writeln(getFileSize("./rename_extern_record.good"));

extern "struct stat" record chpl_stat {
  var st_size: off_t;
}

proc getFileSize(path:c_string) : int {
  extern proc stat(x: c_string, ref chpl_stat): c_int;
  var buf: chpl_stat;

  if (stat(path, buf) == 0) {
    return buf.st_size;
  }
  return -1;
}

writeln(getFileSize("./rename_extern_record.good"));