update srve

update srve

// use Logging, Memory, LibEv, IO, Random, SearchIndex, SyntheticDataIndexer, Time;
use Logging, LibEv, IO, SearchIndex, Time;

// ****
// NOTE: this is very much in progress
// ****

// TODO: port to pure chapel
extern proc initialize_socket(port: c_int): c_int;

// trampolines
extern var c_accept_cb: opaque;

extern proc send(sockfd:c_int, buffer: c_string, len: size_t, flags: c_int);

config const port: c_int = 3033;
config const post_load_test: bool = false;
config const load_from_partitions: bool = true;

// TODO: is the fd unique enough to bind a multi-handle processing context to?
export proc handle_received_data(fd: c_int, tcp_buffer: c_string, read: size_t, buffer_size: size_t) {
  // simulate processing the query
  var buffer = new InstructionBuffer(32);
  var writer = new InstructionWriter(buffer);
  buffer.clear();
  writer.write_push();
  writer.write_term(2);

  forall result in query(new Query(buffer)) {
    writeln(result);
  }

  send(fd, tcp_buffer, read, 0);
}

proc main() {

  info("initializing index");
  initPartitions();

  // POPULATE THE INDEX

  var t: Timer;
  t.start();

  startBatchIndexers();
  waitForBatchIndexers();

  t.stop();
  timing("indexing complete in ",t.elapsed(TimeUnits.microseconds), " microseconds");

  info("initializing event loop...");

  info("creating socket...");
  var sd: ev_fd = initialize_socket(port);
  debug("socket id = ", sd);
  if (sd == -1) {
    error("socket error");
    return -1;
  }

  // port c_accept_cb, c_read_cb
  var w_accept = new ev_io();
  ev_io_init(w_accept, c_accept_cb, sd, EV_READ);
  ev_io_start(EV_DEFAULT, w_accept);

  while (1) {
    ev_loop_fn(EV_DEFAULT, 0); // EVLOOP_NONBLOCK
    // chpl_task_yield();
  }

  return 0;
}

use Logging, Memory, LibEv, IO, Random, SearchIndex, SyntheticDataIndexer, Time;

// ****
// NOTE: this is very much in progress
// ****

// TODO: port to pure chapel
extern proc initialize_socket(port: c_int): c_int;

// trampolines
extern var c_accept_cb: opaque;

extern proc send(sockfd:c_int, buffer: c_string, len: size_t, flags: c_int);

config const port: c_int = 3033;
config const post_load_test: bool = false;
config const load_from_partitions: bool = true;

export proc handle_received_data(fd: c_int, tcp_buffer: c_string, read: size_t, buffer_size: size_t) {
  // is the fd unique enough to bind a multi-handle processing context to?

  // simulate processing the query
  var buffer = new InstructionBuffer(32);
  var writer = new InstructionWriter(buffer);
  buffer.clear();
  writer.write_push();
  writer.write_term(2);

  forall result in query(new Query(buffer)) {
    writeln(result);
  }

  send(fd, tcp_buffer, read, 0);
}

proc main() {

  info("creating socket...");
  var sd: ev_fd = initialize_socket(port);
  debug("socket id = ", sd);
  if (sd == -1) {
   error("socket error");
   return -1;
  }

  info("initializing index");
  initPartitions();

  // POPULATE THE INDEX

  var t: Timer;
  t.start();

  startBatchIndexers();
  waitForBatchIndexers();

  t.stop();
  timing("indexing complete in ",t.elapsed(TimeUnits.microseconds), " microseconds");

  info("initializing event loop...");

  // port c_accept_cb, c_read_cb
  var w_accept: ev_io = new ev_io();
  ev_io_init(w_accept, c_accept_cb, sd, EV_READ);
  ev_io_start(EV_DEFAULT, w_accept);

  while (1) {
   ev_loop_fn(EV_DEFAULT, 0);
  }

  return 0;
}


Discovered the cause of the time discrepancy between Chapel loops and C loops
was because the variables the loops were accessing were in module code as
opposed to method code.  Moving that section inside of a main method to get a
more fair comparison.

Additionally, it seems that certain basic operators will affect the timing
difference between module and method level accesses.  Another test exercising
this is forthcoming.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@23297 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use Time;

config const numIterations = 1000000;
config const numTrials = 10000;
config const verbose = false;

extern proc c_trial(): int;

proc main() {
  var t1, t2: Timer;
  var res1, res2 = 0;
  t1.start();
  for i in 1..#numTrials {
    for j in 1..#numIterations {
      if j % 2 == 0 then
        res1 = res1 + 3;
      else
        res1 = res1 / 2;
    }
  }
  t1.stop();
  t2.start();
  var i = 0;
  while (i < numTrials) {
    var j = 0;
    while (j < numIterations) {
      if j % 2 == 1 then
        res2 = res2 + 3;
      else
        res2 = res2 / 2;
      j += 1;
    }
    i += 1;
  }
  t2.stop();
  if (res1 != res2) {
    writeln("These results should have matched, got ", res1, " and ", res2);
  } else {
    writeln("Chapel verification successful");
  }

  if verbose {
    writeln("For loop underwent ", numIterations, " iterations ", numTrials,
            " times in ", t1.elapsed(TimeUnits.milliseconds)/1000, " seconds");
    writeln("While loop underwent ", numIterations, " iterations ", numTrials,
            " times in ", t2.elapsed(TimeUnits.milliseconds)/1000, " seconds");
    var res3 = c_trial();
    if (res1 != res3) {
      writeln("Chapel results did not match C results");
    }
  }
}

use Time;

config const numIterations = 1000000;
config const numTrials = 10000;
config const verbose = false;

var t1, t2: Timer;
var res1, res2 = 0;

extern proc c_trial(): int;

t1.start();
for i in 1..#numTrials {
  for j in 1..#numIterations {
    if j % 2 == 0 then
      res1 = res1 + 3;
    else
      res1 = res1 / 2;
  }
}
t1.stop();
t2.start();
var i = 0;
while (i < numTrials) {
  var j = 0;
  while (j < numIterations) {
    if j % 2 == 1 then
      res2 = res2 + 3;
    else
      res2 = res2 / 2;
    j += 1;
  }
  i += 1;
}
t2.stop();
if (res1 != res2) {
  writeln("These results should have matched, got ", res1, " and ", res2);
} else {
  writeln("Chapel verification successful");
}

if verbose {
  writeln("For loop underwent ", numIterations, " iterations ", numTrials,
          " times in ", t1.elapsed(TimeUnits.milliseconds)/1000, " seconds");
  writeln("While loop underwent ", numIterations, " iterations ", numTrials,
          " times in ", t2.elapsed(TimeUnits.milliseconds)/1000, " seconds");
  var res3 = c_trial();
  if (res1 != res3) {
    writeln("Chapel results did not match C results");
  }
}


Removing square brackets from loops that could iterate over ranges

Javalearner's email that came in today made me wonder (as I often do)
"What are we doing that causes people to write their for loops as:

      for i in [1..n]

rather than

      for i in 1..n

I checked the release/examples directory and found that the
taskParallel case did this, so simplified it.  This is a new test, so
doesn't explain earlier users who've done this, but could explain
where javalearner got it form.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@17135 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// This example illustrates Chapel's parallel tasking features,
// namely the begin, cobegin, and coforall statements.

config const n = 10;

writeln("1: ### The begin statement ###");

// The begin statement spawns a thread of execution that is independent
// of the current (main) thread of execution.
begin writeln("1: output from spawned task");

// The main thread of execution continues on to the next statement.
// There is no guarantee as to which statment will execute first.
writeln("1: output from main task");

writeln("2: ### The cobegin statement ###");

// For more structured behavior, the cobegin statement can be used to
// spawn a block of tasks, one for each statement.  Control continues
// after the cobegin block, but only after all the tasks within the
// cobegin block have completed.
cobegin {
  writeln("2: output from spawned task 1");
  writeln("2: output from spawned task 2");
}

// The output from within the cobegin statement will always precede the
// following output from the main thread of execution.
writeln("2: output from main task");

writeln("3: ### The cobegin statement with nested begin statements ###");

// If any begin statements are used within a cobegin statement,
// the thread of execution does not a wait for those begin statements
// to complete.
cobegin {
  begin writeln("3: output from spawned task 1");
  begin writeln("3: output from spawned task 2");
}

// The order of the output is again undefined because the begin
// statements in the above cobegin statment are not guaranteed to
// have been executed before control reaches the following statement.
writeln("3: output from main task");

writeln("4: ### The coforall statement ###");

// Another more structured form of task parallelism is the coforall
// statement.  The coforall statement is a loop variant of the cobegin
// statement where each iteration of the loop is a separate task.
// Similar to the cobegin statement, every iteration of the coforall
// loop is a separate task and the main thread of execution does not
// continue until every iteration is complete.
coforall i in 1..n {
  writeln("4: output from spawned task 1 (iteration ", i, ")");
  writeln("4: output from spawned task 2 (iteration ", i, ")");
}

// While the order of output within an iteration is deterministic ("1"
// executes before "2"), the order of output relative to other
// iterations is not defined.  As with the cobegin statement, the output
// from within the coforall statement will always precede the following
// output.
writeln("4: output from main task");

writeln("5: ### The coforall statement with nested begin statements ###");
// As with the cobegin statement, any begin statements spawned within
// a coforall loop are not guaranteed to be complete before the main
// thread of execution continues.
coforall i in 1..n {
  begin writeln("5: output from spawned task 1 (iteration ", i, ")");
  begin writeln("5: output from spawned task 2 (iteration ", i, ")");
}
// The order of output is undefined.
writeln("5: output from main task");

// This example illustrates Chapel's parallel tasking features,
// namely the begin, cobegin, and coforall statements.

config const n = 10;

writeln("1: ### The begin statement ###");

// The begin statement spawns a thread of execution that is independent
// of the current (main) thread of execution.
begin writeln("1: output from spawned task");

// The main thread of execution continues on to the next statement.
// There is no guarantee as to which statment will execute first.
writeln("1: output from main task");

writeln("2: ### The cobegin statement ###");

// For more structured behavior, the cobegin statement can be used to
// spawn a block of tasks, one for each statement.  Control continues
// after the cobegin block, but only after all the tasks within the
// cobegin block have completed.
cobegin {
  writeln("2: output from spawned task 1");
  writeln("2: output from spawned task 2");
}

// The output from within the cobegin statement will always precede the
// following output from the main thread of execution.
writeln("2: output from main task");

writeln("3: ### The cobegin statement with nested begin statements ###");

// If any begin statements are used within a cobegin statement,
// the thread of execution does not a wait for those begin statements
// to complete.
cobegin {
  begin writeln("3: output from spawned task 1");
  begin writeln("3: output from spawned task 2");
}

// The order of the output is again undefined because the begin
// statements in the above cobegin statment are not guaranteed to
// have been executed before control reaches the following statement.
writeln("3: output from main task");

writeln("4: ### The coforall statement ###");

// Another more structured form of task parallelism is the coforall
// statement.  The coforall statement is a loop variant of the cobegin
// statement where each iteration of the loop is a separate task.
// Similar to the cobegin statement, every iteration of the coforall
// loop is a separate task and the main thread of execution does not
// continue until every iteration is complete.
coforall i in [1..n] {
  writeln("4: output from spawned task 1 (iteration ", i, ")");
  writeln("4: output from spawned task 2 (iteration ", i, ")");
}

// While the order of output within an iteration is deterministic ("1"
// executes before "2"), the order of output relative to other
// iterations is not defined.  As with the cobegin statement, the output
// from within the coforall statement will always precede the following
// output.
writeln("4: output from main task");

writeln("5: ### The coforall statement with nested begin statements ###");
// As with the cobegin statement, any begin statements spawned within
// a coforall loop are not guaranteed to be complete before the main
// thread of execution continues.
coforall i in [1..n] {
  begin writeln("5: output from spawned task 1 (iteration ", i, ")");
  begin writeln("5: output from spawned task 2 (iteration ", i, ")");
}
// The order of output is undefined.
writeln("5: output from main task");


Corrected gaxpy

Corrected gaxpy

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@12227 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/** MRA -- MultiResolution Analysis Module

    Multiresolution representation of 1-d functions using a multiwavelet
    basis (similar to discontinuous spectral element with a hierarchal
    decomposition).

    Mad Chapel Author: James Dinan <dinan@cray.com>
                 Date: August, 2007

    This code is an adaptation of 1d Madness, originally written by Robert
    Harrison <harrisonrj@ornl.gov>, et al.
 */

use Math;
use Tensor;
use FTree;
use Quadrature;
use TwoScale;
use AnalyticFcn;
use StringConvert;  // FIXME: workaround for formatting reals

config const verbose = false;
config const debug   = false;

class Function {
    const k             = 5;    // use first k Legendre polynomials as the basis in each box
    const thresh        = 1e-5; // truncation threshold for small wavelet coefficients
    const f: AFcn       = nil;  // analytic f(x) to project into the numerical represntation
    const initial_level = 2;    // initial level of refinement
    const max_level     = 30;   // maximum level of refinement mostly as a sanity check
    var   autorefine    = true; // automatically refine during multiplication
    var   compressed    = false;// keep track of what basis we are in

    // Sum and Difference coefficients
    var   s = FTree(order=k);
    var   d = FTree(order=k);

    // FIXME: Ideally all of these matrices should be const as well but they
    //        can't be presently since they must be assigned in initialize()

    // Two-Scale relationship matrices
    const hgDom = [0..2*k-1, 0..2*k-1];
    const hg    : [hgDom] real = hg_getCoeffs(k); // FIXME: hg  =  hg_getCoeffs(k);
    var   hgT   : [hgDom] real;                   //        hgT => transpose(hg);

    // Quadrature coefficients
    const quadDom   = [0..k-1];
    var   quad_x    : [quadDom] real; // points
    var   quad_w    : [quadDom] real; // weights

    const quad_phiDom = [0..k-1, 0..k-1];
    var   quad_phi    : [quad_phiDom] real; // phi[point,i]
    var   quad_phiT   : [quad_phiDom] real; // phi[point,i] transpose
    var   quad_phiw   : [quad_phiDom] real; // phi[point,i]*weight[point]

    // blocks of the block tridiagonal derivative operator
    const dcDom = [0..k-1, 0..k-1];
    var   rm    : [dcDom] real;
    var   r0    : [dcDom] real;
    var   rp    : [dcDom] real;

    def initialize() {
        if debug then writeln("Creating Function: k=", k, " thresh=", thresh);

        if debug then writeln("  initializing two-scale relation coefficients");
        hg  = hg_getCoeffs(k);
        transposeCopy(hgT, hg);

        if debug then writeln("  initializing quadrature coefficients");
        init_quadrature(k);

        // blocks of the block tridiagonal derivative operator
        if debug then writeln("  initializing tridiagonal derivative operator");
        make_dc_periodic();

        // initial refinement of analytic function f(x)
        if f != nil {
            if debug then writeln("  performing initial refinement of f(x)");
            for l in 0..2**initial_level-1 do
                refine(initial_level, l);
        }

        if debug then writeln("done.");
    }

    /** Initialize the quadrature coefficient matricies.
     */
    def init_quadrature(order: int) {
        quad_x   = gl_getPoints(k);
        quad_w   = gl_getWeights(k);

        for i in quad_phiDom.dim(1) {
            var p = phi(quad_x[i], k);
            quad_phi [i, ..] = p;
            quad_phiw[i, ..] = quad_w[i] * p;
            quad_phiT[.., i] = p;
        }
    }

    /** Return the level-0 blocks rm, r0, rp of the central
        difference derivative operator with periodic boundary
        conditions on either side.
     */
    def make_dc_periodic() {
        var iphase = 1.0;
        for i in dcDom.dim(1) {
            var jphase = 1.0;

            for j in dcDom.dim(2) {
                var gammaij = sqrt((2*i+1) * (2*j+1));
                var Kij = if (i-j) > 0 && ((i-j) % 2) == 1 then
                        2.0 else 0.0;
                r0[i,j] = 0.5*(1.0 - iphase*jphase - 2.0*Kij)*gammaij;
                rm[i,j] = 0.5*jphase*gammaij;
                rp[i,j] =-0.5*iphase*gammaij;
                jphase = -jphase;
            }
            iphase = -iphase;
        }
    }

    /** Return a deep copy of this Function
     */
    def copy() {
        return Function(k=k, thresh=thresh, f=f, initial_level=initial_level,
                max_level=max_level, autorefine=autorefine, compressed=compressed,
                s=s.copy(), d=d.copy());
    }

    /** Return a copy of this function's skeleton
     */
    def skeletonCopy() {
        // Omit: f, compressed, s, d
        return Function(k=k, thresh=thresh, initial_level=initial_level,
                max_level=max_level, autorefine=autorefine);
    }

    /** s[n, l] = integral(phi[n][l](x) * f(x))
        for box (n,l) project f(x) using quadrature rule
        into scaling function basis
     */
    def project(n: int, l: int) {
        var s     : [quadDom] real;
        var h     = 0.5 ** n;
        var scale = sqrt(h);

        for mu in quad_phiDom.dim(1) {
            var x  = (l + quad_x[mu]) * h;
            var fx = f(x);
            for i in quad_phiDom.dim(2) do
                s[i] += scale * fx * quad_phiw[mu, i];
        }

        return s;
    }

    /** Refine numerical representation of f(x) to desired tolerance
     */
    def refine(n, l) {
        // project f(x) at next level
        var s0 = project(n+1, 2*l);
        var s1 = project(n+1, 2*l+1);
        var sc : [0..2*k-1] real;

        sc[0..k-1]   = s0;
        sc[k..2*k-1] = s1;

        // apply the two scale relationship to get difference coeff
        // in 1d this is O(k^2) flops
        var dc = sc*hgT;

        // check to see if within tolerance
        // normf() is Frobenius norm == 2-norm for vectors
        var nf = normf(dc[k..2*k-1]);
        if((nf < thresh) || (n >= (max_level-1))) {
            s[n+1, 2*l]   = s0;
            s[n+1, 2*l+1] = s1;
        }
        else {
            // these recursive calls on sub-trees can go in parallel
            // if the HashMap is syncronized
            refine(n+1, 2*l  );
            refine(n+1, 2*l+1);
        }
    }

    /** Evaluate numerical representation of f at x.  Performed in scaling
        function basis only.
     */
    def this(x) {
        if compressed then reconstruct();
        return evaluate(0, 0, x);
    }

    /** eval f(x) using adaptively refined numerical representation of f(x)
        answer should be within tolerance of the analytical f(x)

        Descend tree looking for box (n,l) with scaling function
        coefficients containing the point x.
     */
    def evaluate(in n=0, in l=0, in x): real {
        if s.has_coeffs(n, l) {
            var p = phi(x, k);
            return inner(s[n, l], p)*sqrt(2.0**n);

        } else {
            (n, l, x) = (n+1, 2*l, 2*x);
            if x >= 1 then (l, x) = (l+1, x-1);
            return evaluate(n, l, x);
        }
    }

    /** change from scaling function basis to multi-wavelet basis (s -> d)
        tree is filled out with s[0][0] and d
        n is level in tree
        l is box index
     */
    def compress(n=0, l=0) {
        if compressed then return;

        // sub-trees can be done in parallel
        if !s.has_coeffs(n+1,   2*l) then compress(n+1,   2*l);
        if !s.has_coeffs(n+1, 2*l+1) then compress(n+1, 2*l+1);

        var sc: [0..2*k-1] real;
        sc[0..k-1]   = s[n+1,   2*l];
        sc[k..2*k-1] = s[n+1, 2*l+1];

        // apply the two scale relationship to get difference coeff
        // in 1d this is O(k^2) flops (in 3d this is O(k^4) flops)
        var dc = sc*hgT;

        s[n, l] = dc[0..k-1];
        d[n, l] = dc[k..2*k-1];

        s.remove(n+1,   2*l);
        s.remove(n+1, 2*l+1);

        if n==0 then compressed = true;
    }

    /** change from multi-wavelet basis to scaling function basis (d -> s)
        tree just has s at leaves
        n is level in tree
        l is box index
     */
    def reconstruct(n=0, l=0) {
        if !compressed then return;

        if d.has_coeffs(n, l) {
            var dc: [0..2*k-1] real;
            dc[0..k-1]   = s[n, l];
            dc[k..2*k-1] = d[n, l];
            d.remove(n, l);
            s.remove(n, l);

            // apply the two scale relationship to get difference coeff
            // in 1d this is O(k^2) flops (in 3d this is O(k^4) flops)
            var sc = dc*hg;

            s[n+1, 2*l  ] = sc[0..k-1];
            s[n+1, 2*l+1] = sc[k..2*k-1];

            // sub-trees can be done in parallel
            reconstruct(n+1, 2*l);
            reconstruct(n+1, 2*l+1);
        }

        if n == 0 then compressed = false;
    }

    /** In s are scaling coefficients for box n,l ... apply twoscale to generate
        the corresponding coefficients on level n+1 and insert the results into
        the tree of scaling function coefficients.
     */
    def recur_down(n, l) {
        if debug then writeln(" + recur_down(", n, ", ", l, ")");

        var sc : [0..2*k-1] real;
        sc[0..k-1] = s[n, l];

        var new_sc = sc*hg;
        s[n+1, 2*l  ] = new_sc[0..k-1];
        s[n+1, 2*l+1] = new_sc[k..2*k-1];
    }

    /** If the scaling coefficeints in box n,l exist, return them.
        (allow here for zero boundary conditions for boxes just off
        the ends of the domain)

        Else recur up to the next level looking for a parent.  If a
        parent exists, use two scale to recur those coefficients down
        to make n,l.  Note that this modifies the tree in place and
        you should eventually call sclean to tidy up when finished.

        Else, return None (corresponding child boxes exist at a finer scale)
     */
    def get_coeffs(_n, _l) {
        if debug then writeln(" @ get_coeffs(", _n, ", ", _l, ")");
        var n = _n, l = _l;

        // Walk up the tree to find scaling coeffs
        while n >= 0 {
            if l < 0 || l >= 2**n {
                var coeffs : [0..k-1] real;
                return coeffs;
            }

            if s.has_coeffs(n, l) then break;

            n -= 1;
            l  = l/2;
        }

        // Didn't find coeffs, they must be below (_n, _l)
        if n < 0 then return None;

        // Recur down to (_n, _l)
        for i in n.._n-1 {
            l = _l/(2**(_n-i));
            recur_down(i, l);
        }

        return s[_n, _l];
    }

    /** Differentiation (also inner and mul) may leave scaling function
        coefficients below their original level.  Recur down to the
        locally first box with scaling function coefficients.
        Delete all children below there.
     */
    def sclean(n=0, l=0, in cleaning=false) {
        if cleaning { //then
            if debug then writeln(" + sclean deleting (", n, ", ", l, ")");
            s.remove(n, l);
        } else
            cleaning = s.has_coeffs(n, l);

        // Sub trees can run in parallel
        if n < max_level {
            if !cleaning || s.has_coeffs(n+1, 2*l) then
                sclean(n+1, 2*l, cleaning);
            if !cleaning || s.has_coeffs(n+1, 2*l+1) then
                sclean(n+1, 2*l+1, cleaning);
        }
    }

    /** Differentiate the function, which corresponds to application
        of a block triadiagonal matrix.  For an adaptively refined
        target function we may need to refine boxes down until three
        boxes exist in the same scale.
     */
    def diff() {
        def diffHelper(n = 0, l = 0, result) {
            if debug then writeln(" * diff(", n, ", ", l, ")");
            if !s.has_coeffs(n, l) {
                // Sub trees can run in parallel
                // Run down tree until we hit scaling function coefficients
                diffHelper(n+1, 2*l  , result);
                diffHelper(n+1, 2*l+1, result);
            } else {
                // These can also go in parallel since may involve
                // recurring up & down the tree.
                if debug then writeln(" * diff: coeffs at (", n, ", ", l, ")");
                var sm = get_coeffs(n, l-1);
                var sp = get_coeffs(n, l+1);
                var s0 = s[n, l];

                // We have s0, check if we found sm and sp at this level
                if !isNone(sm) && !isNone(sp) {
                    var r = rp*sm + r0*s0 + rm*sp;
                    result.s[n, l] = r * 2.0**n;
                } else {
                    recur_down(n, l);
                    // Sub trees can run in parallel
                    diffHelper(n+1, 2*l  , result);
                    diffHelper(n+1, 2*l+1, result);
                }
            }
        }

        if compressed then reconstruct();
        var result = skeletonCopy();

        diffHelper(0, 0, result);

        sclean();
        return result;
    }

    /** Return sqrt(integral(f(x)**2))
     */
    def norm2() {
        if compressed then reconstruct();
        writeln(s.idx_iter());
        writeln([i in s.idx_iter()] i);
        return sqrt(+ reduce [i in s] normf(i)**2);
    }

    /** Perform GAXPY in the multi-wavelet (compressed) basis
        this = alpha*this + beta*other (other is not changed).
     */
    def gaxpy(alpha, other, beta) {
        // recursive "iteration" for gaxpy
        def gaxpy_iter(n=0, l=0) {
            if d.has_coeffs(n, l) || other.d.has_coeffs(n, l) {
                if d.has_coeffs(n, l) && other.d.has_coeffs(n, l) then
                    d[n, l] = d[n, l]*alpha + other.d[n, l]*beta;

                else if !d.has_coeffs(n, l) && other.d.has_coeffs(n, l) then
                    d[n, l] = other.d[n, l] * beta;

                else /* d.has_coeffs(n, l) && !other.d.has_coeffs(n, l) */
                    d[n, l] *= alpha;

                // calls on sub-trees can go in parallel
                gaxpy_iter(n+1, 2*l);
                gaxpy_iter(n+1, 2*l+1);
            }
        }

        if !compressed then compress();
        if !other.compressed then other.compress();

        s[0, 0] = s[0, 0]*alpha + other.s[0, 0]*beta; // Do scaling coeffs
        gaxpy_iter();                                 // Do multi-wavelet coeffs

        // return this so operations can be chained
        return this;
    }

    /** Add this function to another and return the result in a new
        function.  This and other are unchanged.
     */
    def add(other) {
        return copy().gaxpy(1.0, other, 1.0);
    }

    /** Subtract this function from another and return the result in a new
        function.  This and other are unchanged.
     */
    def subtract(other) {
        return copy().gaxpy(1.0, other, -1.0);
    }

    /** Recursively multiply f1 and f2 put the result into this
     */
    def multiply(f1, f2, n=0, l=0) {
        if f1.s.has_coeffs(n, l) && f2.s.has_coeffs(n, l) {
            if autorefine && n+1 <= max_level {
                // if autorefine is set we are multiplying two polynomials
                // of order k-1 the result could be up to order 2(k-1) so
                // refine one more level to reduce the error and try to
                // keep it within the threshold

                // refine both one more level
                f1.recur_down(n, l);
                f2.recur_down(n, l);

                // scale factor for this level = sqrt((2^d)^(n+1))
                var scale_factor = sqrt(2.0**(n+1));

                // multiply f1.s[n+1][2*l] and f2.s[n+1][2*l]
                var f = f1.s[n+1, 2*l] * quad_phiT;
                var g = f2.s[n+1, 2*l] * quad_phiT;
                f *= g; // Note: Element-wise multiplication
                s[n+1, 2*l] = (f * quad_phiw) * scale_factor;

                // multiply f1.s[n+1][2*l+1] and f2.s[n+1][2*l+1]
                f  =  f1.s[n+1, 2*l+1] * quad_phiT;
                g  =  f2.s[n+1, 2*l+1] * quad_phiT;
                f *= g;
                s[n+1, 2*l+1] = (f * quad_phiw) * scale_factor;

            } else {
                // if autorefine is not set or we are at the max_level
                // live with what you get
                var f = f1.s[n, l] * quad_phiT;
                var g = f2.s[n, l] * quad_phiT;
                f *= g;

                // scale factor for this level = sqrt((2^d)^(n+1))
                s[n, l] = (f * quad_phiw) * sqrt(2.0**(n));
            }

        } else {
            if f1.s.has_coeffs[n, l] && !f2.s.has_coeffs[n, l] then
                // refine this box down to next level in f1
                f1.recur_down(n, l);
            else if !f1.s.has_coeffs[n, l] && f2.s.has_coeffs[n, l] then
                // refine this box down to next level in f2
                f2.recur_down(n, l);

            // calls on sub-trees can go in parallel
            multiply(f1, f2, n+1, 2*l);
            multiply(f1, f2, n+1, 2*l+1);
        }
    }

    /** Function Multiplication:
        For multiply, both operands need to be in the scaling function basis
        so possibly call reconstruct on one or both of them first
     */
    def multiply(other) {
        if compressed then reconstruct();
        if other.compressed then other.reconstruct();

        // Store the result in a new Function
        var result = skeletonCopy();
        result.multiply(this, other);

        sclean();
        other.sclean();

        return result;
    }

    /** Mostly for debugging, print summary of coefficients,
        optionally printing the norm of each block
     */
    def summarize() {
        writeln("\n-----------------------------------------------------");
        writeln("k=", k, " thresh=", thresh, " compressed=", compressed);
        writeln("sum coefficients:");
        for n in 0..max_level {
            var sum = 0.0, ncoeffs = 0;
            for box in s.lvl_iter(n) {
                sum     += normf(box)**2;
                ncoeffs += 1;
            }
            if ncoeffs != 0 then
                writeln("   level ", toString("%2d", n), "   #boxes=",
                        toString("%4d", ncoeffs), "  norm=", toString("%0.2e", truncate(sqrt(sum))));
        }

        writeln("difference coefficients:");
        for n in 0..max_level {
            var sum = 0.0, ncoeffs = 0;
            for box in d.lvl_iter(n) {
                sum     += normf(box)**2;
                ncoeffs += 1;
            }
            if ncoeffs != 0 then
                writeln("   level ", toString("%2d", n), "   #boxes=",
                        toString("%4d", ncoeffs), "  norm=", toString("%0.2e", truncate(sqrt(sum))));
        }

        writeln("-----------------------------------------------------\n");
    }

    /** Evaluate the analytic and numerical functions over the given interval
        and print the error.
     */
    def evalNPT(npt) {
        for i in 0..npt {
            var (fval, Fval) = (truncate(f(i/npt:real)), truncate(this(i/npt:real)));
            //writeln(" -- ", toString("%0.2f", i/npt:real), ":  F_numeric()=", toString("% 0.5e", Fval),
            //        "  f_analytic()=", toString("% 0.5e", fval), " err=", toString("% 0.5e", Fval-fval),
            //        if abs(Fval-fval) > thresh then "  > thresh" else "");
            writeln(" -- ", toString("%0.2f", i/npt:real), ":  F_numeric()=", toString("% 0.8f", Fval),
                    "  f_analytic()=", toString("% 0.8f", fval), " err=", toString("% 0.1e", Fval-fval),
                    if abs(Fval-fval) > thresh then "  > thresh" else "");
        }
    }
}

/*************************************************************************/
/* Standard operations on Function objects:                              */
/*************************************************************************/

def +(F: Function, G: Function): Function {
    return F.add(G);
}

def -(F: Function, G: Function): Function {
    return F.subtract(G);
}

def *(F: Function, G: Function): Function {
    return F.multiply(G);
}

/** MRA -- MultiResolution Analysis Module

    Multiresolution representation of 1-d functions using a multiwavelet
    basis (similar to discontinuous spectral element with a hierarchal
    decomposition).

    Mad Chapel Author: James Dinan <dinan@cray.com>
                 Date: August, 2007

    This code is an adaptation of 1d Madness, originally written by Robert
    Harrison <harrisonrj@ornl.gov>, et al.
 */

use Math;
use Tensor;
use FTree;
use Quadrature;
use TwoScale;
use AnalyticFcn;
use StringConvert;  // FIXME: workaround for formatting reals

config const verbose = false;
config const debug   = false;

class Function {
    const k             = 5;    // use first k Legendre polynomials as the basis in each box
    const thresh        = 1e-5; // truncation threshold for small wavelet coefficients
    const f: AFcn       = nil;  // analytic f(x) to project into the numerical represntation
    const initial_level = 2;    // initial level of refinement
    const max_level     = 30;   // maximum level of refinement mostly as a sanity check
    var   autorefine    = true; // automatically refine during multiplication
    var   compressed    = false;// keep track of what basis we are in

    // Sum and Difference coefficients
    var   s = FTree(order=k);
    var   d = FTree(order=k);

    // FIXME: Ideally all of these matrices should be const as well but they
    //        can't be presently since they must be assigned in initialize()

    // Two-Scale relationship matrices
    const hgDom = [0..2*k-1, 0..2*k-1];
    const hg    : [hgDom] real = hg_getCoeffs(k); // FIXME: hg  =  hg_getCoeffs(k);
    var   hgT   : [hgDom] real;                   //        hgT => transpose(hg);

    // Quadrature coefficients
    const quadDom   = [0..k-1];
    var   quad_x    : [quadDom] real; // points
    var   quad_w    : [quadDom] real; // weights

    const quad_phiDom = [0..k-1, 0..k-1];
    var   quad_phi    : [quad_phiDom] real; // phi[point,i]
    var   quad_phiT   : [quad_phiDom] real; // phi[point,i] transpose
    var   quad_phiw   : [quad_phiDom] real; // phi[point,i]*weight[point]

    // blocks of the block tridiagonal derivative operator
    const dcDom = [0..k-1, 0..k-1];
    var   rm    : [dcDom] real;
    var   r0    : [dcDom] real;
    var   rp    : [dcDom] real;

    def initialize() {
        if debug then writeln("Creating Function: k=", k, " thresh=", thresh);

        if debug then writeln("  initializing two-scale relation coefficients");
        hg  = hg_getCoeffs(k);
        transposeCopy(hgT, hg);

        if debug then writeln("  initializing quadrature coefficients");
        init_quadrature(k);

        // blocks of the block tridiagonal derivative operator
        if debug then writeln("  initializing tridiagonal derivative operator");
        make_dc_periodic();

        // initial refinement of analytic function f(x)
        if f != nil {
            if debug then writeln("  performing initial refinement of f(x)");
            for l in 0..2**initial_level-1 do
                refine(initial_level, l);
        }

        if debug then writeln("done.");
    }

    /** Initialize the quadrature coefficient matricies.
     */
    def init_quadrature(order: int) {
        quad_x   = gl_getPoints(k);
        quad_w   = gl_getWeights(k);

        for i in quad_phiDom.dim(1) {
            var p = phi(quad_x[i], k);
            quad_phi [i, ..] = p;
            quad_phiw[i, ..] = quad_w[i] * p;
            quad_phiT[.., i] = p;
        }
    }

    /** Return the level-0 blocks rm, r0, rp of the central
        difference derivative operator with periodic boundary
        conditions on either side.
     */
    def make_dc_periodic() {
        var iphase = 1.0;
        for i in dcDom.dim(1) {
            var jphase = 1.0;

            for j in dcDom.dim(2) {
                var gammaij = sqrt((2*i+1) * (2*j+1));
                var Kij = if (i-j) > 0 && ((i-j) % 2) == 1 then
                        2.0 else 0.0;
                r0[i,j] = 0.5*(1.0 - iphase*jphase - 2.0*Kij)*gammaij;
                rm[i,j] = 0.5*jphase*gammaij;
                rp[i,j] =-0.5*iphase*gammaij;
                jphase = -jphase;
            }
            iphase = -iphase;
        }
    }

    /** Return a deep copy of this Function
     */
    def copy() {
        return Function(k=k, thresh=thresh, f=f, initial_level=initial_level,
                max_level=max_level, autorefine=autorefine, compressed=compressed,
                s=s.copy(), d=d.copy());
    }

    /** Return a copy of this function's skeleton
     */
    def skeletonCopy() {
        // Omit: f, compressed, s, d
        return Function(k=k, thresh=thresh, initial_level=initial_level,
                max_level=max_level, autorefine=autorefine);
    }

    /** s[n, l] = integral(phi[n][l](x) * f(x))
        for box (n,l) project f(x) using quadrature rule
        into scaling function basis
     */
    def project(n: int, l: int) {
        var s     : [quadDom] real;
        var h     = 0.5 ** n;
        var scale = sqrt(h);

        for mu in quad_phiDom.dim(1) {
            var x  = (l + quad_x[mu]) * h;
            var fx = f(x);
            for i in quad_phiDom.dim(2) do
                s[i] += scale * fx * quad_phiw[mu, i];
        }

        return s;
    }

    /** Refine numerical representation of f(x) to desired tolerance
     */
    def refine(n, l) {
        // project f(x) at next level
        var s0 = project(n+1, 2*l);
        var s1 = project(n+1, 2*l+1);
        var sc : [0..2*k-1] real;

        sc[0..k-1]   = s0;
        sc[k..2*k-1] = s1;

        // apply the two scale relationship to get difference coeff
        // in 1d this is O(k^2) flops
        var dc = sc*hgT;

        // check to see if within tolerance
        // normf() is Frobenius norm == 2-norm for vectors
        var nf = normf(dc[k..2*k-1]);
        if((nf < thresh) || (n >= (max_level-1))) {
            s[n+1, 2*l]   = s0;
            s[n+1, 2*l+1] = s1;
        }
        else {
            // these recursive calls on sub-trees can go in parallel
            // if the HashMap is syncronized
            refine(n+1, 2*l  );
            refine(n+1, 2*l+1);
        }
    }

    /** Evaluate numerical representation of f at x.  Performed in scaling
        function basis only.
     */
    def this(x) {
        if compressed then reconstruct();
        return evaluate(0, 0, x);
    }

    /** eval f(x) using adaptively refined numerical representation of f(x)
        answer should be within tolerance of the analytical f(x)

        Descend tree looking for box (n,l) with scaling function
        coefficients containing the point x.
     */
    def evaluate(in n=0, in l=0, in x): real {
        if s.has_coeffs(n, l) {
            var p = phi(x, k);
            return inner(s[n, l], p)*sqrt(2.0**n);

        } else {
            (n, l, x) = (n+1, 2*l, 2*x);
            if x >= 1 then (l, x) = (l+1, x-1);
            return evaluate(n, l, x);
        }
    }

    /** change from scaling function basis to multi-wavelet basis (s -> d)
        tree is filled out with s[0][0] and d
        n is level in tree
        l is box index
     */
    def compress(n=0, l=0) {
        if compressed then return;

        // sub-trees can be done in parallel
        if !s.has_coeffs(n+1,   2*l) then compress(n+1,   2*l);
        if !s.has_coeffs(n+1, 2*l+1) then compress(n+1, 2*l+1);

        var sc: [0..2*k-1] real;
        sc[0..k-1]   = s[n+1,   2*l];
        sc[k..2*k-1] = s[n+1, 2*l+1];

        // apply the two scale relationship to get difference coeff
        // in 1d this is O(k^2) flops (in 3d this is O(k^4) flops)
        var dc = sc*hgT;

        s[n, l] = dc[0..k-1];
        d[n, l] = dc[k..2*k-1];

        s.remove(n+1,   2*l);
        s.remove(n+1, 2*l+1);

        if n==0 then compressed = true;
    }

    /** change from multi-wavelet basis to scaling function basis (d -> s)
        tree just has s at leaves
        n is level in tree
        l is box index
     */
    def reconstruct(n=0, l=0) {
        if !compressed then return;

        if d.has_coeffs(n, l) {
            var dc: [0..2*k-1] real;
            dc[0..k-1]   = s[n, l];
            dc[k..2*k-1] = d[n, l];
            d.remove(n, l);
            s.remove(n, l);

            // apply the two scale relationship to get difference coeff
            // in 1d this is O(k^2) flops (in 3d this is O(k^4) flops)
            var sc = dc*hg;

            s[n+1, 2*l  ] = sc[0..k-1];
            s[n+1, 2*l+1] = sc[k..2*k-1];

            // sub-trees can be done in parallel
            reconstruct(n+1, 2*l);
            reconstruct(n+1, 2*l+1);
        }

        if n == 0 then compressed = false;
    }

    /** In s are scaling coefficients for box n,l ... apply twoscale to generate
        the corresponding coefficients on level n+1 and insert the results into
        the tree of scaling function coefficients.
     */
    def recur_down(n, l) {
        if debug then writeln(" + recur_down(", n, ", ", l, ")");

        var sc : [0..2*k-1] real;
        sc[0..k-1] = s[n, l];

        var new_sc = sc*hg;
        s[n+1, 2*l  ] = new_sc[0..k-1];
        s[n+1, 2*l+1] = new_sc[k..2*k-1];
    }

    /** If the scaling coefficeints in box n,l exist, return them.
        (allow here for zero boundary conditions for boxes just off
        the ends of the domain)

        Else recur up to the next level looking for a parent.  If a
        parent exists, use two scale to recur those coefficients down
        to make n,l.  Note that this modifies the tree in place and
        you should eventually call sclean to tidy up when finished.

        Else, return None (corresponding child boxes exist at a finer scale)
     */
    def get_coeffs(_n, _l) {
        if debug then writeln(" @ get_coeffs(", _n, ", ", _l, ")");
        var n = _n, l = _l;

        // Walk up the tree to find scaling coeffs
        while n >= 0 {
            if l < 0 || l >= 2**n {
                var coeffs : [0..k-1] real;
                return coeffs;
            }

            if s.has_coeffs(n, l) then break;

            n -= 1;
            l  = l/2;
        }

        // Didn't find coeffs, they must be below (_n, _l)
        if n < 0 then return None;

        // Recur down to (_n, _l)
        for i in n.._n-1 {
            l = _l/(2**(_n-i));
            recur_down(i, l);
        }

        return s[_n, _l];
    }

    /** Differentiation (also inner and mul) may leave scaling function
        coefficients below their original level.  Recur down to the
        locally first box with scaling function coefficients.
        Delete all children below there.
     */
    def sclean(n=0, l=0, in cleaning=false) {
        if cleaning { //then
            if debug then writeln(" + sclean deleting (", n, ", ", l, ")");
            s.remove(n, l);
        } else
            cleaning = s.has_coeffs(n, l);

        // Sub trees can run in parallel
        if n < max_level {
            if !cleaning || s.has_coeffs(n+1, 2*l) then
                sclean(n+1, 2*l, cleaning);
            if !cleaning || s.has_coeffs(n+1, 2*l+1) then
                sclean(n+1, 2*l+1, cleaning);
        }
    }

    /** Differentiate the function, which corresponds to application
        of a block triadiagonal matrix.  For an adaptively refined
        target function we may need to refine boxes down until three
        boxes exist in the same scale.
     */
    def diff() {
        def diffHelper(n = 0, l = 0, result) {
            if debug then writeln(" * diff(", n, ", ", l, ")");
            if !s.has_coeffs(n, l) {
                // Sub trees can run in parallel
                // Run down tree until we hit scaling function coefficients
                diffHelper(n+1, 2*l  , result);
                diffHelper(n+1, 2*l+1, result);
            } else {
                // These can also go in parallel since may involve
                // recurring up & down the tree.
                if debug then writeln(" * diff: coeffs at (", n, ", ", l, ")");
                var sm = get_coeffs(n, l-1);
                var sp = get_coeffs(n, l+1);
                var s0 = s[n, l];

                // We have s0, check if we found sm and sp at this level
                if !isNone(sm) && !isNone(sp) {
                    var r = rp*sm + r0*s0 + rm*sp;
                    result.s[n, l] = r * 2.0**n;
                } else {
                    recur_down(n, l);
                    // Sub trees can run in parallel
                    diffHelper(n+1, 2*l  , result);
                    diffHelper(n+1, 2*l+1, result);
                }
            }
        }

        if compressed then reconstruct();
        var result = skeletonCopy();

        diffHelper(0, 0, result);

        sclean();
        return result;
    }

    /** Return sqrt(integral(f(x)**2))
     */
    def norm2() {
        if compressed then reconstruct();
        writeln(s.idx_iter());
        writeln([i in s.idx_iter()] i);
        //return sqrt(+ reduce [i in s] normf(i)**2);
        return 1.0;
    }

    /** Perform GAXPY in the multi-wavelet (compressed) basis
        this = alpha*this + beta*other (other is not changed).
     */
    def gaxpy(alpha, other, beta) {
        // recursive "iteration" for gaxpy
        def gaxpy_iter(n=0, l=0) {
            if d.has_coeffs(n, l) || other.d.has_coeffs(n, l) {
                if d.has_coeffs(n, l) && other.d.has_coeffs(n, l) then
                    d[n, l] = d[n, l]*alpha + other.d[n, l]*beta;

                else if !d.has_coeffs(n, l) && other.d.has_coeffs(n, l) then
                    d[n, l] = other.d[n, l] * beta;

                else // d.has_coeffs(n, l) && !other.d.has_coeffs(n, l)
                    d[n, l] *= alpha;

                // calls on sub-trees can go in parallel
                gaxpy_iter(n+1, 2*l);
                gaxpy_iter(n+1, 2*l+1);
            }
        }
    }

    /*
    def gaxpy(alpha, other, beta) {
        if !compressed then compress();
        if !other.compressed then other.compress();

        s[0, 0] = s[0, 0]*alpha + other.s[0, 0]*beta; // Do scaling coeffs
        for (n, l) in d.idx_iter() {
            d[n, l] *= alpha;
        }
        for (n, l) in other.d.idx_iter() {
            d[n, l] += other.d[n, l]*beta;
        }

        // return this so operations can be chained
        return this;
    }
    */

    /** Add this function to another and return the result in a new
        function.  This and other are unchanged.
     */
    def add(other) {
        return copy().gaxpy(1.0, other, 1.0);
    }

    /** Subtract this function from another and return the result in a new
        function.  This and other are unchanged.
     */
    def subtract(other) {
        return copy().gaxpy(1.0, other, -1.0);
    }

    /** Recursively multiply f1 and f2 put the result into this
     */
    def multiply(f1, f2, n=0, l=0) {
        if f1.s.has_coeffs(n, l) && f2.s.has_coeffs(n, l) {
            if autorefine && n+1 <= max_level {
                // if autorefine is set we are multiplying two polynomials
                // of order k-1 the result could be up to order 2(k-1) so
                // refine one more level to reduce the error and try to
                // keep it within the threshold

                // refine both one more level
                f1.recur_down(n, l);
                f2.recur_down(n, l);

                // scale factor for this level = sqrt((2^d)^(n+1))
                var scale_factor = sqrt(2.0**(n+1));

                // multiply f1.s[n+1][2*l] and f2.s[n+1][2*l]
                var f = f1.s[n+1, 2*l] * quad_phiT;
                var g = f2.s[n+1, 2*l] * quad_phiT;
                f *= g; // Note: Element-wise multiplication
                s[n+1, 2*l] = (f * quad_phiw) * scale_factor;

                // multiply f1.s[n+1][2*l+1] and f2.s[n+1][2*l+1]
                f  =  f1.s[n+1, 2*l+1] * quad_phiT;
                g  =  f2.s[n+1, 2*l+1] * quad_phiT;
                f *= g;
                s[n+1, 2*l+1] = (f * quad_phiw) * scale_factor;

            } else {
                // if autorefine is not set or we are at the max_level
                // live with what you get
                var f = f1.s[n, l] * quad_phiT;
                var g = f2.s[n, l] * quad_phiT;
                f *= g;

                // scale factor for this level = sqrt((2^d)^(n+1))
                s[n, l] = (f * quad_phiw) * sqrt(2.0**(n));
            }

        } else {
            if f1.s.has_coeffs[n, l] && !f2.s.has_coeffs[n, l] then
                // refine this box down to next level in f1
                f1.recur_down(n, l);
            else if !f1.s.has_coeffs[n, l] && f2.s.has_coeffs[n, l] then
                // refine this box down to next level in f2
                f2.recur_down(n, l);

            // calls on sub-trees can go in parallel
            multiply(f1, f2, n+1, 2*l);
            multiply(f1, f2, n+1, 2*l+1);
        }
    }

    /** Function Multiplication:
        For multiply, both operands need to be in the scaling function basis
        so possibly call reconstruct on one or both of them first
     */
    def multiply(other) {
        if compressed then reconstruct();
        if other.compressed then other.reconstruct();

        // Store the result in a new Function
        var result = skeletonCopy();
        result.multiply(this, other);

        sclean();
        other.sclean();

        return result;
    }

    /** Mostly for debugging, print summary of coefficients,
        optionally printing the norm of each block
     */
    def summarize() {
        writeln("\n-----------------------------------------------------");
        writeln("k=", k, " thresh=", thresh, " compressed=", compressed);
        writeln("sum coefficients:");
        for n in 0..max_level {
            var sum = 0.0, ncoeffs = 0;
            for box in s.lvl_iter(n) {
                sum     += normf(box)**2;
                ncoeffs += 1;
            }
            if ncoeffs != 0 then
                writeln("   level ", toString("%2d", n), "   #boxes=",
                        toString("%4d", ncoeffs), "  norm=", toString("%0.2e", truncate(sqrt(sum))));
        }

        writeln("difference coefficients:");
        for n in 0..max_level {
            var sum = 0.0, ncoeffs = 0;
            for box in d.lvl_iter(n) {
                sum     += normf(box)**2;
                ncoeffs += 1;
            }
            if ncoeffs != 0 then
                writeln("   level ", toString("%2d", n), "   #boxes=",
                        toString("%4d", ncoeffs), "  norm=", toString("%0.2e", truncate(sqrt(sum))));
        }

        writeln("-----------------------------------------------------\n");
    }

    /** Evaluate the analytic and numerical functions over the given interval
        and print the error.
     */
    def evalNPT(npt) {
        for i in 0..npt {
            var (fval, Fval) = (truncate(f(i/npt:real)), truncate(this(i/npt:real)));
            //writeln(" -- ", toString("%0.2f", i/npt:real), ":  F_numeric()=", toString("% 0.5e", Fval),
            //        "  f_analytic()=", toString("% 0.5e", fval), " err=", toString("% 0.5e", Fval-fval),
            //        if abs(Fval-fval) > thresh then "  > thresh" else "");
            writeln(" -- ", toString("%0.2f", i/npt:real), ":  F_numeric()=", toString("% 0.8f", Fval),
                    "  f_analytic()=", toString("% 0.8f", fval), " err=", toString("% 0.1e", Fval-fval),
                    if abs(Fval-fval) > thresh then "  > thresh" else "");
        }
    }
}

/*************************************************************************/
/* Standard operations on Function objects:                              */
/*************************************************************************/

def +(F: Function, G: Function): Function {
    return F.add(G);
}

def -(F: Function, G: Function): Function {
    return F.subtract(G);
}

def *(F: Function, G: Function): Function {
    return F.multiply(G);
}


Make sure chpl_executeOn*() accept source line and file arguments.

[Reviewed by hildeth.]

After r21536 we observed a few test failures due to chpl_executeOn*()
not always being emitted such that they accepted a final pair of
parameters giving the source line and file.  Add pragmas to make sure
these procedures always take such arguments.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@21540 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// LocaleModel.chpl
//
// This provides a flat locale model architectural description.  The
// locales contain memory and a multi-core processor with homogeneous
// cores, and we ignore any affinity (NUMA effects) between the
// processor cores and the memory.  This architectural description is
// backward compatible with the architecture implicitly provided by
// releases 1.6 and preceding.
//
pragma "no use ChapelStandard"
module LocaleModel {

  use ChapelLocale;
  use DefaultRectangular;
  use ChapelNumLocales;
  use RootLocale;
  use Sys;

  config param debugLocaleModel = false;

  // We would really like a class-static storage class.(C++ nomenclature)
  var doneCreatingLocales: bool = false;

  //
  // A concrete class representing the nodes in this architecture.
  //
  class LocaleModel : locale {
    const callStackSize: int;
    const _node_id : int;
    const local_name : string;

    // This constructor must be invoked "on" the node
    // that it is intended to represent.  This trick is used
    // to establish the equivalence the "locale" field of the locale object
    // and the node ID portion of any wide pointer referring to it.
    proc LocaleModel() {
      if doneCreatingLocales {
        halt("Cannot create additional LocaleModel instances");
      }
      init();
    }

    proc LocaleModel(parent_loc : locale) {
      if doneCreatingLocales {
        halt("Cannot create additional LocaleModel instances");
      }
      parent = parent_loc;
      init();
    }

    // chpl_nodeID is the node ID associated with the running image.
    proc chpl_id() return _node_id;
    proc chpl_name() return local_name;

    proc readWriteThis(f) {
      // Most classes will define it like this:
//      f <~> name;
      // but here it is defined thus for backward compatibility.
      f <~> new ioLiteral("LOCALE") <~> _node_id;
    }

    proc getChildSpace() return emptyLocaleSpace;

    proc getChildCount() return 0;

    iter getChildIndices() : int {
      for idx in emptyLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) : locale {
      // This is a temporary implementation, where an index of zero is forced
      // to mean "here".  A better solution is on the way.
      // TODO: Run index lookup through rootLocale.findLocale(lid:chpl_localeID_t);
      if idx == 0 then return this;
      else
        if boundsChecking then
          halt("requesting a child from a LocaleModel locale");
      return nil;
    }

    iter getChldren() : locale  {
      for loc in emptyLocales do
        yield loc;
    }

    proc getChildArray() {
      return emptyLocales;
    }

    // Part of the public interface required by the compiler
    // These are dynamically dispatched, so cannot be inlined.
    proc taskInit() {}
    proc taskExit() {}

    //------------------------------------------------------------------------{
    //- Implementation (private)
    //-
    proc init() {
      _node_id = __primitive("cast", int(64), __primitive("chpl_nodeID"));

      // chpl_nodeName is defined in chplsys.c.
      // It supplies a node name obtained by running uname(3) on the current node.
      // For this reason (as well), the constructor (or at least this init method)
      // must be run on the node it is intended to describe.
//      extern proc getenv(s:string) : string;
      var comm, spawnfn : string;
      extern proc chpl_nodeName() : string;
      // sys_getenv returns zero on success.
      if sys_getenv("CHPL_COMM", comm) == 0 && comm == "gasnet" &&
        sys_getenv("GASNET_SPAWNFN", spawnfn) == 0 && spawnfn == "L"
      then local_name = chpl_nodeName() + "-" + _node_id : string;
      else local_name = chpl_nodeName();

      extern proc chpl_task_getCallStackSize(): int;
      callStackSize = chpl_task_getCallStackSize();

      extern proc chpl_numCoresOnThisLocale(): int;
      numCores = chpl_numCoresOnThisLocale();
    }
    //------------------------------------------------------------------------}
  }

  //
  // An instance of this class is the default contents 'rootLocale'.
  //
  // In the current implementation a platform-specific architectural description
  // may overwrite this instance or any of its children to establish a more customized
  // representation of the system resources.
  //
  class DefaultRootLocale : RootLocale {

    // Would like to make myLocaleSpace distributed with one index per node.
    const myLocaleSpace: domain(1) = {0..numLocales-1};
    const myLocales: [myLocaleSpace] locale;

    proc DefaultRootLocale()
    {
      // A bootstrap routine that returns (chpl_localeID_t){.node = <arg>, .subloc = 0}.
      extern proc chpl_return_localeID_node(node:int(32)) : chpl_localeID_t;

      parent = nil;
      numCores = 0;

      // We cannot use a forall here because the default leader iterator will
      // access 'Locales' and 'here', which are not yet initialized.
      for locIdx in myLocaleSpace
      {
        var locID = chpl_return_localeID_node(locIdx:int(32));
        on __primitive("chpl_on_locale_num", locID)
        {
          // chpl_on_locale_num sets the localeID portion of "here", but
          // leaves the addr portion as a NULL pointer.
          const node = new LocaleModel(this);

          // So immediately after creating a new locale to represent here, we
          // have to call this primitive to insert it into task-private storage.
          __primitive("_task_set_here_ptr", node);

          myLocales[locIdx] = node;
          numCores += node.numCores;
        }
      }
      doneCreatingLocales = true;

      // Sanity check.
      // Ensure that the locale representing node x has a nodeID of x.
      // At the top level, this lets us look up the locale representing the node on which
      // an object o lives by executing
      //  myLocales[__primitive("_wide_get_node", o)].
      if (debugLocaleModel) then
        for locIdx in myLocaleSpace {
          var loc = myLocales[locIdx];
          if __primitive("_wide_get_node", loc) != locIdx then
            halt("In this architecture, we expect the locale whose index is x to live on node x.");
        }

      // Programs traditionally expect the startup process to run on Locales[0], so
      // this is how we mimic that behavior.
      // Note that this means we have to ask for here.parent or rootLocale to get
      // the root locale of the default architecture.
      var loc = myLocales[0];
      __primitive("_task_set_here_ptr", loc);
      var locID = chpl_return_localeID_node(0:int(32));
      __primitive("_task_set_locale_id", locID);
    }

    // Has to be globally unique and not equal to a node ID.
    // We return numLocales for now, since we expect nodes to be numbered less than this.
    // -1 is used in the abstract locale class to specify an invalid node ID.
    proc chpl_id() return numLocales;
    proc chpl_name() return local_name();
    proc local_name() return "rootLocale";

    proc readWriteThis(f) {
      f <~> name;
    }

    proc getChildCount() return this.myLocaleSpace.numIndices;

    proc getChildSpace() return this.myLocaleSpace;

    iter getChildIndices() : int {
      for idx in this.myLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) return this.myLocales[idx];

    iter getChldren() : locale  {
      for loc in this.myLocales do
        yield loc;
    }

    proc getDefaultLocaleSpace() return this.myLocaleSpace;
    proc getDefaultLocaleArray() return myLocales;

    proc localeIDtoLocale(id : chpl_localeID_t) {
      // In the default architecture, there are only nodes and no sublocales.
      // What is more, the nodeID portion of a wide pointer is the same as
      // the index into myLocales that yields the locale representing that node.
      return myLocales[id.node];
    }
  }

  // Use the DefaultArchitecture only if no other architecture has already been defined.
  if (rootLocale == nil) then
    rootLocale = new DefaultRootLocale();

  // Expose the underlying locales array (and its domain)
  // for user convenience and backward compatibility.
  // The downcast is because we cannot move the domain and array return types
  // into the base (abstract) locale class.
  // That would make locales depend on arrays which depend on locales....
  // If we had a way to express domain and array return types abstractly,
  // that problem would go away. <hilde>
  const Locales => (rootLocale:RootLocale).getDefaultLocaleArray();
  const LocaleSpace = Locales.domain;

  //////////////////////////////////////////
  //
  // runtime interface
  //

  extern proc chpl_comm_fork(loc_id: int, subloc_id: int,
                             fn: int, args: c_ptr, arg_size: int(32));
  extern proc chpl_comm_fork_fast(loc_id: int, subloc_id: int,
                                  fn: int, args: c_ptr, args_size: int(32));
  extern proc chpl_comm_fork_nb(loc_id: int, subloc_id: int,
                                fn: int, args: c_ptr, args_size: int(32));

  //
  // regular "on"
  //
  pragma "insert line file info"
  export
  proc chpl_executeOn(loc: chpl_localeID_t, // target locale
                      fn: int,              // on-body function idx
                      args: c_ptr,          // function args
                      args_size: int(32)    // args size
                     ) {
    chpl_comm_fork(loc.node, loc.subloc, fn, args, args_size);
  }

  //
  // fast "on" (doesn't do anything that could deadlock a comm layer,
  // in the Active Messages sense)
  //
  pragma "insert line file info"
  export
  proc chpl_executeOnFast(loc: chpl_localeID_t, // target locale
                          fn: int,              // on-body function idx
                          args: c_ptr,          // function args
                          args_size: int(32)    // args size
                         ) {
    chpl_comm_fork_fast(loc.node, loc.subloc, fn, args, args_size);
  }

  //
  // nonblocking "on" (doesn't wait for completion)
  //
  pragma "insert line file info"
  export
  proc chpl_executeOnNB(loc: chpl_localeID_t, // target locale
                        fn: int,              // on-body function idx
                        args: c_ptr,          // function args
                        args_size: int(32)    // args size
                       ) {
    //
    // If we're in serial mode, we should use blocking rather than
    // non-blocking "on" in order to serialize the forks.
    //
    if __primitive("task_get_serial") then
      chpl_comm_fork(loc.node, loc.subloc, fn, args, args_size);
    else
      chpl_comm_fork_nb(loc.node, loc.subloc, fn, args, args_size);
  }
}

// LocaleModel.chpl
//
// This provides a flat locale model architectural description.  The
// locales contain memory and a multi-core processor with homogeneous
// cores, and we ignore any affinity (NUMA effects) between the
// processor cores and the memory.  This architectural description is
// backward compatible with the architecture implicitly provided by
// releases 1.6 and preceding.
//
pragma "no use ChapelStandard"
module LocaleModel {

  use ChapelLocale;
  use DefaultRectangular;
  use ChapelNumLocales;
  use RootLocale;
  use Sys;

  config param debugLocaleModel = false;

  // We would really like a class-static storage class.(C++ nomenclature)
  var doneCreatingLocales: bool = false;

  //
  // A concrete class representing the nodes in this architecture.
  //
  class LocaleModel : locale {
    const callStackSize: int;
    const _node_id : int;
    const local_name : string;

    // This constructor must be invoked "on" the node
    // that it is intended to represent.  This trick is used
    // to establish the equivalence the "locale" field of the locale object
    // and the node ID portion of any wide pointer referring to it.
    proc LocaleModel() {
      if doneCreatingLocales {
        halt("Cannot create additional LocaleModel instances");
      }
      init();
    }

    proc LocaleModel(parent_loc : locale) {
      if doneCreatingLocales {
        halt("Cannot create additional LocaleModel instances");
      }
      parent = parent_loc;
      init();
    }

    // chpl_nodeID is the node ID associated with the running image.
    proc chpl_id() return _node_id;
    proc chpl_name() return local_name;

    proc readWriteThis(f) {
      // Most classes will define it like this:
//      f <~> name;
      // but here it is defined thus for backward compatibility.
      f <~> new ioLiteral("LOCALE") <~> _node_id;
    }

    proc getChildSpace() return emptyLocaleSpace;

    proc getChildCount() return 0;

    iter getChildIndices() : int {
      for idx in emptyLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) : locale {
      // This is a temporary implementation, where an index of zero is forced
      // to mean "here".  A better solution is on the way.
      // TODO: Run index lookup through rootLocale.findLocale(lid:chpl_localeID_t);
      if idx == 0 then return this;
      else
        if boundsChecking then
          halt("requesting a child from a LocaleModel locale");
      return nil;
    }

    iter getChldren() : locale  {
      for loc in emptyLocales do
        yield loc;
    }

    proc getChildArray() {
      return emptyLocales;
    }

    // Part of the public interface required by the compiler
    // These are dynamically dispatched, so cannot be inlined.
    proc taskInit() {}
    proc taskExit() {}

    //------------------------------------------------------------------------{
    //- Implementation (private)
    //-
    proc init() {
      _node_id = __primitive("cast", int(64), __primitive("chpl_nodeID"));

      // chpl_nodeName is defined in chplsys.c.
      // It supplies a node name obtained by running uname(3) on the current node.
      // For this reason (as well), the constructor (or at least this init method)
      // must be run on the node it is intended to describe.
//      extern proc getenv(s:string) : string;
      var comm, spawnfn : string;
      extern proc chpl_nodeName() : string;
      // sys_getenv returns zero on success.
      if sys_getenv("CHPL_COMM", comm) == 0 && comm == "gasnet" &&
        sys_getenv("GASNET_SPAWNFN", spawnfn) == 0 && spawnfn == "L"
      then local_name = chpl_nodeName() + "-" + _node_id : string;
      else local_name = chpl_nodeName();

      extern proc chpl_task_getCallStackSize(): int;
      callStackSize = chpl_task_getCallStackSize();

      extern proc chpl_numCoresOnThisLocale(): int;
      numCores = chpl_numCoresOnThisLocale();
    }
    //------------------------------------------------------------------------}
  }

  //
  // An instance of this class is the default contents 'rootLocale'.
  //
  // In the current implementation a platform-specific architectural description
  // may overwrite this instance or any of its children to establish a more customized
  // representation of the system resources.
  //
  class DefaultRootLocale : RootLocale {

    // Would like to make myLocaleSpace distributed with one index per node.
    const myLocaleSpace: domain(1) = {0..numLocales-1};
    const myLocales: [myLocaleSpace] locale;

    proc DefaultRootLocale()
    {
      // A bootstrap routine that returns (chpl_localeID_t){.node = <arg>, .subloc = 0}.
      extern proc chpl_return_localeID_node(node:int(32)) : chpl_localeID_t;

      parent = nil;
      numCores = 0;

      // We cannot use a forall here because the default leader iterator will
      // access 'Locales' and 'here', which are not yet initialized.
      for locIdx in myLocaleSpace
      {
        var locID = chpl_return_localeID_node(locIdx:int(32));
        on __primitive("chpl_on_locale_num", locID)
        {
          // chpl_on_locale_num sets the localeID portion of "here", but
          // leaves the addr portion as a NULL pointer.
          const node = new LocaleModel(this);

          // So immediately after creating a new locale to represent here, we
          // have to call this primitive to insert it into task-private storage.
          __primitive("_task_set_here_ptr", node);

          myLocales[locIdx] = node;
          numCores += node.numCores;
        }
      }
      doneCreatingLocales = true;

      // Sanity check.
      // Ensure that the locale representing node x has a nodeID of x.
      // At the top level, this lets us look up the locale representing the node on which
      // an object o lives by executing
      //  myLocales[__primitive("_wide_get_node", o)].
      if (debugLocaleModel) then
        for locIdx in myLocaleSpace {
          var loc = myLocales[locIdx];
          if __primitive("_wide_get_node", loc) != locIdx then
            halt("In this architecture, we expect the locale whose index is x to live on node x.");
        }

      // Programs traditionally expect the startup process to run on Locales[0], so
      // this is how we mimic that behavior.
      // Note that this means we have to ask for here.parent or rootLocale to get
      // the root locale of the default architecture.
      var loc = myLocales[0];
      __primitive("_task_set_here_ptr", loc);
      var locID = chpl_return_localeID_node(0:int(32));
      __primitive("_task_set_locale_id", locID);
    }

    // Has to be globally unique and not equal to a node ID.
    // We return numLocales for now, since we expect nodes to be numbered less than this.
    // -1 is used in the abstract locale class to specify an invalid node ID.
    proc chpl_id() return numLocales;
    proc chpl_name() return local_name();
    proc local_name() return "rootLocale";

    proc readWriteThis(f) {
      f <~> name;
    }

    proc getChildCount() return this.myLocaleSpace.numIndices;

    proc getChildSpace() return this.myLocaleSpace;

    iter getChildIndices() : int {
      for idx in this.myLocaleSpace do
        yield idx;
    }

    proc getChild(idx:int) return this.myLocales[idx];

    iter getChldren() : locale  {
      for loc in this.myLocales do
        yield loc;
    }

    proc getDefaultLocaleSpace() return this.myLocaleSpace;
    proc getDefaultLocaleArray() return myLocales;

    proc localeIDtoLocale(id : chpl_localeID_t) {
      // In the default architecture, there are only nodes and no sublocales.
      // What is more, the nodeID portion of a wide pointer is the same as
      // the index into myLocales that yields the locale representing that node.
      return myLocales[id.node];
    }
  }

  // Use the DefaultArchitecture only if no other architecture has already been defined.
  if (rootLocale == nil) then
    rootLocale = new DefaultRootLocale();

  // Expose the underlying locales array (and its domain)
  // for user convenience and backward compatibility.
  // The downcast is because we cannot move the domain and array return types
  // into the base (abstract) locale class.
  // That would make locales depend on arrays which depend on locales....
  // If we had a way to express domain and array return types abstractly,
  // that problem would go away. <hilde>
  const Locales => (rootLocale:RootLocale).getDefaultLocaleArray();
  const LocaleSpace = Locales.domain;

  //////////////////////////////////////////
  //
  // runtime interface
  //

  extern proc chpl_comm_fork(loc_id: int, subloc_id: int,
                             fn: int, args: c_ptr, arg_size: int(32));
  extern proc chpl_comm_fork_fast(loc_id: int, subloc_id: int,
                                  fn: int, args: c_ptr, args_size: int(32));
  extern proc chpl_comm_fork_nb(loc_id: int, subloc_id: int,
                                fn: int, args: c_ptr, args_size: int(32));

  //
  // regular "on"
  //
  export
  proc chpl_executeOn(loc: chpl_localeID_t, // target locale
                      fn: int,              // on-body function idx
                      args: c_ptr,          // function args
                      args_size: int(32)    // args size
                     ) {
    chpl_comm_fork(loc.node, loc.subloc, fn, args, args_size);
  }

  //
  // fast "on" (doesn't do anything that could deadlock a comm layer,
  // in the Active Messages sense)
  //
  export
  proc chpl_executeOnFast(loc: chpl_localeID_t, // target locale
                          fn: int,              // on-body function idx
                          args: c_ptr,          // function args
                          args_size: int(32)    // args size
                         ) {
    chpl_comm_fork_fast(loc.node, loc.subloc, fn, args, args_size);
  }

  //
  // nonblocking "on" (doesn't wait for completion)
  //
  export
  proc chpl_executeOnNB(loc: chpl_localeID_t, // target locale
                        fn: int,              // on-body function idx
                        args: c_ptr,          // function args
                        args_size: int(32)    // args size
                       ) {
    //
    // If we're in serial mode, we should use blocking rather than
    // non-blocking "on" in order to serialize the forks.
    //
    if __primitive("task_get_serial") then
      chpl_comm_fork(loc.node, loc.subloc, fn, args, args_size);
    else
      chpl_comm_fork_nb(loc.node, loc.subloc, fn, args, args_size);
  }
}