Moving the conditional checking of whether numTasks is 0 and assigning it a
default value (here.numCores or dataParTasksPerLocale) into the body of the
function rather than in the argument.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@19069 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config param verbose:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int) {

  if verbose then
    compilerWarning("Working with serial dynamic Iterator");
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterator.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/chunkSize < nTasks then {
    writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    //yield tuple(c.translate(-c.low)) ;
    yield tuple(totalRange);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#nTasks do {
      while moreWork do {
	  // There is local work in remain
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=splitChunk(remain, chunkSize, moreWork);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType= densify(current,c);
	    if verbose then
	      writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int) {

  if verbose then
    compilerWarning("Working with serial guided Iterator");
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterator, c:range(?), numTasks:int=0)
where tag == iterator.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield tuple(totalRange);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=nTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#nTasks do {
      while undone do {
	  // There is local work in remain(tid)
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=adaptSplit(remain, factor, undone);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType = densify(current,c);
	    if verbose then
	      writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter guided(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower

{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  if verbose then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int) {

  if verbose then
    compilerWarning("Working with serial adaptive Iterator");
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterator, c:range(?), numTasks:int=0)
where tag == iterator.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    halt("methodStealing value must be between 0 and 2");

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield tuple(totalRange);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination
    var nVisitedVictims: int=0;

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#nTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if verbose then
	writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
	  // There is local work
	  const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
	  if zeroBasedIters.length !=0 then {
	    if verbose then
	      writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
	    yield tuple(zeroBasedIters);}
	}

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
	  if verbose then
	    writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

	  // Perform the spliting at the victim remaining range

	  select methodStealing {
	    when 0 do { // Steals from the victim until it exhausts its range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                        //after splitting from a victim range
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }

	    when 1 do { // Steals once from victim. If it has sucess on the stealing,
	                // it chooses another victim and tries to steal again.

	      if moreLocalWork[victim] then {
		// There is work in victim
		const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                   //after splitting from a victim range
		if zeroBasedIters2.length !=0 then {
		  if verbose then
		    writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		  yield tuple(zeroBasedIters2);}
	      }
	      else {
		stealFailed=true;
	      }
	    }
	    otherwise { // Tt steals from the victim, but now splitting
	                //from the tail of the victim's range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
		                        // after splitting from a victim range:
		                       // now we set splitTatil to true (second parameter in the call)
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }
	    }
	  // If here, then it can have failed the stealing intent at the victim (method 1),
	  // or we have exhausted the victim range (methods 0, 2)

	  if (methodStealing==0 || methodStealing==2 || methodStealing==1 && stealFailed) then {
	    nVisitedVictims += 1; // Signal that there is no more work in victim
	    if verbose then
	      writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
	    stealFailed=false;
	  }
	  // Check if there is no more work
	  if nVisitedVictims >= nTasks-1 then
	    moreWork=false; // Signal that there is no more work in any victim
	  else {  // There can be still work in other victim
	    victim=(victim+1) % nTasks; // New victim to steal
	    if methodStealing==1 then
	      if victim==tid then victim=(victim+1) % nTasks;
	  }
	}
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/nTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==nTasks-1 then
      rLocal=c#(size*(nTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==nTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  var current:rType=unDensify(follower(1),c);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.numCores;
      else dnTasks=dataParTasksPerLocale;
  }
  return dnTasks;
}

proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}

config param verbose:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int) {

  if verbose then
    compilerWarning("Working with serial dynamic Iterator");
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int= if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader
{
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length/chunkSize < numTasks then {
    writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    //yield tuple(c.translate(-c.low)) ;
    yield tuple(totalRange);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#numTasks do {
      while moreWork do {
	  // There is local work in remain
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=splitChunk(remain, chunkSize, moreWork);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType= densify(current,c);
	    //const zeroBasedIters= current.translate(-c.low);
	    if verbose then
	      writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter dynamic(param tag:iterator, c:range(?), chunkSize:int, numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  //follower(1).translate(c.low);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int) {

  if verbose then
    compilerWarning("Working with serial guided Iterator");
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterator, c:range(?), numTasks:int = if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader
{
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < numTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield tuple(totalRange);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=numTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#numTasks do {
      while undone do {
	  // There is local work in remain(tid)
	  splitLock$; // Perform the splitting in a critical section
	  const current:rType=adaptSplit(remain, factor, undone);
	  splitLock$=true;
	  if current.length !=0 then {
	    const zeroBasedIters:rType = densify(current,c);
	    if verbose then
	      writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);

	    yield tuple(zeroBasedIters);}
	}
    }
  }
}

// Follower
iter guided(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower

{
  type rType=c.type;
  const current:rType=unDensify(follower(1),c);
  if verbose then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int) {

  if verbose then
    compilerWarning("Working with serial adaptive Iterator");
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterator, c:range(?), numTasks:int = if dataParTasksPerLocale==0 then here.numCores else dataParTasksPerLocale)
where tag == iterator.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    halt("methodStealing value must be between 0 and 2");

  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < numTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield tuple(totalRange);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#numTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination
    var nVisitedVictims: int=0;

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#numTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if verbose then
	writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
	  // There is local work
	  const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
	  if zeroBasedIters.length !=0 then {
	    if verbose then
	      writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
	    yield tuple(zeroBasedIters);}
	}

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var victim=(tid+1) % numTasks;
      var stealFailed:bool=false;

      while moreWork do {
	  if verbose then
	    writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

	  // Perform the spliting at the victim remaining range

	  select methodStealing {
	    when 0 do { // Steals from the victim until it exhausts its range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                        //after splitting from a victim range
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }

	    when 1 do { // Steals once from victim. If it has sucess on the stealing,
	                // it chooses another victim and tries to steal again.

	      if moreLocalWork[victim] then {
		// There is work in victim
		const zeroBasedIters2:rType=splitWork(victim); // The current range we get
		                                   //after splitting from a victim range
		if zeroBasedIters2.length !=0 then {
		  if verbose then
		    writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		  yield tuple(zeroBasedIters2);}
	      }
	      else {
		stealFailed=true;
	      }
	    }
	    otherwise { // Tt steals from the victim, but now splitting
	                //from the tail of the victim's range

	      while moreLocalWork[victim] do {
		  // There is work in victim
		  const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
		                        // after splitting from a victim range:
		                       // now we set splitTatil to true (second parameter in the call)
		  if zeroBasedIters2.length !=0 then {
		    if verbose then
		      writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
		    yield tuple(zeroBasedIters2);}
		}
	    }
	    }
	  // If here, then it can have failed the stealing intent at the victim (method 1),
	  // or we have exhausted the victim range (methods 0, 2)

	  if (methodStealing==0 || methodStealing==2 || methodStealing==1 && stealFailed) then {
	    nVisitedVictims += 1; // Signal that there is no more work in victim
	    if verbose then
	      writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
	    stealFailed=false;
	  }
	  // Check if there is no more work
	  if nVisitedVictims >= numTasks-1 then
	    moreWork=false; // Signal that there is no more work in any victim
	  else {  // There can be still work in other victim
	    victim=(victim+1) % numTasks; // New victim to steal
	    if methodStealing==1 then
	      if victim==tid then victim=(victim+1) % numTasks;
	  }
	}
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/numTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==numTasks-1 then
      rLocal=c#(size*(numTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==numTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    //var current:rType;
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      //const zeroBasedCurrent = densify(current,c);
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterator, c:range(?), numTasks:int, follower)
where tag == iterator.follower
{
  type rType=c.type;
  var current:rType=unDensify(follower(1),c);
  //follower(1).translate(c.low);
  if (verbose) then
    writeln("Follower received range ", follower, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}


Got CMO_array.chpl working again...

Got CMO_array.chpl working again...

It turns out that the only thing that needed fixing here was to
make the _dist_build_domain() calls generic rather than
inheritance-based as in my last checkin.  By the time I had made
this change yesterday, I was apparently way too deep into a number
of other speculative changes to have determined how close this
was to working.

The only other required change was rename buildDomain to newDomain.
I also made Mary Beth's classes into subclasses of Distribution
and BaseDenseArithmeticDomain respectively just to make them that
much more accurate, but these changes were not necessary for
correctness.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14584 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

class CMODist : Distribution {
  def newDomain(param rank: int, type dimensional_index_type, param stridable: bool) {
    return new CMODomain(rank=rank, idxType=dimensional_index_type, stridable=stridable);
  }
}

class CMODomain: BaseDenseArithmeticDomain {
  param rank : int;
  type idxType;
  param stridable: bool;
  var ranges : rank*range(idxType,BoundedRangeType.bounded,stridable);

  def getIndices() return ranges;

  def setIndices(x) {
    if ranges.size != x.size then
      compilerError("rank mismatch in domain assignment");
    if ranges(1).eltType != x(1).eltType then
      compilerError("index type mismatch in domain assignment");
    ranges = x;
  }

  def buildEmptyDomain()
    return new CMODomain(rank=rank, idxType=idxType, stridable=stridable);

  def buildOpenIntervalUpper() {
    var x = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    for param i in 1..rank {
      if ranges(i)._stride != 1 then
        halt("syntax [domain-specification) requires a stride of one");
      x.ranges(i) = ranges(i)._low..ranges(i)._high-1;
    }
    return x;
  }

  def these_help(param dim: int) {
    if dim == rank - 1 {
      for j in ranges(rank) do
        for i in ranges(dim) do
          yield (i, j);
    } else {
      for j in these_help(dim+1) do
        for i in ranges(dim) do
          yield (i, (...j));
    }
  }

  def these() {
    if rank == 1 {
      for i in ranges(1) do
        yield i;
    } else {
      for i in these_help(1) do
        yield i;
    }
  }

  def this(dim : int)
    return ranges(dim);

  def member(ind: idxType) where rank == 1 {
    if !ranges(1).member(ind) then
      return false;
    return true;
  }

  def member(ind: rank*idxType) {
    for param i in 1..rank do
      if !ranges(i).member(ind(i)) then
        return false;
    return true;
  }

  def dim(d : int)
    return ranges(d);

  def bbox(d: int) {
    const r: range(idxType,BoundedRangeType.bounded,false) = ranges(d);
    return r;
  }

  def numIndices {
    var sum = 1:idxType;
    for param i in 1..rank do
      sum *= ranges(i).length;
    return sum;
    // WANT: return * reduce (this(1..rank).length);
  }

  def low {
    if rank == 1 {
      return ranges(1)._low;
    } else {
      var result: rank*idxType;
      for param i in 1..rank do
        result(i) = ranges(i)._low;
      return result;
    }
  }

  def high {
    if rank == 1 {
      return ranges(1)._high;
    } else {
      var result: rank*idxType;
      for param i in 1..rank do
        result(i) = ranges(i)._high;
      return result;
    }
  }

  def buildArray(type eltType) {
    return new CMOArray(eltType, rank, idxType, stridable, dom=this);
  }

  def buildSubdomain()
    return new CMODomain(rank=rank, idxType=idxType, stridable=stridable);

  def rankChange(param rank: int, param stridable: bool, args) {
    def isRange(r: range(?e,?b,?s)) param return 1;
    def isRange(r) param return 0;

    var d = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    var i = 1;
    for param j in 1..args.size {
      if isRange(args(j)) {
        d.ranges(i) = dim(j)(args(j));
        i += 1;
      }
    }
    return d;
  }

  def translate(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable = stridable);
    for i in 1..rank do
      x.ranges(i) = dim(i)._translate(off(i));
    return x;
  }

  def interior(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do {
      if ((off(i) > 0) && (dim(i)._high+1-dim(i) < dim(i)._low) ||
          (off(i) < 0) && (dim(i)._low-1-dim(i) > dim(i)._high)) {
        halt("***Error: Argument to 'interior' function out of range in dimension ", i, "***");
      }
      x.ranges(i) = dim(i)._interior(off(i));
    }
    return x;
  }

  def exterior(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = dim(i)._exterior(off(i));
    return x;
  }

  def expand(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do {
      x.ranges(i) = ranges(i)._expand(off(i));
      if (x.ranges(i)._low > x.ranges(i)._high) {
        halt("***Error: Degenerate dimension created in dimension ", i, "***");
      }
    }
    return x;
  }

  def expand(off: int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = ranges(i)._expand(off);
    return x;
  }

  // this should eventually...
  //  ...be in the distribtion's implementation of the blocked domain
  // ...yield a subBlock of the domain per thread per locale
  def subBlocks {
    yield this;
  }

  def strideBy(str : rank*int) {
    var x = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = ranges(i) by str(i);
    return x;
  }

  def strideBy(str : int) {
    var x = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = ranges(i) by str;
    return x;
  }
}

class CMOArray:BaseArray {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;
  param reindexed: bool = false;

  var dom: CMODomain(rank=rank,idxType=idxType, stridable=stridable);
  var off: rank*idxType;
  var blk: rank*idxType;
  var str: rank*int;
  var origin: idxType;
  var factoredOffs: idxType;
  var size: idxType;
  var D1: domain(1, idxType);
  var data: [D1] eltType;
  var noinit: bool = false;

  def computeFactoredOffs() {
    factoredOffs = 0:idxType;
    for i in 1..rank do {
      factoredOffs += blk(i) * off(i);
    }
  }

  def initialize() {
    if noinit == true then return;
    for param dim in 1..rank {
      off(dim) = dom.dim(dim)._low;
      str(dim) = dom.dim(dim)._stride;
    }
    blk(1) = 1:idxType;
    for dim in 2..rank do
      blk(dim) = blk(dim-1) * dom.dim(dim-1).length;
    computeFactoredOffs();
    size = blk(rank) * dom.dim(rank).length;
    D1 = [0:idxType..#size:idxType];
    data = 0:eltType;
  }

  def these() {
    for i in dom do {
      yield this(i);
    }
  }

  def this(ind: idxType ...1) var where rank == 1
    return this(ind);

  def this(ind : rank*idxType) var {
    if boundsChecking then
      if !dom.member(ind) then
        halt("array index out of bounds: ", ind);
    var sum = origin;
    if stridable {
      for param i in 1..rank do
        sum = sum + (ind(i) - off(i)) * blk(i) / str(i):idxType;
    } else {
      if reindexed {
        for param i in 1..rank do
          sum += ind(i) * blk(i);
      } else {
        for param i in 1..rank do
          sum += ind(i) * blk(i);
      }
      sum -= factoredOffs;
    }
    return data(sum);
  }

  def reindex(d: CMODomain) {
    if rank != d.rank then
      compilerError("illegal implicit rank change");
    for param i in 1..rank do
      if d.dim(i).length != dom.dim(i).length then
        halt("extent in dimension ", i, " does not match actual");
    var alias = new CMOArray(eltType, d.rank, d.idxType, d.stridable, true, d, noinit=true);
    //    was:  (eltType, rank, idxType, d.stridable, true, d, noinit=true);
    alias.D1 = [0:idxType..#size:idxType];
    alias.data = data;
    alias.size = size: d.idxType;
    for param i in 1..rank {
      alias.off(i) = d.dim(i)._low;
      alias.blk(i) = (blk(i) * dom.dim(i)._stride / str(i)) : d.idxType;
      alias.str(i) = d.dim(i)._stride;
    }
    alias.origin = origin:d.idxType;
    alias.computeFactoredOffs();
    return alias;
  }

  def checkSlice(d) {
    for param i in 1..rank {
      if d(i).boundedType == BoundedRangeType.bounded then
        if !dom.dim(i).member(d(i)) then
          halt("array slice out of bounds in dimension ", i, ": ", d(i));
    }
  }

  def slice(d: CMODomain) {
    var alias = new CMOArray(eltType, rank, idxType, d.stridable, reindexed, d, noinit=true);
    alias.D1 = [0:idxType..#size:idxType];
    alias.data = data;
    alias.size = size;
    alias.blk = blk;
    alias.str = str;
    alias.origin = origin;
    for param i in 1..rank {
      alias.off(i) = d.dim(i)._low;
      alias.origin += blk(i) *(d.dim(i)._low - off(i))/str(i);
    }
    alias.computeFactoredOffs();
    return alias;
  }

  def checkRankChange(args) {
    def isRange(r: range(?e,?b,?s)) param return 1;
    def isRange(r) param return 0;

    for param i in 1..args.size do
      if isRange(args(i)) then
        if args(i).boundedType == BoundedRangeType.bounded then
          if !dom.dim(i).member(args(i)) then
            halt("array slice out of bounds in dimension ", i, ": ", args(i));
  }

  def rankChange(param newRank: int, param newStridable: bool, irs) {
    def isRange(r: range(?e,?b,?s)) param return 1;
    def isRange(r) param return 0;
    var d = dom.rankChange(newRank, newStridable, irs);
    var alias = new CMOArray(eltType, newRank, idxType, newStridable, true, d, noinit=true);
    alias.D1 = [0:idxType..#size:idxType];
    alias.data = data;
    alias.size = size;
    var i = 1;
    alias.origin = origin;
    for param j in 1..irs.size {
      if isRange(irs(j)) {
        alias.off(i) = d.dim(i)._low;
        alias.origin += blk(j) * (d.dim(i)._low - off(j)) / str(j);
        alias.blk(i) = blk(j);
        alias.str(i) = str(j);
        i += 1;
      } else {
        alias.origin += blk(j) * (irs(j) - off(j)) / str(j);
      }
    }
    alias.computeFactoredOffs();
    return alias;
  }

  def reallocate(d: _domain) {
    if (d._value.type == dom.type) {
      var copy = new CMOArray(eltType, rank, idxType, d._value.stridable, reindexed, d._value);
      for i in _intersect(d._value, dom) do
        copy(i) = this(i);
      off = copy.off;
      blk = copy.blk;
      str = copy.str;
      origin = copy.origin;
      factoredOffs = copy.factoredOffs;
      size = copy.size;
      data = copy.data;
    } else {
      halt("illegal reallocation");
    }
  }

  def tupleInit(b: _tuple) {
    def _tupleInitHelp(j, param rank: int, b: _tuple) {
      if rank == 1 {
        for param i in 1..b.size {
          j(this.rank-rank+1) = dom.dim(this.rank-rank+1).low + i - 1;
          this(j) = b(i);
        }
      } else {
        for param i in 1..b.size {
          j(this.rank-rank+1) = dom.dim(this.rank-rank+1).low + i - 1;
          _tupleInitHelp(j, rank-1, b(i));
        }
      }
    }

    if rank == 1 {
      for param i in 1..b.size do
        this(this.dom.dim(1).low + i - 1) = b(i);
    } else {
      var j: rank*int;
      _tupleInitHelp(this, j, rank, b);
    }
  }
}

def CMODomain.writeThis(f: Writer) {
  f.write("[", this(1));
  for i in 2..rank do
    f.write(", ", this(i));
  f.write("]");
}

def CMOArray.writeThis(f: Writer) {
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dim(dim)._low;
  label next while true {
    f.write(this(i));
    if i(rank) <= (dom.dim(rank)._high - dom.dim(rank)._stride:idxType) {
      f.write(" ");
      i(rank) += dom.dim(rank)._stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dim(dim)._high - dom.dim(dim)._stride:idxType) {
          i(dim) += dom.dim(dim)._stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dim(dim2)._low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

def _intersect(a: CMODomain, b: CMODomain) {
  var c = new CMODomain(a.rank, a.idxType, stridable=a.stridable);
  for param i in 1..a.rank do
    c.ranges(i) = a.dim(i)(b.dim(i));
  return c;
}

def main() {
  param n = 5;
  const D = [1..n,1..n];
  const D2: domain(2) distributed(new CMODist()) = [1..n,1..n];
  var A: [ij in D2] real = ij(1) + (ij(2) - 1)*n;
  var B: [ij in D] real = ij(2) + (ij(1) - 1)*n;

  writeln('A by columns');
  for j in 1..n {
    writeln(A(1..n,j));
  }
  writeln();
  writeln('B by rows');
  for i in 1..n {
    writeln(B(i,1..n));
  }

  writeln();
  write('A = ');
  writeln(A);
  writeln();
  write('B = ');
  writeln(B);
}

class CMODist {
  def buildDomain(param rank: int, type dimensional_index_type, param stridable: bool) {
    return new CMODomain(rank=rank, idxType=dimensional_index_type, stridable=stridable);
  }
}

class CMODomain: BaseDomain {
  param rank : int;
  type idxType;
  param stridable: bool;
  var ranges : rank*range(idxType,BoundedRangeType.bounded,stridable);

  def getIndices() return ranges;

  def setIndices(x) {
    if ranges.size != x.size then
      compilerError("rank mismatch in domain assignment");
    if ranges(1).eltType != x(1).eltType then
      compilerError("index type mismatch in domain assignment");
    ranges = x;
  }

  def buildEmptyDomain()
    return new CMODomain(rank=rank, idxType=idxType, stridable=stridable);

  def buildOpenIntervalUpper() {
    var x = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    for param i in 1..rank {
      if ranges(i)._stride != 1 then
        halt("syntax [domain-specification) requires a stride of one");
      x.ranges(i) = ranges(i)._low..ranges(i)._high-1;
    }
    return x;
  }

  def these_help(param dim: int) {
    if dim == rank - 1 {
      for j in ranges(rank) do
        for i in ranges(dim) do
          yield (i, j);
    } else {
      for j in these_help(dim+1) do
        for i in ranges(dim) do
          yield (i, (...j));
    }
  }

  def these() {
    if rank == 1 {
      for i in ranges(1) do
        yield i;
    } else {
      for i in these_help(1) do
        yield i;
    }
  }

  def this(dim : int)
    return ranges(dim);

  def member(ind: idxType) where rank == 1 {
    if !ranges(1).member(ind) then
      return false;
    return true;
  }

  def member(ind: rank*idxType) {
    for param i in 1..rank do
      if !ranges(i).member(ind(i)) then
        return false;
    return true;
  }

  def dim(d : int)
    return ranges(d);

  def bbox(d: int) {
    const r: range(idxType,BoundedRangeType.bounded,false) = ranges(d);
    return r;
  }

  def numIndices {
    var sum = 1:idxType;
    for param i in 1..rank do
      sum *= ranges(i).length;
    return sum;
    // WANT: return * reduce (this(1..rank).length);
  }

  def low {
    if rank == 1 {
      return ranges(1)._low;
    } else {
      var result: rank*idxType;
      for param i in 1..rank do
        result(i) = ranges(i)._low;
      return result;
    }
  }

  def high {
    if rank == 1 {
      return ranges(1)._high;
    } else {
      var result: rank*idxType;
      for param i in 1..rank do
        result(i) = ranges(i)._high;
      return result;
    }
  }

  def buildArray(type eltType) {
    return new CMOArray(eltType, rank, idxType, stridable, dom=this);
  }

  def buildSubdomain()
    return new CMODomain(rank=rank, idxType=idxType, stridable=stridable);

  def rankChange(param rank: int, param stridable: bool, args) {
    def isRange(r: range(?e,?b,?s)) param return 1;
    def isRange(r) param return 0;

    var d = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    var i = 1;
    for param j in 1..args.size {
      if isRange(args(j)) {
        d.ranges(i) = dim(j)(args(j));
        i += 1;
      }
    }
    return d;
  }

  def translate(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable = stridable);
    for i in 1..rank do
      x.ranges(i) = dim(i)._translate(off(i));
    return x;
  }

  def interior(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do {
      if ((off(i) > 0) && (dim(i)._high+1-dim(i) < dim(i)._low) ||
          (off(i) < 0) && (dim(i)._low-1-dim(i) > dim(i)._high)) {
        halt("***Error: Argument to 'interior' function out of range in dimension ", i, "***");
      }
      x.ranges(i) = dim(i)._interior(off(i));
    }
    return x;
  }

  def exterior(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = dim(i)._exterior(off(i));
    return x;
  }

  def expand(off: rank*int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do {
      x.ranges(i) = ranges(i)._expand(off(i));
      if (x.ranges(i)._low > x.ranges(i)._high) {
        halt("***Error: Degenerate dimension created in dimension ", i, "***");
      }
    }
    return x;
  }

  def expand(off: int) {
    var x = new CMODomain(rank=rank, idxType=int, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = ranges(i)._expand(off);
    return x;
  }

  // this should eventually...
  //  ...be in the distribtion's implementation of the blocked domain
  // ...yield a subBlock of the domain per thread per locale
  def subBlocks {
    yield this;
  }

  def strideBy(str : rank*int) {
    var x = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = ranges(i) by str(i);
    return x;
  }

  def strideBy(str : int) {
    var x = new CMODomain(rank=rank, idxType=idxType, stridable=stridable);
    for i in 1..rank do
      x.ranges(i) = ranges(i) by str;
    return x;
  }
}

class CMOArray:BaseArray {
  type eltType;
  param rank: int;
  type idxType;
  param stridable: bool;
  param reindexed: bool = false;

  var dom: CMODomain(rank=rank,idxType=idxType, stridable=stridable);
  var off: rank*idxType;
  var blk: rank*idxType;
  var str: rank*int;
  var origin: idxType;
  var factoredOffs: idxType;
  var size: idxType;
  var D1: domain(1, idxType);
  var data: [D1] eltType;
  var noinit: bool = false;

  def computeFactoredOffs() {
    factoredOffs = 0:idxType;
    for i in 1..rank do {
      factoredOffs += blk(i) * off(i);
    }
  }

  def initialize() {
    if noinit == true then return;
    for param dim in 1..rank {
      off(dim) = dom.dim(dim)._low;
      str(dim) = dom.dim(dim)._stride;
    }
    blk(1) = 1:idxType;
    for dim in 2..rank do
      blk(dim) = blk(dim-1) * dom.dim(dim-1).length;
    computeFactoredOffs();
    size = blk(rank) * dom.dim(rank).length;
    D1 = [0:idxType..#size:idxType];
    data = 0:eltType;
  }

  def these() {
    for i in dom do {
      yield this(i);
    }
  }

  def this(ind: idxType ...1) var where rank == 1
    return this(ind);

  def this(ind : rank*idxType) var {
    if boundsChecking then
      if !dom.member(ind) then
        halt("array index out of bounds: ", ind);
    var sum = origin;
    if stridable {
      for param i in 1..rank do
        sum = sum + (ind(i) - off(i)) * blk(i) / str(i):idxType;
    } else {
      if reindexed {
        for param i in 1..rank do
          sum += ind(i) * blk(i);
      } else {
        for param i in 1..rank do
          sum += ind(i) * blk(i);
      }
      sum -= factoredOffs;
    }
    return data(sum);
  }

  def reindex(d: CMODomain) {
    if rank != d.rank then
      compilerError("illegal implicit rank change");
    for param i in 1..rank do
      if d.dim(i).length != dom.dim(i).length then
        halt("extent in dimension ", i, " does not match actual");
    var alias = new CMOArray(eltType, d.rank, d.idxType, d.stridable, true, d, noinit=true);
    //    was:  (eltType, rank, idxType, d.stridable, true, d, noinit=true);
    alias.D1 = [0:idxType..#size:idxType];
    alias.data = data;
    alias.size = size: d.idxType;
    for param i in 1..rank {
      alias.off(i) = d.dim(i)._low;
      alias.blk(i) = (blk(i) * dom.dim(i)._stride / str(i)) : d.idxType;
      alias.str(i) = d.dim(i)._stride;
    }
    alias.origin = origin:d.idxType;
    alias.computeFactoredOffs();
    return alias;
  }

  def checkSlice(d) {
    for param i in 1..rank {
      if d(i).boundedType == BoundedRangeType.bounded then
        if !dom.dim(i).member(d(i)) then
          halt("array slice out of bounds in dimension ", i, ": ", d(i));
    }
  }

  def slice(d: CMODomain) {
    var alias = new CMOArray(eltType, rank, idxType, d.stridable, reindexed, d, noinit=true);
    alias.D1 = [0:idxType..#size:idxType];
    alias.data = data;
    alias.size = size;
    alias.blk = blk;
    alias.str = str;
    alias.origin = origin;
    for param i in 1..rank {
      alias.off(i) = d.dim(i)._low;
      alias.origin += blk(i) *(d.dim(i)._low - off(i))/str(i);
    }
    alias.computeFactoredOffs();
    return alias;
  }

  def checkRankChange(args) {
    def isRange(r: range(?e,?b,?s)) param return 1;
    def isRange(r) param return 0;

    for param i in 1..args.size do
      if isRange(args(i)) then
        if args(i).boundedType == BoundedRangeType.bounded then
          if !dom.dim(i).member(args(i)) then
            halt("array slice out of bounds in dimension ", i, ": ", args(i));
  }

  def rankChange(param newRank: int, param newStridable: bool, irs) {
    def isRange(r: range(?e,?b,?s)) param return 1;
    def isRange(r) param return 0;
    var d = dom.rankChange(newRank, newStridable, irs);
    var alias = new CMOArray(eltType, newRank, idxType, newStridable, true, d, noinit=true);
    alias.D1 = [0:idxType..#size:idxType];
    alias.data = data;
    alias.size = size;
    var i = 1;
    alias.origin = origin;
    for param j in 1..irs.size {
      if isRange(irs(j)) {
        alias.off(i) = d.dim(i)._low;
        alias.origin += blk(j) * (d.dim(i)._low - off(j)) / str(j);
        alias.blk(i) = blk(j);
        alias.str(i) = str(j);
        i += 1;
      } else {
        alias.origin += blk(j) * (irs(j) - off(j)) / str(j);
      }
    }
    alias.computeFactoredOffs();
    return alias;
  }

  def reallocate(d: _domain) {
    if (d._value.type == dom.type) {
      var copy = new CMOArray(eltType, rank, idxType, d._value.stridable, reindexed, d._value);
      for i in _intersect(d._value, dom) do
        copy(i) = this(i);
      off = copy.off;
      blk = copy.blk;
      str = copy.str;
      origin = copy.origin;
      factoredOffs = copy.factoredOffs;
      size = copy.size;
      data = copy.data;
    } else {
      halt("illegal reallocation");
    }
  }

  def tupleInit(b: _tuple) {
    def _tupleInitHelp(j, param rank: int, b: _tuple) {
      if rank == 1 {
        for param i in 1..b.size {
          j(this.rank-rank+1) = dom.dim(this.rank-rank+1).low + i - 1;
          this(j) = b(i);
        }
      } else {
        for param i in 1..b.size {
          j(this.rank-rank+1) = dom.dim(this.rank-rank+1).low + i - 1;
          _tupleInitHelp(j, rank-1, b(i));
        }
      }
    }

    if rank == 1 {
      for param i in 1..b.size do
        this(this.dom.dim(1).low + i - 1) = b(i);
    } else {
      var j: rank*int;
      _tupleInitHelp(this, j, rank, b);
    }
  }
}

def CMODomain.writeThis(f: Writer) {
  f.write("[", this(1));
  for i in 2..rank do
    f.write(", ", this(i));
  f.write("]");
}

def CMOArray.writeThis(f: Writer) {
  var i : rank*idxType;
  for dim in 1..rank do
    i(dim) = dom.dim(dim)._low;
  label next while true {
    f.write(this(i));
    if i(rank) <= (dom.dim(rank)._high - dom.dim(rank)._stride:idxType) {
      f.write(" ");
      i(rank) += dom.dim(rank)._stride:idxType;
    } else {
      for dim in 1..rank-1 by -1 {
        if i(dim) <= (dom.dim(dim)._high - dom.dim(dim)._stride:idxType) {
          i(dim) += dom.dim(dim)._stride:idxType;
          for dim2 in dim+1..rank {
            f.writeln();
            i(dim2) = dom.dim(dim2)._low;
          }
          continue next;
        }
      }
      break;
    }
  }
}

def _intersect(a: CMODomain, b: CMODomain) {
  var c = new CMODomain(a.rank, a.idxType, stridable=a.stridable);
  for param i in 1..a.rank do
    c.ranges(i) = a.dim(i)(b.dim(i));
  return c;
}

def main() {
  param n = 5;
  const D = [1..n,1..n];
  const D2: domain(2) distributed(new CMODist()) = [1..n,1..n];
  var A: [ij in D2] real = ij(1) + (ij(2) - 1)*n;
  var B: [ij in D] real = ij(2) + (ij(1) - 1)*n;

  writeln('A by columns');
  for j in 1..n {
    writeln(A(1..n,j));
  }
  writeln();
  writeln('B by rows');
  for i in 1..n {
    writeln(B(i,1..n));
  }

  writeln();
  write('A = ');
  writeln(A);
  writeln();
  write('B = ');
  writeln(B);
}


Rewrote HPCC FFT in order to take advantage of strided counted ranges

Converted some ranges of the form:

  (0..#radix)*str + lo

into:

  lo.. by str #radix

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@14580 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use BitOps, Random, Time;

use HPCCProblemSize;

param radix = 4;

param numVectors = 2;
type elemType = complex(128);

config const n = computeProblemSize(elemType, numVectors, returnLog2 = true);
const m = 2**n;

config const epsilon = 2.0 ** -51.0,
             threshold = 16.0;

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

config const printParams = true,
             printArrays = false,
             printStats = true;

def main() {
  printConfiguration();

  const TwiddleDom: domain(1, int(64)) distributed(Block) = [0:int(64)..#m/4];
  var Twiddles: [TwiddleDom] elemType;

  const ProblemDom: domain(1, int(64)) distributed(Block) = [0:int(64)..#m];
  var Z, z: [ProblemDom] elemType;

  initVectors(Twiddles, z);

  const startTime = getCurrentTime();

  Z = conjg(z);
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  const execTime = getCurrentTime() - startTime;

  const validAnswer = verifyResults(z, Z, Twiddles);
  printResults(validAnswer, execTime);
}

def printConfiguration() {
  if (printParams) then printProblemSize(elemType, numVectors, m);
}

def initVectors(Twiddles, z) {
  computeTwiddles(Twiddles);
  bitReverseShuffle(Twiddles);

  fillRandom(z, seed);

  if (printArrays) {
    writeln("After initialization, Twiddles is: ", Twiddles, "\n");
    writeln("z is: ", z, "\n");
  }
}

def computeTwiddles(Twiddles) {
  const numTwdls = Twiddles.numElements,
        delta = 2.0 * atan(1.0) / numTwdls;

  Twiddles(0) = 1.0;
  Twiddles(numTwdls/2) = let x = cos(delta * numTwdls/2)
                          in (x, x):elemType;
  forall i in 1..numTwdls/2-1 {
    const x = cos(delta*i),
          y = sin(delta*i);
    Twiddles(i)            = (x, y):elemType;
    Twiddles(numTwdls - i) = (y, x):elemType;
  }
}

def bitReverseShuffle(Vect: [?Dom]) {
  const numBits = log2(Vect.numElements),
        Perm: [i in Dom] Vect.eltType = Vect(bitReverse(i, revBits=numBits));
  Vect = Perm;
}

def bitReverse(val: ?valType, revBits = 64) {
  param mask = 0x0102040810204080;
  const valReverse64 = bitMatMultOr(mask, bitMatMultOr(val:uint(64), mask)),
        valReverse = bitRotLeft(valReverse64, revBits);
  return valReverse: valType;
}

def dfft(A: [?ADom], W) {
  const numElements = A.numElements;

  for (str, span) in genDFTPhases(numElements, radix) {
    forall (bankStart, twidIndex) in (ADom by 2*span, 0..) {
      var wk2 = W(twidIndex),
          wk1 = W(2*twidIndex),
          wk3 = (wk1.re - 2 * wk2.im * wk1.im,
                 2 * wk2.im * wk1.re - wk1.im):elemType;

      forall lo in bankStart..#str do
        butterfly(wk1, wk2, wk3, A[lo.. by str #radix]);

      wk1 = W(2*twidIndex+1);
      wk3 = (wk1.re - 2 * wk2.re * wk1.im,
             2 * wk2.re * wk1.re - wk1.im):elemType;
      wk2 *= 1.0i;

      forall lo in bankStart+span..#str do
        butterfly(wk1, wk2, wk3, A[lo.. by str #radix]);
    }
  }

  const str = radix**log4(numElements-1);
  if (str*radix == numElements) then
    forall lo in 0..#str do
      butterfly(1.0, 1.0, 1.0, A[lo.. by str #radix]);
  else
    forall lo in 0..#str {
      const a = A(lo),
            b = A(lo+str);
      A(lo)     = a + b;
      A(lo+str) = a - b;
    }
}

def genDFTPhases(numElements, radix) {
  var stride = 1;
  for 1..log4(numElements-1) {
    const span = stride * radix;
    yield (stride, span);
    stride = span;
  }
}

def butterfly(wk1, wk2, wk3, A) {
  var X: [0..#radix] elemType = A;
  var x0 = X(0) + X(1),
      x1 = X(0) - X(1),
      x2 = X(2) + X(3),
      x3rot = (X(2) - X(3))*1.0i;

  X(0) = x0 + x2;
  x0 -= x2;
  X(2) = wk2 * x0;
  x0 = x1 + x3rot;
  X(1) = wk1 * x0;
  x0 = x1 - x3rot;
  X(3) = wk3 * x0;

  A = X;
}

def log4(x) return logBasePow2(x, 2);

def verifyResults(z, Z, Twiddles) {
  if (printArrays) then writeln("After FFT, Z is: ", Z, "\n");

  Z = conjg(Z) / m;
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  if (printArrays) then writeln("After inverse FFT, Z is: ", Z, "\n");

  var maxerr = max reduce sqrt((z.re - Z.re)**2 + (z.im - Z.im)**2);
  maxerr /= (epsilon * n);
  if (printStats) then writeln("error = ", maxerr);

  return (maxerr < threshold);
}

def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (Gflop/s) = ", 5 * (m * n / execTime) * 1e-9);
  }
}

use BitOps, Random, Time;

use HPCCProblemSize;

param radix = 4;

param numVectors = 2;
type elemType = complex(128);

config const n = computeProblemSize(elemType, numVectors, returnLog2 = true);
const m = 2**n;

config const epsilon = 2.0 ** -51.0,
             threshold = 16.0;

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

config const printParams = true,
             printArrays = false,
             printStats = true;

def main() {
  printConfiguration();

  const TwiddleDom: domain(1, int(64)) distributed(Block) = [0:int(64)..#m/4];
  var Twiddles: [TwiddleDom] elemType;

  const ProblemDom: domain(1, int(64)) distributed(Block) = [0:int(64)..#m];
  var Z, z: [ProblemDom] elemType;

  initVectors(Twiddles, z);

  const startTime = getCurrentTime();

  Z = conjg(z);
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  const execTime = getCurrentTime() - startTime;

  const validAnswer = verifyResults(z, Z, Twiddles);
  printResults(validAnswer, execTime);
}

def printConfiguration() {
  if (printParams) then printProblemSize(elemType, numVectors, m);
}

def initVectors(Twiddles, z) {
  computeTwiddles(Twiddles);
  bitReverseShuffle(Twiddles);

  fillRandom(z, seed);

  if (printArrays) {
    writeln("After initialization, Twiddles is: ", Twiddles, "\n");
    writeln("z is: ", z, "\n");
  }
}

def computeTwiddles(Twiddles) {
  const numTwdls = Twiddles.numElements,
        delta = 2.0 * atan(1.0) / numTwdls;

  Twiddles(0) = 1.0;
  Twiddles(numTwdls/2) = let x = cos(delta * numTwdls/2)
                          in (x, x):elemType;
  forall i in 1..numTwdls/2-1 {
    const x = cos(delta*i),
          y = sin(delta*i);
    Twiddles(i)            = (x, y):elemType;
    Twiddles(numTwdls - i) = (y, x):elemType;
  }
}

def bitReverseShuffle(Vect: [?Dom]) {
  const numBits = log2(Vect.numElements),
        Perm: [i in Dom] Vect.eltType = Vect(bitReverse(i, revBits=numBits));
  Vect = Perm;
}

def bitReverse(val: ?valType, revBits = 64) {
  param mask = 0x0102040810204080;
  const valReverse64 = bitMatMultOr(mask, bitMatMultOr(val:uint(64), mask)),
        valReverse = bitRotLeft(valReverse64, revBits);
  return valReverse: valType;
}

def dfft(A: [?ADom], W) {
  const numElements = A.numElements;

  for (str, span) in genDFTPhases(numElements, radix) {
    forall (bankStart, twidIndex) in (ADom by 2*span, 0..) {
      var wk2 = W(twidIndex),
          wk1 = W(2*twidIndex),
          wk3 = (wk1.re - 2 * wk2.im * wk1.im,
                 2 * wk2.im * wk1.re - wk1.im):elemType;

      forall lo in bankStart..#str do
        butterfly(wk1, wk2, wk3, A[(0..#radix)*str + lo]); // BLC: want strided#

      wk1 = W(2*twidIndex+1);
      wk3 = (wk1.re - 2 * wk2.re * wk1.im,
             2 * wk2.re * wk1.re - wk1.im):elemType;
      wk2 *= 1.0i;

      forall lo in bankStart+span..#str do
        butterfly(wk1, wk2, wk3, A[(0..#radix)*str + lo]); // BLC: want strided#
    }
  }

  const str = radix**log4(numElements-1);
  if (str*radix == numElements) then
    forall lo in 0..#str do
      butterfly(1.0, 1.0, 1.0, A[(0..#radix)*str + lo]);
  else
    forall lo in 0..#str {
      const a = A(lo),
            b = A(lo+str);
      A(lo)     = a + b;
      A(lo+str) = a - b;
    }
}

def genDFTPhases(numElements, radix) {
  var stride = 1;
  for 1..log4(numElements-1) {
    const span = stride * radix;
    yield (stride, span);
    stride = span;
  }
}

def butterfly(wk1, wk2, wk3, A) {
  var X: [0..#radix] elemType = A;
  var x0 = X(0) + X(1),
      x1 = X(0) - X(1),
      x2 = X(2) + X(3),
      x3rot = (X(2) - X(3))*1.0i;

  X(0) = x0 + x2;
  x0 -= x2;
  X(2) = wk2 * x0;
  x0 = x1 + x3rot;
  X(1) = wk1 * x0;
  x0 = x1 - x3rot;
  X(3) = wk3 * x0;

  A = X;
}

def log4(x) return logBasePow2(x, 2);

def verifyResults(z, Z, Twiddles) {
  if (printArrays) then writeln("After FFT, Z is: ", Z, "\n");

  Z = conjg(Z) / m;
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  if (printArrays) then writeln("After inverse FFT, Z is: ", Z, "\n");

  var maxerr = max reduce sqrt((z.re - Z.re)**2 + (z.im - Z.im)**2);
  maxerr /= (epsilon * n);
  if (printStats) then writeln("error = ", maxerr);

  return (maxerr < threshold);
}

def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (Gflop/s) = ", 5 * (m * n / execTime) * 1e-9);
  }
}


Remove "no doc" pragmas now that chpldoc recognizes and ignores private symbols

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  This module provides support for reporting and counting
  communication operations between network-connected locales.  The
  operations include various kinds of remote reads (GETs), remote
  writes (PUTs), and remote forks (which are actually more like remote
  procedure calls, and are used to implement ``on`` statements).
  Callers can request on-the-fly output each time a remote operation
  occurs, or count such operations as they occur and retrieve the
  counts later.  The former gives more detailed information but has
  much more overhead.  The latter has much less overhead but only
  provides aggregate information.

  **On-the-fly Reporting**

  All forms of communication reporting and counting are done between
  pairs of function calls that turn it on and off.  On-the-fly
  reporting across all locales is done like this::

    startVerboseComm();
    // between start/stop calls, report comm ops initiated on any locale
    stopVerboseComm();

  On-the-fly reporting for just the calling locale is similar.  Only
  the procedure names change::

    startVerboseCommHere();
    // between start/stop calls, report comm ops initiated on this locale
    stopVerboseCommHere();

  In either case, the output produced consists of a line written to
  ``stdout`` for each communication operation.  (Here ``stdout`` means
  the file associated with the process, not the Chapel channel with the
  same name.)

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startVerboseComm();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopVerboseComm();
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    0: remote task created on 1
    1: t.chpl:6: remote get from 0
    1: t.chpl:6: remote put to 0

  The initial number refers to the locale reporting the communication
  event.  The file name and line number point to the place in the
  code that triggered the communication event.  (For remote forks,
  file name and line number information is not yet reported.)

  **Counting Communication Operations**

  Counting communication operations requires a few more calls then just
  reporting them does.  In particular, the counts have to be retrieved
  after they are collected and, if they have been used previously, the
  internal counters have to be reset before counting is turned on.
  Counting across all locales is done like this::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnostics();
    startCommDiagnostics();
    // between start/stop calls, count comm ops initiated on any locale
    stopCommDiagnostics();
    // retrieve the counts and report the results
    writeln(getCommDiagnostics());

  Counting on just the calling locale is similar.  Just as for
  on-the-fly reporting, only the procedure names change::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnosticsHere();
    startCommDiagnosticsHere();
    // between start/stop calls, count comm ops initiated on this locale
    stopCommDiagnosticsHere();
    // retrieve the counts and report the results
    writeln(getCommDiagnosticsHere());

  The optional call to reset the counters is only needed when a program
  collects counts more than once.  In this case, the counters have to
  be set back to zero before starting the second and succeeding
  counting periods.  By far the most common situation is that programs
  only collect communication counts once per run, in which case this
  step is not needed.

  Note that the same internal mechanisms and counters are used for
  counting on all locales and counting on just the calling locale, so
  trying to do both at once may lead to surprising turn-on/turn-off
  behavior and/or incorrect results.

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startCommDiagnostics();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopCommDiagnostics();
      writeln(getCommDiagnostics());
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    (get = 0, get_nb = 0, put = 0, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 1, fork_fast = 0, fork_nb = 0) (get = 1, get_nb = 0, put = 1, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 0, fork_fast = 0, fork_nb = 0)

  The first parenthesized group contains the counts for locale 0, and
  the second contains the counts for locale 1.  So, for the
  instrumented section of this program we can say that a remote fork
  was executed on locale 0, and a remote get and a remote put were
  executed on locale 1.

  **Studying Communication During Module Initialization**

  It is hard for a programmer to determine exactly what happens during
  initialization or teardown of a module, because the code that runs
  then does so only implicitly, as a result of the declarations
  present.  And even if that code can be identified, doing debug
  output or logging data for later reporting might not work because
  the Chapel capabilities needed to do so could be unavailable due to
  being implemented by built-in modules which themselves are not yet
  initialized, or have already been torn down.

  To help with that problem, this module provides built-in support for
  studying communication operations during module initialization and
  teardown.  To use it, set either or both of the config params
  :param:`printInitVerboseComm` and :param:`printInitCommCounts`,
  described below.  You can do this by using appropriate
  ``-sconfigParamName=value`` command line options when you compile
  your program.

  The reporting and/or counting enabled by these covers all of program
  execution, from just before the first module is initialized until
  just after the last one is torn down.  This is almost always a
  superset of the part of the program that is of interest, which is
  often just a single module.  To learn what communication is being
  done by a single module during its initialization and teardown it is
  often necessary to run a small test program twice, once with that
  module present and once without it.
 */
module CommDiagnostics
{
  // There should be a type like this declared in chpl-comm.h with a single
  // function that returns the C struct.  We're not doing it that way yet
  // due to some shortcomings in our extern records implementation.
  // Once that gets sorted out, we can turn this into an extern record,
  // and remove the 8 or so individual functions below that return the
  // various counters.

  /* Aggregated communication operation counts.  This record type is
     defined in the same way by both the underlying comm layer(s) and
     this module, because we don't have a good way to inherit types back
     and forth between the two.  This first definition duplicates the
     one in the comm layer(s).
   */
  extern record chpl_commDiagnostics {
    /*
      blocking GETs, in which initiator waits for completion
     */
    var get: uint(64);
    /*
      non-blocking GETs
     */
    var get_nb: uint(64);
    /*
      blocking PUTs, in which initiator waits for completion
     */
    var put: uint(64);
    /*
      non-blocking PUTs
     */
    var put_nb: uint(64);
    /*
      tests for non-blocking GET/PUT completions
     */
    var test_nb: uint(64);
    /*
      blocking waits for non-blocking GET/PUT completions
     */
    var wait_nb: uint(64);
    /*
      non-blocking waits for non-blocking GET/PUT completions
     */
    var try_nb: uint(64);
    /*
      blocking remote executions, in which initiator waits for completion
     */
    var fork: uint(64);
    /*
      blocking remote executions performed by the target locale's
      Active Message handler
     */
    var fork_fast: uint(64);
    /*
      non-blocking remote executions
     */
    var fork_nb: uint(64);
  };

  /*
    The Chapel record type inherits the comm layer definition of it.
   */
  type commDiagnostics = chpl_commDiagnostics;

  private extern proc chpl_startVerboseComm();

  private extern proc chpl_stopVerboseComm();

  private extern proc chpl_startVerboseCommHere();

  private extern proc chpl_stopVerboseCommHere();

  private extern proc chpl_gen_startCommDiagnostics();

  private extern proc chpl_gen_stopCommDiagnostics();

  private extern proc chpl_gen_startCommDiagnosticsHere();

  private extern proc chpl_gen_stopCommDiagnosticsHere();

  private extern proc chpl_resetCommDiagnosticsHere();

  private extern proc chpl_getCommDiagnosticsHere(out cd: commDiagnostics);

  /*
    Start on-the-fly reporting of communication initiated on any locale.
   */
  proc startVerboseComm() { chpl_startVerboseComm(); }

  /*
    Stop on-the-fly reporting of communication initiated on any locale.
   */
  proc stopVerboseComm() { chpl_stopVerboseComm(); }

  /*
    Start on-the-fly reporting of communication initiated on this locale.
   */
  proc startVerboseCommHere() { chpl_startVerboseCommHere(); }

  /*
    Stop on-the-fly reporting of communication initiated on this locale.
   */
  proc stopVerboseCommHere() { chpl_stopVerboseCommHere(); }

  /*
    Start counting communication operations across the whole program.
   */
  proc startCommDiagnostics() {
    chpl_gen_startCommDiagnostics();
  }

  /*
    Stop counting communication operations across the whole program.
   */
  proc stopCommDiagnostics() {
    chpl_gen_stopCommDiagnostics();
  }

  /*
    Start counting communication operations initiated on this locale.
   */
  proc startCommDiagnosticsHere() {
    chpl_gen_startCommDiagnosticsHere();
  }

  /*
    Stop counting communication operations initiated on this locale.
   */
  proc stopCommDiagnosticsHere() {
    chpl_gen_stopCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts across the whole program.
   */
  proc resetCommDiagnostics() {
    for loc in Locales do on loc do
      resetCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts on the calling locale.
   */
  inline proc resetCommDiagnosticsHere() {
    chpl_resetCommDiagnosticsHere();
  }

  // See note above regarding extern records
  private extern proc chpl_numCommGets(): uint(64);

  private extern proc chpl_numCommNBGets(): uint(64);

  private extern proc chpl_numCommPuts(): uint(64);

  private extern proc chpl_numCommNBPuts(): uint(64);

  private extern proc chpl_numCommTestNB(): uint(64);

  private extern proc chpl_numCommWaitNB(): uint(64);

  private extern proc chpl_numCommTryNB(): uint(64);

  private extern proc chpl_numCommForks(): uint(64);

  private extern proc chpl_numCommFastForks(): uint(64);

  private extern proc chpl_numCommNBForks(): uint(64);

  /*
    Retrieve aggregate communication counts for the whole program.

    :returns: array of counts of comm ops initiated on each locale
    :rtype: `[LocaleSpace] commDiagnostics`
   */
  proc getCommDiagnostics() {
    var D: [LocaleSpace] commDiagnostics;
    for loc in Locales do on loc {
      // See note above regarding extern records
      D(loc.id).get = chpl_numCommGets();
      D(loc.id).get_nb = chpl_numCommNBGets();
      D(loc.id).put = chpl_numCommPuts();
      D(loc.id).put_nb = chpl_numCommNBPuts();
      D(loc.id).test_nb = chpl_numCommTestNB();
      D(loc.id).wait_nb = chpl_numCommWaitNB();
      D(loc.id).try_nb = chpl_numCommTryNB();
      D(loc.id).fork = chpl_numCommForks();
      D(loc.id).fork_fast = chpl_numCommFastForks();
      D(loc.id).fork_nb = chpl_numCommNBForks();
    }
    return D;
  }

  /*
    Retrieve aggregate communication counts for this locale.

    :returns: counts of comm ops initiated on this locale
    :rtype: `commDiagnostics`
   */
  proc getCommDiagnosticsHere() {
    var cd: commDiagnostics;
    cd.get = chpl_numCommGets();
    cd.get_nb = chpl_numCommNBGets();
    cd.put = chpl_numCommPuts();
    cd.put_nb = chpl_numCommNBPuts();
    cd.test_nb = chpl_numCommTestNB();
    cd.wait_nb = chpl_numCommWaitNB();
    cd.try_nb = chpl_numCommTryNB();
    cd.fork = chpl_numCommForks();
    cd.fork_fast = chpl_numCommFastForks();
    cd.fork_nb = chpl_numCommNBForks();
    return cd;
  }

  /*
    If this is set, on-the-fly reporting of communication operations
    will be turned on before any module initialization begins and
    turned off after all module teardown ends.  See procedures
    :proc:`startVerboseComm` and :proc:`stopVerboseComm` for more
    information.
   */
  config param printInitVerboseComm = false;

  /*
    If this is set, communication operations are counted from before
    any module initialization begins until after all module teardown
    ends, and then the aggregate counts are printed.  See procedures
    :proc:`startCommDiagnostics`, :proc:`stopCommDiagnostics`, and
    :proc:`getCommDiagnostics` for more information.
   */
  config param printInitCommCounts = false;

}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  This module provides support for reporting and counting
  communication operations between network-connected locales.  The
  operations include various kinds of remote reads (GETs), remote
  writes (PUTs), and remote forks (which are actually more like remote
  procedure calls, and are used to implement ``on`` statements).
  Callers can request on-the-fly output each time a remote operation
  occurs, or count such operations as they occur and retrieve the
  counts later.  The former gives more detailed information but has
  much more overhead.  The latter has much less overhead but only
  provides aggregate information.

  **On-the-fly Reporting**

  All forms of communication reporting and counting are done between
  pairs of function calls that turn it on and off.  On-the-fly
  reporting across all locales is done like this::

    startVerboseComm();
    // between start/stop calls, report comm ops initiated on any locale
    stopVerboseComm();

  On-the-fly reporting for just the calling locale is similar.  Only
  the procedure names change::

    startVerboseCommHere();
    // between start/stop calls, report comm ops initiated on this locale
    stopVerboseCommHere();

  In either case, the output produced consists of a line written to
  ``stdout`` for each communication operation.  (Here ``stdout`` means
  the file associated with the process, not the Chapel channel with the
  same name.)

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startVerboseComm();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopVerboseComm();
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    0: remote task created on 1
    1: t.chpl:6: remote get from 0
    1: t.chpl:6: remote put to 0

  The initial number refers to the locale reporting the communication
  event.  The file name and line number point to the place in the
  code that triggered the communication event.  (For remote forks,
  file name and line number information is not yet reported.)

  **Counting Communication Operations**

  Counting communication operations requires a few more calls then just
  reporting them does.  In particular, the counts have to be retrieved
  after they are collected and, if they have been used previously, the
  internal counters have to be reset before counting is turned on.
  Counting across all locales is done like this::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnostics();
    startCommDiagnostics();
    // between start/stop calls, count comm ops initiated on any locale
    stopCommDiagnostics();
    // retrieve the counts and report the results
    writeln(getCommDiagnostics());

  Counting on just the calling locale is similar.  Just as for
  on-the-fly reporting, only the procedure names change::

    // (optional) if we counted previously, reset the counters to zero
    resetCommDiagnosticsHere();
    startCommDiagnosticsHere();
    // between start/stop calls, count comm ops initiated on this locale
    stopCommDiagnosticsHere();
    // retrieve the counts and report the results
    writeln(getCommDiagnosticsHere());

  The optional call to reset the counters is only needed when a program
  collects counts more than once.  In this case, the counters have to
  be set back to zero before starting the second and succeeding
  counting periods.  By far the most common situation is that programs
  only collect communication counts once per run, in which case this
  step is not needed.

  Note that the same internal mechanisms and counters are used for
  counting on all locales and counting on just the calling locale, so
  trying to do both at once may lead to surprising turn-on/turn-off
  behavior and/or incorrect results.

  Consider this little example program::

    use CommDiagnostics;
    proc main() {
      startCommDiagnostics();
      var x: int = 1;
      on Locales(1) {     // should fork a blocking task onto locale 1
        x = x + 1;        // should invoke a remote put and a remote get
      }
      stopCommDiagnostics();
      writeln(getCommDiagnostics());
    }

  Executing this on two locales with the ``-nl 2`` command line
  option results in the following output::

    (get = 0, get_nb = 0, put = 0, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 1, fork_fast = 0, fork_nb = 0) (get = 1, get_nb = 0, put = 1, put_nb = 0, test_nb = 0, wait_nb = 0, try_nb = 0, fork = 0, fork_fast = 0, fork_nb = 0)

  The first parenthesized group contains the counts for locale 0, and
  the second contains the counts for locale 1.  So, for the
  instrumented section of this program we can say that a remote fork
  was executed on locale 0, and a remote get and a remote put were
  executed on locale 1.

  **Studying Communication During Module Initialization**

  It is hard for a programmer to determine exactly what happens during
  initialization or teardown of a module, because the code that runs
  then does so only implicitly, as a result of the declarations
  present.  And even if that code can be identified, doing debug
  output or logging data for later reporting might not work because
  the Chapel capabilities needed to do so could be unavailable due to
  being implemented by built-in modules which themselves are not yet
  initialized, or have already been torn down.

  To help with that problem, this module provides built-in support for
  studying communication operations during module initialization and
  teardown.  To use it, set either or both of the config params
  :param:`printInitVerboseComm` and :param:`printInitCommCounts`,
  described below.  You can do this by using appropriate
  ``-sconfigParamName=value`` command line options when you compile
  your program.

  The reporting and/or counting enabled by these covers all of program
  execution, from just before the first module is initialized until
  just after the last one is torn down.  This is almost always a
  superset of the part of the program that is of interest, which is
  often just a single module.  To learn what communication is being
  done by a single module during its initialization and teardown it is
  often necessary to run a small test program twice, once with that
  module present and once without it.
 */
module CommDiagnostics
{
  // There should be a type like this declared in chpl-comm.h with a single
  // function that returns the C struct.  We're not doing it that way yet
  // due to some shortcomings in our extern records implementation.
  // Once that gets sorted out, we can turn this into an extern record,
  // and remove the 8 or so individual functions below that return the
  // various counters.

  /* Aggregated communication operation counts.  This record type is
     defined in the same way by both the underlying comm layer(s) and
     this module, because we don't have a good way to inherit types back
     and forth between the two.  This first definition duplicates the
     one in the comm layer(s).
   */
  extern record chpl_commDiagnostics {
    /*
      blocking GETs, in which initiator waits for completion
     */
    var get: uint(64);
    /*
      non-blocking GETs
     */
    var get_nb: uint(64);
    /*
      blocking PUTs, in which initiator waits for completion
     */
    var put: uint(64);
    /*
      non-blocking PUTs
     */
    var put_nb: uint(64);
    /*
      tests for non-blocking GET/PUT completions
     */
    var test_nb: uint(64);
    /*
      blocking waits for non-blocking GET/PUT completions
     */
    var wait_nb: uint(64);
    /*
      non-blocking waits for non-blocking GET/PUT completions
     */
    var try_nb: uint(64);
    /*
      blocking remote executions, in which initiator waits for completion
     */
    var fork: uint(64);
    /*
      blocking remote executions performed by the target locale's
      Active Message handler
     */
    var fork_fast: uint(64);
    /*
      non-blocking remote executions
     */
    var fork_nb: uint(64);
  };

  /*
    The Chapel record type inherits the comm layer definition of it.
   */
  type commDiagnostics = chpl_commDiagnostics;

  pragma "no doc"
  private extern proc chpl_startVerboseComm();

  pragma "no doc"
  private extern proc chpl_stopVerboseComm();

  pragma "no doc"
  private extern proc chpl_startVerboseCommHere();

  pragma "no doc"
  private extern proc chpl_stopVerboseCommHere();

  pragma "no doc"
  private extern proc chpl_gen_startCommDiagnostics();

  pragma "no doc"
  private extern proc chpl_gen_stopCommDiagnostics();

  pragma "no doc"
  private extern proc chpl_gen_startCommDiagnosticsHere();

  pragma "no doc"
  private extern proc chpl_gen_stopCommDiagnosticsHere();

  pragma "no doc"
  private extern proc chpl_resetCommDiagnosticsHere();

  pragma "no doc"
  private extern proc chpl_getCommDiagnosticsHere(out cd: commDiagnostics);

  /*
    Start on-the-fly reporting of communication initiated on any locale.
   */
  proc startVerboseComm() { chpl_startVerboseComm(); }

  /*
    Stop on-the-fly reporting of communication initiated on any locale.
   */
  proc stopVerboseComm() { chpl_stopVerboseComm(); }

  /*
    Start on-the-fly reporting of communication initiated on this locale.
   */
  proc startVerboseCommHere() { chpl_startVerboseCommHere(); }

  /*
    Stop on-the-fly reporting of communication initiated on this locale.
   */
  proc stopVerboseCommHere() { chpl_stopVerboseCommHere(); }

  /*
    Start counting communication operations across the whole program.
   */
  proc startCommDiagnostics() {
    chpl_gen_startCommDiagnostics();
  }

  /*
    Stop counting communication operations across the whole program.
   */
  proc stopCommDiagnostics() {
    chpl_gen_stopCommDiagnostics();
  }

  /*
    Start counting communication operations initiated on this locale.
   */
  proc startCommDiagnosticsHere() {
    chpl_gen_startCommDiagnosticsHere();
  }

  /*
    Stop counting communication operations initiated on this locale.
   */
  proc stopCommDiagnosticsHere() {
    chpl_gen_stopCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts across the whole program.
   */
  proc resetCommDiagnostics() {
    for loc in Locales do on loc do
      resetCommDiagnosticsHere();
  }

  /*
    Reset aggregate communication counts on the calling locale.
   */
  inline proc resetCommDiagnosticsHere() {
    chpl_resetCommDiagnosticsHere();
  }

  // See note above regarding extern records
  pragma "no doc"
  private extern proc chpl_numCommGets(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommNBGets(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommPuts(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommNBPuts(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommTestNB(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommWaitNB(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommTryNB(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommForks(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommFastForks(): uint(64);

  pragma "no doc"
  private extern proc chpl_numCommNBForks(): uint(64);

  /*
    Retrieve aggregate communication counts for the whole program.

    :returns: array of counts of comm ops initiated on each locale
    :rtype: `[LocaleSpace] commDiagnostics`
   */
  proc getCommDiagnostics() {
    var D: [LocaleSpace] commDiagnostics;
    for loc in Locales do on loc {
      // See note above regarding extern records
      D(loc.id).get = chpl_numCommGets();
      D(loc.id).get_nb = chpl_numCommNBGets();
      D(loc.id).put = chpl_numCommPuts();
      D(loc.id).put_nb = chpl_numCommNBPuts();
      D(loc.id).test_nb = chpl_numCommTestNB();
      D(loc.id).wait_nb = chpl_numCommWaitNB();
      D(loc.id).try_nb = chpl_numCommTryNB();
      D(loc.id).fork = chpl_numCommForks();
      D(loc.id).fork_fast = chpl_numCommFastForks();
      D(loc.id).fork_nb = chpl_numCommNBForks();
    }
    return D;
  }

  /*
    Retrieve aggregate communication counts for this locale.

    :returns: counts of comm ops initiated on this locale
    :rtype: `commDiagnostics`
   */
  proc getCommDiagnosticsHere() {
    var cd: commDiagnostics;
    cd.get = chpl_numCommGets();
    cd.get_nb = chpl_numCommNBGets();
    cd.put = chpl_numCommPuts();
    cd.put_nb = chpl_numCommNBPuts();
    cd.test_nb = chpl_numCommTestNB();
    cd.wait_nb = chpl_numCommWaitNB();
    cd.try_nb = chpl_numCommTryNB();
    cd.fork = chpl_numCommForks();
    cd.fork_fast = chpl_numCommFastForks();
    cd.fork_nb = chpl_numCommNBForks();
    return cd;
  }

  /*
    If this is set, on-the-fly reporting of communication operations
    will be turned on before any module initialization begins and
    turned off after all module teardown ends.  See procedures
    :proc:`startVerboseComm` and :proc:`stopVerboseComm` for more
    information.
   */
  config param printInitVerboseComm = false;

  /*
    If this is set, communication operations are counted from before
    any module initialization begins until after all module teardown
    ends, and then the aggregate counts are printed.  See procedures
    :proc:`startCommDiagnostics`, :proc:`stopCommDiagnostics`, and
    :proc:`getCommDiagnostics` for more information.
   */
  config param printInitCommCounts = false;

}


Fixing a path

Fixing a path

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@23142 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

/*
  chpldoc Primer

  This primer covers the use of chpldoc to document source code. For further
  information, please see $CHPL_HOME/doc/technotes/README.chpldoc

*/

// Creating and accessing the documentation:

// To generate the documentation for this and any source file, you may either
// compile using the command 'chpldoc <sourceName.chpl>' for documentation only
// or using the command 'chpl --docs <sourceName.chpl>' to create both
// documentation and an executable.

// The documentation files will be found in a special folder created by the
// chpldoc tool.  If you are compiling a Chapel file that is deeper than your
// current directory, this heirarchy will be reflected within the documentation
// folder.
// (i.e. foo/bar.chpl will generate documentation within docs/foo/)

// Documentation within the code:

/*
  Multiline comments found before a method are associated with that method
  as long as there are no code blocks between them.
*/

proc commented(val: int): string {
  var totalNum = val + 4;
  var str = "hello agent " + totalNum;
  return str;
}

/*
  The method can be a stub and still output its comment
*/
proc stub(val): bool {
  /* Comments within the method body are ignored */
}

// Single line comments are also ignored.  However, this does not prevent
// the display of the method itself.
proc uncommented() {
  /*
    Longer comments within the method body are also ignored
  */
}

/*
  If the source code defines a module, it can also have a comment associated
  with it
*/
module Defined {

  /*
    And classes can display their comments, too
  */
  class Foo {
    /*
      Including the comments associated with their fields
    */
    var a: int;

    /*
      And class specific methods
    */
    proc getA(): int {
      return a;
    }
  }

  /*
    If a class inherits from another class, those fields, methods, and their
    comments are output in both the super and the subclass
  */
  class Bar : Foo {

    proc setA(newVal: int) {
      a = newVal;
    }
  }
}

/*
  chpldoc Primer

  This primer covers the use of chpldoc to document source code. For further
  information, please see $CHPL_HOME/docs/technotes/README.chpldoc

*/

// Creating and accessing the documentation:

// To generate the documentation for this and any source file, you may either
// compile using the command 'chpldoc <sourceName.chpl>' for documentation only
// or using the command 'chpl --docs <sourceName.chpl>' to create both
// documentation and an executable.

// The documentation files will be found in a special folder created by the
// chpldoc tool.  If you are compiling a Chapel file that is deeper than your
// current directory, this heirarchy will be reflected within the documentation
// folder.
// (i.e. foo/bar.chpl will generate documentation within docs/foo/)

// Documentation within the code:

/*
  Multiline comments found before a method are associated with that method
  as long as there are no code blocks between them.
*/

proc commented(val: int): string {
  var totalNum = val + 4;
  var str = "hello agent " + totalNum;
  return str;
}

/*
  The method can be a stub and still output its comment
*/
proc stub(val): bool {
  /* Comments within the method body are ignored */
}

// Single line comments are also ignored.  However, this does not prevent
// the display of the method itself.
proc uncommented() {
  /*
    Longer comments within the method body are also ignored
  */
}

/*
  If the source code defines a module, it can also have a comment associated
  with it
*/
module Defined {

  /*
    And classes can display their comments, too
  */
  class Foo {
    /*
      Including the comments associated with their fields
    */
    var a: int;

    /*
      And class specific methods
    */
    proc getA(): int {
      return a;
    }
  }

  /*
    If a class inherits from another class, those fields, methods, and their
    comments are output in both the super and the subclass
  */
  class Bar : Foo {

    proc setA(newVal: int) {
      a = newVal;
    }
  }
}