Allows guided and adaptive iterators to parallelize when amount of work is less than the number of tasks, instead of simply serializing. See 21813 for additional details. Also moves serialization warnings inside debug conditional.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@21822 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

config param debugAdvancedIters:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  assert(chunkSize > 0); // caller's responsibility

  use UtilMath;
  // # of tasks the range can fill. (fast) ceil so all work is represented
  const chunkTasks = divceilpos(c.length, chunkSize): int;

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks = min(chunkTasks, defaultNumTasks(numTasks));
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#nTasks do {
     while moreWork do {
      // There is local work in remain
      splitLock$; // Perform the splitting in a critical section
      const current:rType=splitChunk(remain, chunkSize, moreWork);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType= densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=min(c.length, defaultNumTasks(numTasks));
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=nTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#nTasks do {
     while undone do {
      // There is local work in remain(tid)
      splitLock$; // Perform the splitting in a critical section
      const current:rType=adaptSplit(remain, factor, undone);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType = densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    compilerError("methodStealing value must be between 0 and 2");

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=min(c.length, defaultNumTasks(numTasks));
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield (totalRange,);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#nTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if debugAdvancedIters then
        writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
        // There is local work
        const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
        if zeroBasedIters.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
          yield (zeroBasedIters,);
        }
      }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
      if debugAdvancedIters then
        writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

      // Perform the spliting at the victim remaining range

      select methodStealing {
        when 0 do { // Steals from the victim until it exhausts its range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                //after splitting from a victim range
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }

        when 1 do { // Steals once from victim. If it has sucess on the stealing,
                    // it chooses another victim and tries to steal again.

          if moreLocalWork[victim] then {
            // There is work in victim
            const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                 //after splitting from a victim range
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          } else {
            stealFailed=true;
          }
        }
        otherwise { // Tt steals from the victim, but now splitting
                    //from the tail of the victim's range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
                                // after splitting from a victim range:
                               // now we set splitTatil to true (second parameter in the call)
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }
      }
      // If here, then it can have failed the stealing intent at the victim (method 1),
      // or we have exhausted the victim range (methods 0, 2)

      if (methodStealing==0 || methodStealing==2 || (methodStealing==1 && stealFailed)) then {
        nVisitedVictims += 1; // Signal that there is no more work in victim
        if debugAdvancedIters then
          writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        stealFailed=false;
      }
      // Check if there is no more work
      if nVisitedVictims >= nTasks-1 then
        moreWork=false; // Signal that there is no more work in any victim
      else {  // There can be still work in other victim
        victim=(victim+1) % nTasks; // New victim to steal
        if methodStealing==1 then
          if victim==tid then victim=(victim+1) % nTasks;
      }
    }
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/nTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==nTasks-1 then
      rLocal=c#(size*(nTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==nTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.numCores;
      else dnTasks=dataParTasksPerLocale;
  } else if nTasks<0 then {
    halt("'numTasks' is negative");
  }
  return dnTasks;
}

proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}

config param debugAdvancedIters:bool=false;

//************************* Dynamic iterator

// Serial iterator

iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator

iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  assert(chunkSize > 0); // caller's responsibility

  use UtilMath;
  // # of tasks the range can fill. (fast) ceil so all work is represented
  const chunkTasks = divceilpos(c.length, chunkSize): int;

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks = min(chunkTasks, defaultNumTasks(numTasks));
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Dynamic Iterator: serial execution because there is not enough work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);

  }
  else {
    var remain:rType=c;
    var moreWork=true;
    var splitLock$:sync bool=true;   // sync variables to control the splitting

    coforall tid in 0..#nTasks do {
     while moreWork do {
      // There is local work in remain
      splitLock$; // Perform the splitting in a critical section
      const current:rType=splitChunk(remain, chunkSize, moreWork);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType= densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator
//Serial iterator
iter guided(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

// Leader iterator
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Guided Iterator: serial execution because there is not enoguh work");
    const totalRange:rType= densify(c,c);
    yield (totalRange,);
  }

  else {
    var remain:rType=c;
    var undone=true;
    const factor=nTasks;
    var splitLock$:sync bool=true;  // synnc variables to control the splitting

    coforall tid in 0..#nTasks do {
     while undone do {
      // There is local work in remain(tid)
      splitLock$; // Perform the splitting in a critical section
      const current:rType=adaptSplit(remain, factor, undone);
      splitLock$=true;
      if current.length !=0 then {
        const zeroBasedIters:rType = densify(current,c);
        if debugAdvancedIters then
          writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", current, " yielded as ", zeroBasedIters);
        yield (zeroBasedIters,);
      }
     }
    }
  }
}

// Follower
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
//Serial iterator
iter adaptive(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator

config param methodStealing:int=0;

// Leader iterator
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  if (methodStealing > 2 || methodStealing < 0) then
    compilerError("methodStealing value must be between 0 and 2");

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=defaultNumTasks(numTasks);
  type rType=c.type;
  // Check the size and do it serial if not enough work
  if c.length == 0 then halt("The range is empty");
  if c.length < nTasks then {
    writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    const totalRange:rType = densify(c,c);
    yield (totalRange,);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var splitLock$:[SpaceThreads] sync bool=true; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrierCount$:sync int=0;
    var wait$:single bool;

    // Start the parallel work
    coforall tid in 0..#nTasks do {

      // Step 1: Initial range per Thread/Task

      localWork[tid]=splitRange(r,tid);  // Initial Local range in localWork[tid]
      if debugAdvancedIters then
        writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier;

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
        // There is local work
        const zeroBasedIters:rType=splitWork(tid); // The current range we get after splitting locally
        if zeroBasedIters.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
          yield (zeroBasedIters,);
        }
      }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
      if debugAdvancedIters then
        writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

      // Perform the spliting at the victim remaining range

      select methodStealing {
        when 0 do { // Steals from the victim until it exhausts its range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                //after splitting from a victim range
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }

        when 1 do { // Steals once from victim. If it has sucess on the stealing,
                    // it chooses another victim and tries to steal again.

          if moreLocalWork[victim] then {
            // There is work in victim
            const zeroBasedIters2:rType=splitWork(victim); // The current range we get
                                                 //after splitting from a victim range
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          } else {
            stealFailed=true;
          }
        }
        otherwise { // Tt steals from the victim, but now splitting
                    //from the tail of the victim's range

         while moreLocalWork[victim] do {
          // There is work in victim
          const zeroBasedIters2:rType=splitWork(victim, true); // The current range we get
                                // after splitting from a victim range:
                               // now we set splitTatil to true (second parameter in the call)
          if zeroBasedIters2.length !=0 then {
            if debugAdvancedIters then
              writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
            yield (zeroBasedIters2,);
          }
         }
        }
      }
      // If here, then it can have failed the stealing intent at the victim (method 1),
      // or we have exhausted the victim range (methods 0, 2)

      if (methodStealing==0 || methodStealing==2 || (methodStealing==1 && stealFailed)) then {
        nVisitedVictims += 1; // Signal that there is no more work in victim
        if debugAdvancedIters then
          writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        stealFailed=false;
      }
      // Check if there is no more work
      if nVisitedVictims >= nTasks-1 then
        moreWork=false; // Signal that there is no more work in any victim
      else {  // There can be still work in other victim
        victim=(victim+1) % nTasks; // New victim to steal
        if methodStealing==1 then
          if victim==tid then victim=(victim+1) % nTasks;
      }
    }
    }

    proc splitRange(c:range(?),tid:int)
  {
    const size= c.length/nTasks;
    var rLocal:rType=(c+tid*size)#size;
    if tid==nTasks-1 then
      rLocal=c#(size*(nTasks-1)-c.length);
    return rLocal;
  }

  proc barrier
  {
    var tmp=barrierCount$;
    barrierCount$=tmp+1;
    if tmp+1==nTasks then wait$=true;
    wait$;
  }

    proc splitWork(taskid:int, splitTail:bool=false)
  {
    if splitTail then { // Splitting from the tail of the remaining range (method 2)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid], splitTail);
      splitLock$[taskid]=true;
      const zeroBasedCurrent3:rType=current;
      return zeroBasedCurrent3;
    }
    else { // Splitting from the head of the remaining range (methods 0 and 1)
      splitLock$[taskid]; // Perform the splitting in a critical section
      const current:rType=adaptSplit(localWork[taskid], factorSteal, moreLocalWork[taskid]);
      splitLock$[taskid]=true;
      const zeroBasedCurrent:rType=current;
      return zeroBasedCurrent;
      }

  }
  }
}

// Follower
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.numCores;
      else dnTasks=dataParTasksPerLocale;
  } else if nTasks<0 then {
    halt("'numTasks' is negative");
  }
  return dnTasks;
}

proc splitChunk(inout rangeToSplit:range(?), chunkSize:int, inout itLeft:bool)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;

  totLen=rangeToSplit.length;
  size=chunkSize;
  if totLen <= size then
    {
    size=totLen;
    itLeft=false;
    }
  const firstRange:rType=rangeToSplit#size;
  rangeToSplit=rangeToSplit#(size-totLen);
  return firstRange;
}

proc adaptSplit(inout rangeToSplit:range(?), splitFactor:int, inout itLeft:bool, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  var direction: int;
  const profThreshold=1;

  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  if splitTail then
    direction=-1;
  else direction=1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  return firstRange;
}


add document id linked list

add document id linked list

// BUG: declaring use IO, Memory here causes chpl compiler segv

module Search {

  use Logging, Memory, LockFreeHash, GenHashKey32, Partitions;

  config const sync_writers = false;
  config const entry_size: uint(32) = 1024*1024;
  config const max_doc_node_size: uint(32) =  32 * 1024;

  // TODO: should be using a doc index that maps to a doc id?
  type DocId = uint(64);

  /**
    A documentId node
  */
  class DocumentIdNode {
    // controls the size of this document list
    var listSize: int = 1;

    var next: DocumentIdNode;

    // list of documents
    var documents: [0..listSize-1] DocId;

    // number of documents in this node's list
    var documentCount: atomic int;

    proc nextDocumentIdNodeSize() {
      if (documents.size >= max_doc_node_size) {
        return documents.size;
      } else {
        return documents.size * 2;
      }
    }
  }

  // for scoring and compactness purposes, consider making this use more contiguous memory,
  //  or an associative array(s) that use the score or docid as index.
  class Entry {
    var word: string;

    var score: real; // TODO: may not be needed if we use docCount as frequency

    // total number of documents in all the documentId nodes
    var documentCount: atomic int;

    var documentIdNode: DocumentIdNode;
  }

  class PartitionIndex {
    var entryCount: atomic uint(32);
    var entryIndex = new LockFreeHash(entry_size);
    var entries: [1..entry_size] Entry;
    var writerLock: atomicflag;

    inline proc lockIndexWriter() {
      // debug("attempting to get lock");
      if (sync_writers) then while writerLock.testAndSet() do chpl_task_yield();
      // debug("have lock");
    }

    inline proc unlockIndexWriter() {
      // writeln("releasing lock");
      if (sync_writers) then writerLock.clear();
      // writeln("released lock");
    }

    proc PartitionIndex() {
      entryCount.add(1); // start at the first index of entries
    }
  }

  var Indices: [0..Partitions.size-1] PartitionIndex;

  proc initIndices() {
    for i in 0..Partitions.size-1 {
      on Partitions[i] {
        info("index [", i, "] is mapped to partition ", i);

        // allocate the partition index on the partition locale
        Indices[i] = new PartitionIndex();
      }
    }
  }

  proc entryForWord(word: string): Entry {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    var entry: Entry;
    on partitionIndex {
      entry = entryForWordOnPartition(word, partitionIndex);
    }
    return entry;
  }

  proc entryForWordOnPartition(word: string, partitionIndex: PartitionIndex): Entry {
    var entryIndex = entryIndexForWord(word, partitionIndex);
    if (entryIndex > 0) {
      return partitionIndex.entries[entryIndex];
    }
    return nil;
  }

  proc indexWord(word: string, docid: DocId) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      partitionIndex.lockIndexWriter();

      var entry = entryForWordOnPartition(word, partitionIndex);
      if (entry != nil) {
        debug("adding ", word, " to existing entries on partition ", partition);
        var docNode = entry.documentIdNode;
        var docCount = docNode.documentCount.read();
        if (docCount < docNode.documents.size) {
          docNode.documents[docCount] = docid;
          docNode.documentCount.add(1);
        } else {
          var newDocNode = new DocumentIdNode(docNode.nextDocumentIdNodeSize(), docNode);
          debug("adding new document id node of size ", newDocNode.documents.size);
          newDocNode.documents[0] = docid;
          newDocNode.documentCount.write(1);
          entry.documentIdNode = newDocNode;
        }
        entry.documentCount.add(1);
      } else {
        debug("adding new entry ", word , " on partition ", partition);

        var entriesCount = partitionIndex.entryCount.read();
        if (entriesCount < partitionIndex.entries.size) {
          entry = new Entry();
          entry.word = word;
          entry.score = 0;

          var newDocNode = new DocumentIdNode();
          newDocNode.documents[0] = docid;
          newDocNode.documentCount.write(1);
          entry.documentIdNode = newDocNode;
          entry.documentCount.write(1);

          var entryIndex: uint(32) = partitionIndex.entryCount.fetchAdd(1);
          partitionIndex.entries[entryIndex] = entry;
          var success = partitionIndex.entryIndex.setItem(word, entryIndex);
          if (!success) {
            error("indexWord: failed to index ", word);
            exit(0);
            // how do we accumuate per-partition indexing errors for a final response?
          }
        }
      }

      partitionIndex.unlockIndexWriter();
    }
  }

  proc indexContainsWord(word: string, partitionIndex: PartitionIndex): bool {
    return entryIndexForWord(word, partitionIndex) != 0;
  }

  proc entryIndexForWord(word: string, partitionIndex: PartitionIndex): uint(32) {
    return partitionIndex.entryIndex.getItem(word);
  }

  proc dumpEntry(entry: Entry) {
    on entry {
      writeln("word: ", entry.word, " score: ", entry.score);
      var node = entry.documentIdNode;
      while (node != nil) {
        for i in 0..node.documentCount.read()-1 {
          writeln("\t", node.documents[i]);
        }
        node = node.next;
      }
    }
  }

  proc dumpPartition(partition: int) {
    var partitionIndex = Indices[partition];
    on partitionIndex {
      writeln("entries on partition (", partition, ") locale (", here.id, ") ", partitionIndex.entries);

      var word: string;
      for i in 0..partitionIndex.entryCount.read()-1 {
        var entry = partitionIndex.entries[i];
        writeln("word: ", entry.word);
        dumpPostingTableForWord(entry.word);
      }
    }
  }

  proc dumpPostingTableForWord(word: string) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      var entryIndex = entryIndexForWord(word, partitionIndex);
      if (entryIndex > 0) {
        dumpEntry(partitionIndex.entries[entryIndex]);
      } else {
        writeln("word (", word, ") is not in the index");
      }
    }
  }
}

// BUG: declaring use IO, Memory here causes chpl compiler segv

module Search {

  use Logging, Memory, LockFreeHash, GenHashKey32, Partitions;

  config const sync_writers = false;
  config const entry_size: uint(32) = 1024*1024;

  // TODO: should be using a doc index that maps to a doc id?
  type DocId = uint(64);

  // for scoring and compactness purposes, consider making this use more contiguous memory,
  //  or an associative array(s) that use the score or docid as index.
  class Entry {
    var word: string;
    var score: real; // TODO: may not be needed if we use docCount as frequency
    var documentCount: atomic int;
    var documents: [0..256-1] DocId; // TODO: point to the tail of a linked list; keep track of node + index into node
  }

  class PartitionIndex {
    var entryCount: atomic uint(32);
    var entryIndex = new LockFreeHash(entry_size);
    var entries: [1..entry_size] Entry;
    var writerLock: atomicflag;

    inline proc lockIndexWriter() {
      // debug("attempting to get lock");
      if (sync_writers) then while writerLock.testAndSet() do chpl_task_yield();
      // debug("have lock");
    }

    inline proc unlockIndexWriter() {
      // writeln("releasing lock");
      if (sync_writers) then writerLock.clear();
      // writeln("released lock");
    }

    proc PartitionIndex() {
      entryCount.add(1); // start at the first index of entries
    }
  }

  var Indices: [0..Partitions.size-1] PartitionIndex;

  proc initIndices() {
    for i in 0..Partitions.size-1 {
      on Partitions[i] {
        info("index [", i, "] is mapped to partition ", i);

        // allocate the partition index on the partition locale
        Indices[i] = new PartitionIndex();
      }
    }
  }

  proc entryForWord(word: string): Entry {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    var entry: Entry;
    on partitionIndex {
      entry = entryForWordOnPartition(word, partitionIndex);
    }
    return entry;
  }

  proc entryForWordOnPartition(word: string, partitionIndex: PartitionIndex): Entry {
    var entryIndex = entryIndexForWord(word, partitionIndex);
    if (entryIndex > 0) {
      return partitionIndex.entries[entryIndex];
    }
    return nil;
  }

  proc indexWord(word: string, docid: DocId) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      partitionIndex.lockIndexWriter();

      var entry = entryForWordOnPartition(word, partitionIndex);
      if (entry != nil) {
        debug("adding ", word, " to existing entries on partition ", partition);
        var docCount = entry.documentCount.read();
        if (docCount < entry.documents.size) {
          entry.documents[docCount] = docid;
          entry.documentCount.add(1);
        } else {
          // TODO: append node to linked list of document ids
          // error("TODO: realloc documents for word: ", word);
        }
      } else {
        debug("adding new entry ", word , " on partition ", partition);

        var entriesCount = partitionIndex.entryCount.read();
        if (entriesCount < partitionIndex.entries.size) {
          entry = new Entry();
          entry.word = word;
          entry.score = 0;
          entry.documents[0] = docid;
          entry.documentCount.add(1);

          var entryIndex: uint(32) = partitionIndex.entryCount.fetchAdd(1);
          partitionIndex.entries[entryIndex] = entry;
          var success = partitionIndex.entryIndex.setItem(word, entryIndex);
          if (!success) {
            error("indexWord: failed to index ", word);
            exit(0);
            // how do we accumuate per-partition indexing errors for a final response?
          }
        }
      }

      partitionIndex.unlockIndexWriter();
    }
  }

  proc indexContainsWord(word: string, partitionIndex: PartitionIndex): bool {
    return entryIndexForWord(word, partitionIndex) != 0;
  }

  proc entryIndexForWord(word: string, partitionIndex: PartitionIndex): uint(32) {
    return partitionIndex.entryIndex.getItem(word);
  }

  proc dumpEntry(entry: Entry) {
    on entry {
      writeln("word: ", entry.word, " score: ", entry.score);
      for i in 0..entry.documentCount.read()-1 {
        writeln("\t", entry.documents[i]);
      }
    }
  }

  proc dumpPartition(partition: int) {
    var partitionIndex = Indices[partition];
    on partitionIndex {
      writeln("entries on partition (", partition, ") locale (", here.id, ") ", partitionIndex.entries);

      var word: string;
      for i in 0..partitionIndex.entryCount.read()-1 {
        var entry = partitionIndex.entries[i];
        writeln("word: ", entry.word);
        dumpPostingTableForWord(entry.word);
      }
    }
  }

  proc dumpPostingTableForWord(word: string) {
    var partition = partitionForWord(word);
    var partitionIndex = Indices[partition];
    on partitionIndex {
      var entryIndex = entryIndexForWord(word, partitionIndex);
      if (entryIndex > 0) {
        dumpEntry(partitionIndex.entries[entryIndex]);
      } else {
        writeln("word (", word, ") is not in the index");
      }
    }
  }
}


Snapshot.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@8003 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

// TODO: clean up complex arithmetic
use BitOps;
use Random;
use Time;

use HPCCProblemSize;

param radix = 4;

param numVectors = 2;
type elemType = complex(128);

config const n = computeProblemSize(elemType, numVectors, returnLog2 = true);
const m = 2**n;

config const epsilon = 2.0 ** -51.0,
             threshold = 16.0;

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

config const printParams = true,
             printArrays = false,
             printStats = true;

def main() {
  printConfiguration();

  const TwiddleDom = [0..m/4);
  var Twiddles: [TwiddleDom] elemType;

  const ProblemDom = [0..m);
  var Z, z: [ProblemDom] elemType;

  initVectors(Twiddles, z);

  const startTime = getCurrentTime();

  Z = conjg(z);
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  const execTime = getCurrentTime() - startTime;

  const validAnswer = verifyResults(z, Z, Twiddles);
  printResults(validAnswer, execTime);
}

def dfft(Z, Twiddles) {
  cft1st(Z, Twiddles);

  var span = radix,
      lasti = 2;
  for i in radix..n/2 by 2 {
    cftmd1(span, Z, Twiddles);
    span *= radix;
    lasti = i;
  }
  for i in [lasti+2..n) by 2 {
    cftmd2(span, Z, Twiddles);
    span *= radix;
  }

  if (radix*span == Z.numElements) then
    forall j in [0..span) do
      butterfly(1.0, 1.0, 1.0, Z[j..j+3*span by span]);
  else
    forall j in [0..span) {
      const a = Z(j),
            b = Z(j+span);
      Z(j)      = a + b;
      Z(j+span) = a - b;
    }
}

def printConfiguration() {
  if (printParams) then printProblemSize(elemType, numVectors, m);
}

def initVectors(Twiddles, z) {
  computeTwiddles(Twiddles);
  bitReverseShuffle(Twiddles);

  fillRandom(z, seed);

  if (printArrays) {
    writeln("After initialization, Twiddles is: ", Twiddles, "\n");
    writeln("z is: ", z, "\n");
  }
}

def computeTwiddles(Twiddles) {
  const numTwdls = Twiddles.numElements,
        delta = 2.0 * atan(1.0) / numTwdls;

  Twiddles(0) = 1.0;
  Twiddles(numTwdls/2) = let x = cos(delta * numTwdls/2)
                          in (x, x):complex;
  forall i in [1..numTwdls/2) {
    const x = cos(delta*i),
          y = sin(delta*i);
    Twiddles(i)            = (x, y):complex;
    Twiddles(numTwdls - i) = (y, x):complex;
  }
}

def bitReverseShuffle(Vect: [?Dom]) {
  const numBits = log2(Vect.numElements),
        Perm: [i in Dom] index(Dom) = bitReverse(i, revBits = numBits),
        Temp = Vect(Perm);
  Vect = Temp;
}

// reverses numBits low-order bits of val
def bitReverse(val: ?valType, revBits = 64) {
  param mask = 0x0102040810204080;
  const valReverse64 = bitMatMultOr(mask, bitMatMultOr(val:uint(64), mask)),
        valReverse = bitRotLeft(valReverse64, revBits);
  return valReverse: valType;
}

def verifyResults(z, Z, Twiddles) {
  if (printArrays) then writeln("After FFT, Z is: ", Z, "\n");

  Z = conjg(Z) / m;
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  if (printArrays) then writeln("After inverse FFT, Z is: ", Z, "\n");

  var maxerr = max reduce sqrt((z.re - Z.re)**2 + (z.im - Z.im)**2);
  maxerr /= (epsilon * n);
  if (printStats) then writeln("error = ", maxerr);

  return (maxerr < threshold);
}

def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (Gflop/s) = ", 5.0 * (m * n) / execTime / 1.0e-9);
  }
}

def butterfly(wk1, wk2, wk3, inout A:[1..radix]) {
  var x0 = A(1) + A(2),
      x1 = A(1) - A(2),
      x2 = A(3) + A(4),
      x3rot = (A(3) - A(4))*1.0i;

  A(1) = x0 + x2;
  x0 -= x2;
  A(3) = wk2 * x0;
  x0 = x1 + x3rot;
  A(2) = wk1 * x0;
  x0 = x1 - x3rot;
  A(4) = wk3 * x0;
}

def cft1st(A, Twiddles) {
  var x0 = A(0) + A(1),
      x1 = A(0) - A(1),
      x2 = A(2) + A(3),
      x3rot = (A(2) - A(3))*1.0i;
  const wk1r = Twiddles(1).re;

  A(0) = x0 + x2;
  A(2) = x0 - x2;
  A(1) = x1 + x3rot;
  A(3) = x1 - x3rot;

  x0 = A(4) + A(5);
  x1 = A(4) - A(5);
  x2 = A(6) + A(7);
  const x3 = A(6) - A(7);  // BLC: try to eliminate this
  x3rot = (A(6) - A(7))*1.0i;
  A(4) = x0 + x2;
  A(6) = (x2.im - x0.im, x0.re - x2.re):complex;
  x0 = x1 + x3rot;
  A(5) = wk1r * (x0.re - x0.im, x0.re + x0.im):complex;
  x0 = (x3.im + x1.re, x3.re - x1.im):complex;
  A(7) = wk1r * (x0.im - x0.re, x0.im + x0.re):complex;

  forall (j,k1) in ([8..A.numElements) by 8, 1..) {
    var wk2 = Twiddles(k1),
        wk1 = Twiddles(2*k1),
        wk3 = (wk1.re - 2* wk2.im * wk1.im,
               2 * wk2.im * wk1.re - wk1.im):complex;

    butterfly(wk1, wk2, wk3, A[j..j+3]);

    wk1 = Twiddles(2*k1+1);
    wk3 = (wk1.re - 2*wk2.re * wk1.im,
           2*wk2.re * wk1.re - wk1.im):complex;
    wk2 = wk2*1.0i;
    butterfly(wk1, wk2, wk3, A[j+4..j+7]);
  }
}

def cftmd0(span, A, Twiddles) {
  const wk1r = Twiddles(1).re,
        m = radix*span;

  forall j in [0..span) do
    butterfly(1.0, 1.0, 1.0, A[j..j+3*span by span]);

  forall j in [m..m+span) do
    butterfly((wk1r, wk1r):complex, 1.0i, (-wk1r, wk1r):complex,
              A[j..j+3*span by span]);
}

def cftmd1(span, A, Twiddles) {
  const m = radix*span,
        m2 = 2*m;

  cftmd0(span, A, Twiddles);
  forall (k,k1) in ([m2..A.numElements) by m2, 1..) {
    var wk2 = Twiddles(k1),
        wk1 = Twiddles(2*k1),
        wk3 = (wk1.re - 2 * wk2.im * wk1.im,
               2 * wk2.im * wk1.re - wk1.im):complex;
    for j in [k..k+span) do
      butterfly(wk1, wk2, wk3, A[j..j+3*span by span]);

    wk1 = Twiddles(2*k1+1);
    wk3 = (wk1.re - 2 * wk2.re * wk1.im,
           2 * wk2.re * wk1.re - wk1.im):complex;

    for j in [k+m..k+m+span) do
      butterfly(wk1, wk2*1.0i, wk3, A[j..j+3*span by span]);
  }
}

def cftmd2(span, A, Twiddles) {
  const m = radix*span,
        m2 = 2*m,
        numElems = A.numElements;

  cftmd0(span, A, Twiddles);
  if (m2 >= numElems) return;
  if (m2 >= numElems / 8) {
    cftmd21(span, A, Twiddles);
    return;
  }

  forall j in [0..span) {
    forall (k,k1) in ([m2..numElems) by m2, 1..) {
      const wk2 = Twiddles(k1),
            wk1 = Twiddles(k1 + k1),
            wk3 = (wk1.re - 2*wk2.im * wk1.im,
                   2 * wk2.im * wk1.re - wk1.im):complex;
      butterfly(wk1, wk2, wk3, A[j+k..j+k+3*span by span]);
    }

    forall (k,k1) in ([m2..numElems) by m2, 1..) {
      const wk2 = Twiddles(k1),
            wk1 = Twiddles(2*k1 + 1),
            wk3 = (wk1.re - 2*wk2.re * wk1.im,
                   2*wk2.re * wk1.re - wk1.im):complex;
      wk2 = wk2*1.0i;

      butterfly(wk1, wk2, wk3, A[j+k+m..j+k+m+3*span by span]);
    }
  }
}

def cftmd21(span, A, Twiddles) {
  const m = radix*span,
        m2 = 2*m;

  for (k,k1) in ([m2..A.numElements) by m2, 1..) {
    var wk2 = Twiddles(k1),
        wk1 = Twiddles(2*k1),
        wk3 = (wk1.re - 2*wk2.im * wk1.im,
               2*wk2.im * wk1.re - wk1.im):complex;

    forall j in [k..k+span) do
      butterfly(wk1, wk2, wk3, A[j..j+3*span by span]);

    wk1 = Twiddles(2*k1 + 1);
    wk3 = (wk1.re - 2*wk2.re * wk1.im,
           2*wk2.re * wk1.re - wk1.im):complex;
    wk2 = wk2*1.0i;

    forall j in [k+m..k+m+span) do
      butterfly(wk1, wk2, wk3, A[j..j+3*span by span]);
  }
}

// TODO: clean up complex arithmetic
use BitOps;
use Random;
use Time;

use HPCCProblemSize;

param radix = 4;

param numVectors = 2;
type elemType = complex(128);

config const n = computeProblemSize(elemType, numVectors, returnLog2 = true);
const m = 2**n;

config const epsilon = 2.0 ** -51.0,
             threshold = 16.0;

config const useRandomSeed = true,
             seed = if useRandomSeed then SeedGenerator.clockMS else 314159265;

config const printParams = true,
             printArrays = false,
             printStats = true;

def main() {
  printConfiguration();

  const TwiddleDom = [0..m/4);
  var Twiddles: [TwiddleDom] elemType;

  const ProblemDom = [0..m);
  var Z, z: [ProblemDom] elemType;

  initVectors(Twiddles, z);

  const startTime = getCurrentTime();

  Z = conjg(z);
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  const execTime = getCurrentTime() - startTime;

  const validAnswer = verifyResults(z, Z, Twiddles);
  printResults(validAnswer, execTime);
}

def dfft(Z, W) {
  cft1st(Z, W);

  var span = radix,
      lasti = 2;
  for i in radix..n/2 by 2 {
    cftmd1(span, Z, W);
    span *= radix;
    lasti = i;
  }
  for i in [lasti+2..n) by 2 {
    cftmd2(span, Z, W);
    span *= radix;
  }

  if (radix*span == Z.numElements) then
    forall j in [0..span) do
      butterfly(1.0, 1.0, 1.0, Z[j..j+3*span by span]);
  else
    forall j in [0..span) {
      const a = Z(j),
            b = Z(j+span);
      Z(j) = a + b;
      Z(j+span) = a - b;
    }
}

def printConfiguration() {
  if (printParams) then printProblemSize(elemType, numVectors, m);
}

def initVectors(Twiddles, z) {
  computeTwiddles(Twiddles);
  bitReverseShuffle(Twiddles);

  fillRandom(z, seed);

  if (printArrays) {
    writeln("After initialization, Twiddles is: ", Twiddles, "\n");
    writeln("z is: ", z, "\n");
  }
}

def computeTwiddles(W) {
  const numTwids = W.numElements,
        delta = 2.0 * atan(1.0) / numTwids;

  W(0) = 1.0;
  W(numTwids/2) = let x = cos(delta * numTwids/2)
                   in (x, x):complex;
  forall i in [1..numTwids/2) {
    const x = cos(delta*i),
          y = sin(delta*i);
    W(i)            = (x, y):complex;
    W(numTwids - i) = (y, x):complex;
  }
}

def bitReverseShuffle(W: [?WD]) {
  const numBits = log2(W.numElements),
        Perm: [i in WD] index(WD) = bitReverse(i, revBits = numBits),
        V = W(Perm);
  W = V;
}

// reverses numBits low-order bits of val
def bitReverse(val: ?valType, revBits = 64) {
  param mask: uint(64) = 0x0102040810204080;
  const valReverse64 = bitMatMultOr(mask, bitMatMultOr(val:uint(64), mask));
  const valReverse = bitRotLeft(valReverse64, revBits);
  return valReverse: valType;
}

def verifyResults(z, Z, Twiddles) {
  if (printArrays) then writeln("After FFT, Z is: ", Z, "\n");

  Z = conjg(Z) / m;
  bitReverseShuffle(Z);
  dfft(Z, Twiddles);

  if (printArrays) then writeln("After inverse FFT, Z is: ", Z, "\n");

  var maxerr = max reduce sqrt((z.re - Z.re)**2 + (z.im - Z.im)**2);
  maxerr /= (epsilon * n);
  if (printStats) then writeln("error = ", maxerr);

  return (maxerr < threshold);
}

def printResults(successful, execTime) {
  writeln("Validation: ", if successful then "SUCCESS" else "FAILURE");
  if (printStats) {
    writeln("Execution time = ", execTime);
    writeln("Performance (Gflop/s) = ", 5.0 * (m * n) / execTime / 1.0e-9);
  }
}

def butterfly(wk1, wk2, wk3, inout A:[1..radix]) {
  var x0 = A[1] + A[2],
      x1 = A[1] - A[2],
      x2 = A[3] + A[4],
      x3rot = (A[3] - A[4])*1.0i;

  A[1] = x0 + x2;
  x0 -= x2;
  A[3] = wk2 * x0;
  x0 = x1 + x3rot;
  A[2] = wk1 * x0;
  x0 = x1 - x3rot;
  A[4] = wk3 * x0;
}

def cft1st(A, W) {
  var x0 = A(0) + A(1),
      x1 = A(0) - A(1),
      x2 = A(2) + A(3),
      x3rot = (A(2) - A(3))*1.0i;
  const wk1r = W(1).re;

  A(0) = x0 + x2;
  A(2) = x0 - x2;
  A(1) = x1 + x3rot;
  A(3) = x1 - x3rot;

  x0 = A(4) + A(5);
  x1 = A(4) - A(5);
  x2 = A(6) + A(7);
  const x3 = A(6) - A(7);  // BLC: try to eliminate this
  x3rot = (A(6) - A(7))*1.0i;
  A(4) = x0 + x2;
  A(6) = (x2.im - x0.im, x0.re - x2.re):complex;
  x0 = x1 + x3rot;
  A(5) = wk1r * (x0.re - x0.im, x0.re + x0.im):complex;
  x0 = (x3.im + x1.re, x3.re - x1.im):complex;
  A(7) = wk1r * (x0.im - x0.re, x0.im + x0.re):complex;

  forall (j,k1) in ([8..A.numElements) by 8, 1..) {
    var wk2 = W(k1),
        wk1 = W(2*k1),
        wk3 = (wk1.re - 2* wk2.im * wk1.im,
               2 * wk2.im * wk1.re - wk1.im):complex;

    butterfly(wk1, wk2, wk3, A[j..j+3]);

    wk1 = W(2*k1+1);
    wk3 = (wk1.re - 2*wk2.re * wk1.im,
           2*wk2.re * wk1.re - wk1.im):complex;
    wk2 = wk2*1.0i;
    butterfly(wk1, wk2, wk3, A[j+4..j+7]);
  }
}

def cftmd0(span, A, W) {
  const wk1r = W(1).re,
        m = radix*span;

  forall j in [0..span) do
    butterfly(1.0, 1.0, 1.0, A[j..j+3*span by span]);

  forall j in [m..m+span) do
    butterfly((wk1r, wk1r):complex, 1.0i, (-wk1r, wk1r):complex,
              A[j..j+3*span by span]);
}

def cftmd1(span, A, W) {
  const m = radix*span;
  const m2 = 2*m;

  cftmd0(span, A, W);
  forall (k,k1) in ([m2..A.numElements) by m2, 1..) {
    var wk2 = W[k1],
        wk1 = W[2*k1],
        wk3 = (wk1.re - 2 * wk2.im * wk1.im,
               2 * wk2.im * wk1.re - wk1.im):complex;
    for j in [k..k+span) do
      butterfly(wk1, wk2, wk3, A[j..j+3*span by span]);

    wk1 = W[2*k1+1];
    wk3 = (wk1.re - 2 * wk2.re * wk1.im,
           2 * wk2.re * wk1.re - wk1.im):complex;

    for j in [k+m..k+m+span) do
      butterfly(wk1, wk2*1.0i, wk3, A[j..j+3*span by span]);
  }
}

def cftmd2(span, A, W) {
  const m = radix*span,
        m2 = 2*m,
        numElems = A.numElements;

  cftmd0(span, A, W);
  if (m2 >= numElems) return;
  if (m2 >= numElems / 8) {
    cftmd21(span, A, W);
    return;
  }

  forall j in [0..span) {
    forall (k,k1) in ([m2..numElems) by m2, 1..) {
      const wk2 = W[k1],
            wk1 = W[k1 + k1],
            wk3 = (wk1.re - 2*wk2.im * wk1.im,
                   2 * wk2.im * wk1.re - wk1.im):complex;
      butterfly(wk1, wk2, wk3, A[j+k..j+k+3*span by span]);
    }

    forall (k,k1) in ([m2..numElems) by m2, 1..) {
      const wk2 = W[k1],
            wk1 = W[2*k1 + 1],
            wk3 = (wk1.re - 2*wk2.re * wk1.im,
                   2*wk2.re * wk1.re - wk1.im):complex;
      wk2 = wk2*1.0i;

      butterfly(wk1, wk2, wk3, A[j+k+m..j+k+m+3*span by span]);
    }
  }
}

def cftmd21(span, A, W) {
  const m = radix*span,
        m2 = 2*m;

  for (k,k1) in ([m2..A.numElements) by m2, 1..) {
    var wk2 = W[k1],
        wk1 = W[2*k1],
        wk3 = (wk1.re - 2*wk2.im * wk1.im,
               2* wk2.im * wk1.re - wk1.im):complex;

    forall j in [k..k+span) do
      butterfly(wk1, wk2, wk3, A[j..j+3*span by span]);

    wk1 = W[2*k1 + 1];
    wk3 = (wk1.re - 2*wk2.re * wk1.im,
           2*wk2.re * wk1.re - wk1.im):complex;
    wk2 = wk2*1.0i;

    forall j in [k+m..k+m+span) do
      butterfly(wk1, wk2, wk3, A[j..j+3*span by span]);
  }
}