Ignore noinit for the record type regexp, it contains a field which is a c pointer and assumes it needs to clean up that pointer during assignment when, if noinit is used, that field would be null and thus not capable of being cleaned up.

extern type qio_regexp_t;

extern record qio_regexp_options_t {
  var utf8:bool;
  var posix:bool;
  var literal:bool;
  var nocapture:bool;
  // These ones can be set inside the regexp
  var ignorecase:bool; // (?i)
  var multiline:bool; // (?m)
  var dotnl:bool; // (?s)
  var nongreedy:bool; // (?U)
}

extern proc qio_regexp_null():qio_regexp_t;
extern proc qio_regexp_init_default_options(ref options:qio_regexp_options_t);
extern proc qio_regexp_create_compile(str:c_string, strlen:int(64), ref options:qio_regexp_options_t, ref compiled:qio_regexp_t);
extern proc qio_regexp_create_compile_flags(str:c_string, strlen:int(64), flags:c_string, flagslen:int(64), isUtf8:bool, ref compiled:qio_regexp_t);
extern proc qio_regexp_create_compile_flags_2(str:c_void_ptr, strlen:int(64), flags:c_void_ptr, flagslen:int(64), isUtf8:bool, ref compiled:qio_regexp_t);
extern proc qio_regexp_retain(ref compiled:qio_regexp_t);
extern proc qio_regexp_release(ref compiled:qio_regexp_t);

extern proc qio_regexp_get_options(ref regexp:qio_regexp_t, ref options: qio_regexp_options_t);
extern proc qio_regexp_get_pattern(ref regexp:qio_regexp_t, ref pattern: c_string);
extern proc qio_regexp_get_ncaptures(ref regexp:qio_regexp_t):int(64);
extern proc qio_regexp_ok(ref regexp:qio_regexp_t):bool;
extern proc qio_regexp_error(ref regexp:qio_regexp_t):c_string;

extern const QIO_REGEXP_ANCHOR_UNANCHORED:c_int;
extern const QIO_REGEXP_ANCHOR_START:c_int;
extern const QIO_REGEXP_ANCHOR_BOTH:c_int;

extern record qio_regexp_string_piece_t {
  var offset:int(64); // counting from 0, -1 means "NULL"
  var len:int(64);
}

extern proc qio_regexp_string_piece_isnull(ref sp:qio_regexp_string_piece_t):bool;

extern proc qio_regexp_match(ref re:qio_regexp_t, text:c_string, textlen:int(64), startpos:int(64), endpos:int(64), anchor:c_int, submatch:_ddata(qio_regexp_string_piece_t), nsubmatch:int(64)):bool;
extern proc qio_regexp_replace(ref re:qio_regexp_t, repl:c_string, repllen:int(64), text:c_string, textlen:int(64), startpos:int(64), endpos:int(64), global:bool, ref replaced:c_string, ref replaced_len:int(64)):int(64);

// These two could be folded together if we had a way
// to check if a default argument was supplied
// (or any way to use 'nil' in pass-by-ref)
/** Compile a regular expression. This routine will halt if compilation failed
 */
proc compile(pattern: string, utf8=true, posix=false, literal=false, nocapture=false, /*i*/ ignorecase=false, /*m*/ multiline=false, /*s*/ dotnl=false, /*U*/ nongreedy=false):regexp {
  var opts:qio_regexp_options_t;
  qio_regexp_init_default_options(opts);
  opts.utf8 = utf8;
  opts.posix = posix;
  opts.literal = literal;
  opts.nocapture = nocapture;
  opts.ignorecase = ignorecase;
  opts.multiline = multiline;
  opts.dotnl = dotnl;
  opts.nongreedy = nongreedy;

  var ret:regexp;
  qio_regexp_create_compile(pattern.c_str(), pattern.size, opts, ret._regexp);
  if ! qio_regexp_ok(ret._regexp) {
    var err_str = qio_regexp_error(ret._regexp);
    __primitive("chpl_error", "Error " + err_str + " when compiling regexp '" + pattern + "'");
  }
  return ret;
}

/** Compile a regular expression and retrieve an error code if compilation
    failed.
 */
proc compile(pattern: string, out error:syserr, utf8=true, posix=false, literal=false, nocapture=false, /*i*/ ignorecase=false, /*m*/ multiline=false, /*s*/ dotnl=false, /*U*/ nongreedy=false):regexp {
  var opts:qio_regexp_options_t;
  qio_regexp_init_default_options(opts);
  opts.utf8 = utf8;
  opts.posix = posix;
  opts.literal = literal;
  opts.nocapture = nocapture;
  opts.ignorecase = ignorecase;
  opts.multiline = multiline;
  opts.dotnl = dotnl;
  opts.nongreedy = nongreedy;

  var ret:regexp;
  qio_regexp_create_compile(pattern, pattern.size, opts, ret._regexp);

  if qio_regexp_ok(ret._regexp) {
    error = ENOERR;
  } else {
    error = qio_format_error_bad_regexp();
  }

  return ret;
}

/** Regular expression search routines also can work with
    a stringPart, which refers to a substring within another
    string.
 */
record stringPart {
  var offset:int;
  var length:int;
  var from:string;
}

/** The reMatch record records a regular expression search match
    or a capture group.
    Regular expression search routines normally return one of these.
    Also, this type can be passed as a capture group argument.
    Lastly, something of type reMatch can be checked for a match
    in a simple if statement, as in:
      var m:reMatch = ....;
      if m then do_something_if_matched();
      if !m then do_something_if_not_matched();

 */
record reMatch {
  var matched:bool;
  var offset:int; // 0-based, -1 if matched==false
  var length:int; // 0 if matched==false
}

proc _to_reMatch(ref p:qio_regexp_string_piece_t):reMatch {
  if qio_regexp_string_piece_isnull(p) {
    return new reMatch(false, -1, 0);
  } else {
    return new reMatch(true, p.offset, p.len);
  }
}

inline proc !(m: reMatch) return !m.matched;
inline proc _cond_test(m: reMatch) return m.matched;

/** Use someString.substring(anReMatch) to extract the part
    of a string matching a regular expression or capture group.
 */
proc string.substring(m:reMatch) {
  if m.matched then return this.substring(m.offset+1..#m.length);
  else return "";
}

/** This class represents a compiled regular expression. Regular expressions
    are cached on a per-thread basis and are reference counted.
    To create a compiled regular expression, use the compile function.
  */
pragma "ignore noinit"
record regexp {
  var home: locale = here;
  var _regexp:qio_regexp_t = qio_regexp_null();

  proc ok:bool {
    return qio_regexp_ok(_regexp);
  }
  proc error():string {
    return qio_regexp_error(_regexp);
  }

  // note - more = overloads are below.
  proc ~regexp() {
    qio_regexp_release(_regexp);
    _regexp = qio_regexp_null();
  }

  proc _handle_captures(text: string, matches:_ddata(qio_regexp_string_piece_t), nmatches:int, ref captures) {
    assert(nmatches >= captures.size);
    for param i in 1..captures.size {
      var m = _to_reMatch(matches[i]);
      if captures[i].type == reMatch {
        captures[i] = m;
      } else {
        if m.matched {
          captures[i] = text.substring(m):captures[i].type;
        } else {
          var empty:captures[i].type;
          captures[i] = empty;
        }
      }
    }
  }

  // Note - we would only need one version of these routines
  // if we had args ...?k supporting 0 args, or if tuples support zero length
  /** Search within the passed text (which must be a string or a stringPart)
      for the first match at any offset to this regular expression. Returns a
      match object. This routine will try matching the regular expression at
      different offsets until a match is found. If you want to only match at
      the beginning of the pattern, you can start your pattern with ^ or
      use regexp.match. The capture group arguments here can be strings
      or any type that can reasonably cast from string. If a capture group
      was not matched, the corresponding argument will get the default value
      for its type.
    */
  proc search(text: ?t, ref captures ...?k):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1 + captures.size;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
       got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      } else {
       got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }
      // Now try to coerce the read strings into the captures.
      _handle_captures(text, matches, nmatches, captures);
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }
  /** Search for the first match at any offset without returning any capture
      groups.
    */
  proc search(text: ?t):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
       got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      } else {
       got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }

  /** Check for a match to this regular expression at the start of the
      passed text. As with regexp.search, the capture group arguments
      here can be strings or any type that can reasonably cast from string. If
      a capture group was not matched, the corresponding argument will get the
      default value for its type.

      You can do
       if myregexp.match("some string") {
         ... do something if matched;
       }
   */
  proc match(text: ?t, ref captures ...?k):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1 + captures.size;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
        got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      } else {
        got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      }
      // Now try to coerce the read strings into the captures.
      _handle_captures(text, matches, nmatches, captures);
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }
  proc match(text: ?t):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
       got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      } else {
       got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      }
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }

  /** Split the text by occurrences of this regular expression.
     If capturing parentheses are used in pattern, then the text of all
     groups in the pattern are also returned as part of the resulting array.
     If maxsplit is nonzero, at most maxsplit splits occur, and the
     remaining text is returned as the last element.
   */
  iter split(text: ?t, maxsplit: int = 0)
    where t == string || t == stringPart
  {
    var matches:_ddata(qio_regexp_string_piece_t);
    var ncaptures = qio_regexp_get_ncaptures(_regexp);
    var nmatches = 1 + ncaptures;
    var pos:int;
    var endpos:int;

    on this.home {
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
    }

    if t == stringPart {
      pos = text.offset;
      endpos = pos + text.length;
    } else {
      pos = 0;
      endpos = text.size;
    }
    var splits = 0;
    var maxsplits = maxsplit;
    if maxsplit == 0 then maxsplits = max(int);

    while true {
      var splitstart = 0;
      var splitend = 0;
      var got:bool;
      on this.home {
        got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }

      splits += 1;
      if got && splits <= maxsplits {
        splitstart = matches[0].offset;
        splitend = matches[0].offset + matches[0].len;
      } else {
        splitstart = endpos;
        splitend = endpos;
      }

      if pos < endpos {
        // Yield splitted value
        yield text.substring(pos+1..splitstart);
      } else {
        yield "";
      }

      if got {
        // Yield capture groups
        for i in 1..ncaptures {
          yield text.substring(new reMatch(
                !qio_regexp_string_piece_isnull(matches[i]),
                matches[i].offset,
                matches[i].len));
        }
      }

      // Advance to splitend.
      pos = splitend;

      if splits > maxsplits || !got then break;
    }
    on this.home {
      _ddata_free(matches);
    }
  }
  /* Enumerates matches in the string as well as capture groups.
     Returns tuples of reMatch objects, the 1st is always
      the match for the whole pattern and the rest are the capture groups.
   */
  iter matches(text: ?t, param captures=0, maxmatches: int = max(int))
    where t == string || t == stringPart
  {
    var matches:_ddata(qio_regexp_string_piece_t);
    var nmatches = 1 + captures;
    var pos:int;
    var endpos:int;
    on this.home {
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
    }
    if t == stringPart {
      pos = text.offset;
      endpos = pos + text.length;
    } else {
      pos = 0;
      endpos = text.size;
    }
    var nfound = 0;
    var cur = pos;
    while nfound < maxmatches && cur < endpos {
      var got:bool;
      on this.home {
        got = qio_regexp_match(_regexp, text.c_str(), text.size, cur, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }
      if !got then break;
      param nret = captures+1;
      var ret:nret*reMatch;
      for i in 0..captures {
        ret[i+1] = new reMatch(got, matches[i].offset, matches[i].len);
      }
      yield ret;
      cur = matches[0].offset + matches[0].len;
    }
    on this.home {
      _ddata_free(matches);
    }
  }
  proc subn(repl:string, text: ?t, global = true ):(string, int)
    where t == string || t == stringPart
  {
    var pos:int;
    var endpos:int;
    if t == stringPart {
      pos = text.offset;
      endpos = pos + text.length;
    } else {
      pos = 0;
      endpos = text.size;
    }
    var replaced:c_string; // FIX ME: leak c_string
    var nreplaced:int;
    var replaced_len:int(64);
    if t == stringPart {
      nreplaced = qio_regexp_replace(_regexp, repl.c_str(), repl.size, text.from.c_str(), text.from.size, pos, endpos, global, replaced, replaced_len);
    } else {
      nreplaced = qio_regexp_replace(_regexp, repl.c_str(), repl.size, text.c_str(), text.size, pos, endpos, global, replaced, replaced_len);
    }
    return (toString(replaced), nreplaced);
  }
  proc sub(repl:string, text: ?t, global = true )
    where t == string || t == stringPart
  {
    var (str, count) = subn(repl, text, global);
    return str;
  }
  proc writeThis(f: Writer) {
    var pattern:c_string;
    on this.home {
      qio_regexp_get_pattern(this._regexp, pattern);
    }
    // Note -- this is wrong because we didn't quote
    // and there's no way to get the flags
    f.write("new regexp(\"", pattern, "\")");
  }
  proc readThis(f: Reader) {
    var pattern:c_string;
    // Note -- this is wrong because we didn't quote
    // and there's no way to get the flags
    var litOne = new ioLiteral("new regexp(\"");
    var litTwo = new ioLiteral("\")");
    if(f.read(litOne, pattern, litTwo)) {
      on this.home {
        var opts:qio_regexp_options_t;
        qio_regexp_init_default_options(opts);
        qio_regexp_create_compile(pattern, pattern.size, opts, this._regexp);
      }
    }
  }
}

proc =(ref ret:regexp, x:regexp)
{
  // retain -- release
  if x.home == ret.home {
    on x.home {
      qio_regexp_retain(x._regexp);
      qio_regexp_release(ret._regexp);
    }
    ret._regexp = x._regexp;
  } else {
    var pattern:c_string;
    var options:qio_regexp_options_t;

    on ret.home {
      qio_regexp_release(ret._regexp);
    }
    on x.home {
      qio_regexp_get_pattern(x._regexp, pattern);
      qio_regexp_get_options(x._regexp, options);
    }

    qio_regexp_create_compile(pattern, pattern.size, options, ret._regexp);
  }
}

// TODO -- shouldn't have to write this this way!
pragma "init copy fn"
proc chpl__initCopy(x: regexp) {
  on x.home {
    qio_regexp_retain(x._regexp);
  }
  return x;
}

// Cast regexp to string.
inline proc _cast(type t, x: regexp) where t == string {
  var pattern:c_string;
  on x.home {
    qio_regexp_get_pattern(x._regexp, pattern);
  }
  // FIX ME: leak c_string
  return toString(pattern);
}
// Cast string to regexp
inline proc _cast(type t, x: string) where t == regexp {
  return compile(x);
}

// string methods for searching
proc string.search(needle: string, ignorecase=false):reMatch
{
  // Create a regexp matching the literal for needle
  var re = compile(needle, literal=true, nocapture=true, ignorecase=ignorecase);
  return re.search(this);
}
proc string.search(needle: regexp):reMatch
{
  return needle.search(this);
}
proc string.search(needle: regexp, ref captures ...?k):reMatch
{
  return needle.search(this, (...captures));
}
proc string.match(pattern: regexp):reMatch
{
  return pattern.match(this);
}
proc string.match(pattern: regexp, ref captures ...?k):reMatch
{
  return pattern.match(this, (...captures));
}

iter string.split(pattern: regexp, maxsplit: int = 0)
{
  for v in pattern.split(this, maxsplit) {
    yield v;
  }
}

iter string.matches(pattern:regexp, param captures=0, maxmatches:int=max(int))
{
  for v in pattern.matches(this, captures, maxmatches) {
    yield v;
  }
}

extern type qio_regexp_t;

extern record qio_regexp_options_t {
  var utf8:bool;
  var posix:bool;
  var literal:bool;
  var nocapture:bool;
  // These ones can be set inside the regexp
  var ignorecase:bool; // (?i)
  var multiline:bool; // (?m)
  var dotnl:bool; // (?s)
  var nongreedy:bool; // (?U)
}

extern proc qio_regexp_null():qio_regexp_t;
extern proc qio_regexp_init_default_options(ref options:qio_regexp_options_t);
extern proc qio_regexp_create_compile(str:c_string, strlen:int(64), ref options:qio_regexp_options_t, ref compiled:qio_regexp_t);
extern proc qio_regexp_create_compile_flags(str:c_string, strlen:int(64), flags:c_string, flagslen:int(64), isUtf8:bool, ref compiled:qio_regexp_t);
extern proc qio_regexp_create_compile_flags_2(str:c_void_ptr, strlen:int(64), flags:c_void_ptr, flagslen:int(64), isUtf8:bool, ref compiled:qio_regexp_t);
extern proc qio_regexp_retain(ref compiled:qio_regexp_t);
extern proc qio_regexp_release(ref compiled:qio_regexp_t);

extern proc qio_regexp_get_options(ref regexp:qio_regexp_t, ref options: qio_regexp_options_t);
extern proc qio_regexp_get_pattern(ref regexp:qio_regexp_t, ref pattern: c_string);
extern proc qio_regexp_get_ncaptures(ref regexp:qio_regexp_t):int(64);
extern proc qio_regexp_ok(ref regexp:qio_regexp_t):bool;
extern proc qio_regexp_error(ref regexp:qio_regexp_t):c_string;

extern const QIO_REGEXP_ANCHOR_UNANCHORED:c_int;
extern const QIO_REGEXP_ANCHOR_START:c_int;
extern const QIO_REGEXP_ANCHOR_BOTH:c_int;

extern record qio_regexp_string_piece_t {
  var offset:int(64); // counting from 0, -1 means "NULL"
  var len:int(64);
}

extern proc qio_regexp_string_piece_isnull(ref sp:qio_regexp_string_piece_t):bool;

extern proc qio_regexp_match(ref re:qio_regexp_t, text:c_string, textlen:int(64), startpos:int(64), endpos:int(64), anchor:c_int, submatch:_ddata(qio_regexp_string_piece_t), nsubmatch:int(64)):bool;
extern proc qio_regexp_replace(ref re:qio_regexp_t, repl:c_string, repllen:int(64), text:c_string, textlen:int(64), startpos:int(64), endpos:int(64), global:bool, ref replaced:c_string, ref replaced_len:int(64)):int(64);

// These two could be folded together if we had a way
// to check if a default argument was supplied
// (or any way to use 'nil' in pass-by-ref)
/** Compile a regular expression. This routine will halt if compilation failed
 */
proc compile(pattern: string, utf8=true, posix=false, literal=false, nocapture=false, /*i*/ ignorecase=false, /*m*/ multiline=false, /*s*/ dotnl=false, /*U*/ nongreedy=false):regexp {
  var opts:qio_regexp_options_t;
  qio_regexp_init_default_options(opts);
  opts.utf8 = utf8;
  opts.posix = posix;
  opts.literal = literal;
  opts.nocapture = nocapture;
  opts.ignorecase = ignorecase;
  opts.multiline = multiline;
  opts.dotnl = dotnl;
  opts.nongreedy = nongreedy;

  var ret:regexp;
  qio_regexp_create_compile(pattern.c_str(), pattern.size, opts, ret._regexp);
  if ! qio_regexp_ok(ret._regexp) {
    var err_str = qio_regexp_error(ret._regexp);
    __primitive("chpl_error", "Error " + err_str + " when compiling regexp '" + pattern + "'");
  }
  return ret;
}

/** Compile a regular expression and retrieve an error code if compilation
    failed.
 */
proc compile(pattern: string, out error:syserr, utf8=true, posix=false, literal=false, nocapture=false, /*i*/ ignorecase=false, /*m*/ multiline=false, /*s*/ dotnl=false, /*U*/ nongreedy=false):regexp {
  var opts:qio_regexp_options_t;
  qio_regexp_init_default_options(opts);
  opts.utf8 = utf8;
  opts.posix = posix;
  opts.literal = literal;
  opts.nocapture = nocapture;
  opts.ignorecase = ignorecase;
  opts.multiline = multiline;
  opts.dotnl = dotnl;
  opts.nongreedy = nongreedy;

  var ret:regexp;
  qio_regexp_create_compile(pattern, pattern.size, opts, ret._regexp);

  if qio_regexp_ok(ret._regexp) {
    error = ENOERR;
  } else {
    error = qio_format_error_bad_regexp();
  }

  return ret;
}

/** Regular expression search routines also can work with
    a stringPart, which refers to a substring within another
    string.
 */
record stringPart {
  var offset:int;
  var length:int;
  var from:string;
}

/** The reMatch record records a regular expression search match
    or a capture group.
    Regular expression search routines normally return one of these.
    Also, this type can be passed as a capture group argument.
    Lastly, something of type reMatch can be checked for a match
    in a simple if statement, as in:
      var m:reMatch = ....;
      if m then do_something_if_matched();
      if !m then do_something_if_not_matched();

 */
record reMatch {
  var matched:bool;
  var offset:int; // 0-based, -1 if matched==false
  var length:int; // 0 if matched==false
}

proc _to_reMatch(ref p:qio_regexp_string_piece_t):reMatch {
  if qio_regexp_string_piece_isnull(p) {
    return new reMatch(false, -1, 0);
  } else {
    return new reMatch(true, p.offset, p.len);
  }
}

inline proc !(m: reMatch) return !m.matched;
inline proc _cond_test(m: reMatch) return m.matched;

/** Use someString.substring(anReMatch) to extract the part
    of a string matching a regular expression or capture group.
 */
proc string.substring(m:reMatch) {
  if m.matched then return this.substring(m.offset+1..#m.length);
  else return "";
}

/** This class represents a compiled regular expression. Regular expressions
    are cached on a per-thread basis and are reference counted.
    To create a compiled regular expression, use the compile function.
  */
record regexp {
  var home: locale = here;
  var _regexp:qio_regexp_t = qio_regexp_null();

  proc ok:bool {
    return qio_regexp_ok(_regexp);
  }
  proc error():string {
    return qio_regexp_error(_regexp);
  }

  // note - more = overloads are below.
  proc ~regexp() {
    qio_regexp_release(_regexp);
    _regexp = qio_regexp_null();
  }

  proc _handle_captures(text: string, matches:_ddata(qio_regexp_string_piece_t), nmatches:int, ref captures) {
    assert(nmatches >= captures.size);
    for param i in 1..captures.size {
      var m = _to_reMatch(matches[i]);
      if captures[i].type == reMatch {
        captures[i] = m;
      } else {
        if m.matched {
          captures[i] = text.substring(m):captures[i].type;
        } else {
          var empty:captures[i].type;
          captures[i] = empty;
        }
      }
    }
  }

  // Note - we would only need one version of these routines
  // if we had args ...?k supporting 0 args, or if tuples support zero length
  /** Search within the passed text (which must be a string or a stringPart)
      for the first match at any offset to this regular expression. Returns a
      match object. This routine will try matching the regular expression at
      different offsets until a match is found. If you want to only match at
      the beginning of the pattern, you can start your pattern with ^ or
      use regexp.match. The capture group arguments here can be strings
      or any type that can reasonably cast from string. If a capture group
      was not matched, the corresponding argument will get the default value
      for its type.
    */
  proc search(text: ?t, ref captures ...?k):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1 + captures.size;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
       got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      } else {
       got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }
      // Now try to coerce the read strings into the captures.
      _handle_captures(text, matches, nmatches, captures);
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }
  /** Search for the first match at any offset without returning any capture
      groups.
    */
  proc search(text: ?t):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
       got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      } else {
       got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }

  /** Check for a match to this regular expression at the start of the
      passed text. As with regexp.search, the capture group arguments
      here can be strings or any type that can reasonably cast from string. If
      a capture group was not matched, the corresponding argument will get the
      default value for its type.

      You can do
       if myregexp.match("some string") {
         ... do something if matched;
       }
   */
  proc match(text: ?t, ref captures ...?k):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1 + captures.size;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
        got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      } else {
        got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      }
      // Now try to coerce the read strings into the captures.
      _handle_captures(text, matches, nmatches, captures);
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }
  proc match(text: ?t):reMatch
    where t == string || t == stringPart
  {
    var ret:reMatch;
    on this.home {
      var pos:int;
      var endpos:int;
      if t == stringPart {
        pos = text.offset;
        endpos = pos + text.length;
      } else {
        pos = 0;
        endpos = text.size;
      }
      var matches:_ddata(qio_regexp_string_piece_t);
      var nmatches = 1;
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
      var got:bool;
      if t == stringPart {
       got = qio_regexp_match(_regexp, text.from.c_str(), text.from.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      } else {
       got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_START, matches, nmatches);
      }
      // Now return where we matched.
      ret = new reMatch(got, matches[0].offset, matches[0].len);
      _ddata_free(matches);
    }
    return ret;
  }

  /** Split the text by occurrences of this regular expression.
     If capturing parentheses are used in pattern, then the text of all
     groups in the pattern are also returned as part of the resulting array.
     If maxsplit is nonzero, at most maxsplit splits occur, and the
     remaining text is returned as the last element.
   */
  iter split(text: ?t, maxsplit: int = 0)
    where t == string || t == stringPart
  {
    var matches:_ddata(qio_regexp_string_piece_t);
    var ncaptures = qio_regexp_get_ncaptures(_regexp);
    var nmatches = 1 + ncaptures;
    var pos:int;
    var endpos:int;

    on this.home {
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
    }

    if t == stringPart {
      pos = text.offset;
      endpos = pos + text.length;
    } else {
      pos = 0;
      endpos = text.size;
    }
    var splits = 0;
    var maxsplits = maxsplit;
    if maxsplit == 0 then maxsplits = max(int);

    while true {
      var splitstart = 0;
      var splitend = 0;
      var got:bool;
      on this.home {
        got = qio_regexp_match(_regexp, text.c_str(), text.size, pos, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }

      splits += 1;
      if got && splits <= maxsplits {
        splitstart = matches[0].offset;
        splitend = matches[0].offset + matches[0].len;
      } else {
        splitstart = endpos;
        splitend = endpos;
      }

      if pos < endpos {
        // Yield splitted value
        yield text.substring(pos+1..splitstart);
      } else {
        yield "";
      }

      if got {
        // Yield capture groups
        for i in 1..ncaptures {
          yield text.substring(new reMatch(
                !qio_regexp_string_piece_isnull(matches[i]),
                matches[i].offset,
                matches[i].len));
        }
      }

      // Advance to splitend.
      pos = splitend;

      if splits > maxsplits || !got then break;
    }
    on this.home {
      _ddata_free(matches);
    }
  }
  /* Enumerates matches in the string as well as capture groups.
     Returns tuples of reMatch objects, the 1st is always
      the match for the whole pattern and the rest are the capture groups.
   */
  iter matches(text: ?t, param captures=0, maxmatches: int = max(int))
    where t == string || t == stringPart
  {
    var matches:_ddata(qio_regexp_string_piece_t);
    var nmatches = 1 + captures;
    var pos:int;
    var endpos:int;
    on this.home {
      matches = _ddata_allocate(qio_regexp_string_piece_t, nmatches);
    }
    if t == stringPart {
      pos = text.offset;
      endpos = pos + text.length;
    } else {
      pos = 0;
      endpos = text.size;
    }
    var nfound = 0;
    var cur = pos;
    while nfound < maxmatches && cur < endpos {
      var got:bool;
      on this.home {
        got = qio_regexp_match(_regexp, text.c_str(), text.size, cur, endpos, QIO_REGEXP_ANCHOR_UNANCHORED, matches, nmatches);
      }
      if !got then break;
      param nret = captures+1;
      var ret:nret*reMatch;
      for i in 0..captures {
        ret[i+1] = new reMatch(got, matches[i].offset, matches[i].len);
      }
      yield ret;
      cur = matches[0].offset + matches[0].len;
    }
    on this.home {
      _ddata_free(matches);
    }
  }
  proc subn(repl:string, text: ?t, global = true ):(string, int)
    where t == string || t == stringPart
  {
    var pos:int;
    var endpos:int;
    if t == stringPart {
      pos = text.offset;
      endpos = pos + text.length;
    } else {
      pos = 0;
      endpos = text.size;
    }
    var replaced:c_string; // FIX ME: leak c_string
    var nreplaced:int;
    var replaced_len:int(64);
    if t == stringPart {
      nreplaced = qio_regexp_replace(_regexp, repl.c_str(), repl.size, text.from.c_str(), text.from.size, pos, endpos, global, replaced, replaced_len);
    } else {
      nreplaced = qio_regexp_replace(_regexp, repl.c_str(), repl.size, text.c_str(), text.size, pos, endpos, global, replaced, replaced_len);
    }
    return (toString(replaced), nreplaced);
  }
  proc sub(repl:string, text: ?t, global = true )
    where t == string || t == stringPart
  {
    var (str, count) = subn(repl, text, global);
    return str;
  }
  proc writeThis(f: Writer) {
    var pattern:c_string;
    on this.home {
      qio_regexp_get_pattern(this._regexp, pattern);
    }
    // Note -- this is wrong because we didn't quote
    // and there's no way to get the flags
    f.write("new regexp(\"", pattern, "\")");
  }
  proc readThis(f: Reader) {
    var pattern:c_string;
    // Note -- this is wrong because we didn't quote
    // and there's no way to get the flags
    var litOne = new ioLiteral("new regexp(\"");
    var litTwo = new ioLiteral("\")");
    if(f.read(litOne, pattern, litTwo)) {
      on this.home {
        var opts:qio_regexp_options_t;
        qio_regexp_init_default_options(opts);
        qio_regexp_create_compile(pattern, pattern.size, opts, this._regexp);
      }
    }
  }
}

proc =(ref ret:regexp, x:regexp)
{
  // retain -- release
  if x.home == ret.home {
    on x.home {
      qio_regexp_retain(x._regexp);
      qio_regexp_release(ret._regexp);
    }
    ret._regexp = x._regexp;
  } else {
    var pattern:c_string;
    var options:qio_regexp_options_t;

    on ret.home {
      qio_regexp_release(ret._regexp);
    }
    on x.home {
      qio_regexp_get_pattern(x._regexp, pattern);
      qio_regexp_get_options(x._regexp, options);
    }

    qio_regexp_create_compile(pattern, pattern.size, options, ret._regexp);
  }
}

// TODO -- shouldn't have to write this this way!
pragma "init copy fn"
proc chpl__initCopy(x: regexp) {
  on x.home {
    qio_regexp_retain(x._regexp);
  }
  return x;
}

// Cast regexp to string.
inline proc _cast(type t, x: regexp) where t == string {
  var pattern:c_string;
  on x.home {
    qio_regexp_get_pattern(x._regexp, pattern);
  }
  // FIX ME: leak c_string
  return toString(pattern);
}
// Cast string to regexp
inline proc _cast(type t, x: string) where t == regexp {
  return compile(x);
}

// string methods for searching
proc string.search(needle: string, ignorecase=false):reMatch
{
  // Create a regexp matching the literal for needle
  var re = compile(needle, literal=true, nocapture=true, ignorecase=ignorecase);
  return re.search(this);
}
proc string.search(needle: regexp):reMatch
{
  return needle.search(this);
}
proc string.search(needle: regexp, ref captures ...?k):reMatch
{
  return needle.search(this, (...captures));
}
proc string.match(pattern: regexp):reMatch
{
  return pattern.match(this);
}
proc string.match(pattern: regexp, ref captures ...?k):reMatch
{
  return pattern.match(this, (...captures));
}

iter string.split(pattern: regexp, maxsplit: int = 0)
{
  for v in pattern.split(this, maxsplit) {
    yield v;
  }
}

iter string.matches(pattern:regexp, param captures=0, maxmatches:int=max(int))
{
  for v in pattern.matches(this, captures, maxmatches) {
    yield v;
  }
}


spin wait to sync fix

spin wait to sync fix

Change spin waiting on an int into looping until a sync int gets above a value.
Another thread bumps the value a few times to get it high enough.

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@13564 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

var a: sync int;
var b = 0;

cobegin {
  {
    b = 1;
    while (a < 3) {
      b = 2;
    }
  }
  a = 1;
  a += 1;
  a += 1;
}

if (b==2) then {
  writeln ("b is good");
} else {
  writeln ("b is bad");
}

var a = 0;
var b = 0;

cobegin {
  {
    b = 1;
    while (a==0) {
      b = 2;
    }
  }
  a = 1;
}

if (b==1 || b==2) then {
  writeln ("b is good");
} else {
  writeln ("b is bad");
}


Remove "no doc" pragma now that chpldoc recognizes private.

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  This module contains several iterators that can be used to drive a `forall`
  loop by performing dynamic and adaptive splitting of a range's iterations.

  For more information, see *User-Defined Parallel Zippered Iterators
  in Chapel*. Bradford L. Chamberlain, Sung-Eun Choi, Steven J. Deitz,
  Angeles Navarro. *PGAS 2011: Fifth Conference on Partitioned Global
  Address Space Programming Models*, October 2011.
*/
module AdvancedIters {

/*
   An atomic test-and-set lock.
*/
pragma "no doc"
record vlock {
  var l: atomic bool;
  proc lock() {
    on this do while l.testAndSet() != false do chpl_task_yield();
  }
  proc unlock() {
    l.write(false);
  }
}

/*
   Toggle debugging output.
*/
config param debugAdvancedIters:bool=false;

//************************* Dynamic iterator

/*
   The serial version of the dynamic iterator.

   Equivalent to serial iteration over the range ``c``.
*/
iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

/*

  :arg c: The range to iterate over. The length of the range must be greater
          than zero.
  :type c: range(?)

  :arg chunkSize: The size of chunks to be yielded to each thread. Must be
                  greater than zero.
  :type chunkSize: int

  :arg numTasks: The number of tasks to use. Must be >= zero. If this argument
                 has the value 0, it will use the value indicated by
                 ``dataParTasksPerLocale``.
  :type numTasks: int

  :yields: Indices in the range ``c``.

  This iterator is equivalent to the dynamic scheduling approach of OpenMP.

  Given an input range ``c``, each task is assigned chunks of size
  ``chunkSize`` from ``c`` (or the remaining iterations if there are fewer
  than ``chunkSize``). This continues until there are no remaining iterations
  in ``c``.

*/
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  assert(chunkSize > 0); // caller's responsibility

  // # of tasks the range can fill. (fast) ceil so all work is represented
  const chunkTasks = divceilpos(c.length, chunkSize): int;

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks = min(chunkTasks, defaultNumTasks(numTasks));

  type rType=c.type;

  // We're going to have to densify at some point, might as well
  // do it early and make range slicing easier.
  const remain:rType=densify(c,c);

  // If the number of tasks is insufficient, yield in serial
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Dynamic Iterator: serial execution because there is not enough work");
    yield (remain,);
  } else {
    var moreWork : bool =true;
    var curIndex : atomic remain.low.type;
    curIndex.write(remain.low);

    coforall tid in 0..#nTasks with (ref remain, ref moreWork, ref curIndex) do {
      while moreWork do {
        // There is local work in remain
        const low = curIndex.fetchAdd(chunkSize);
        var high = low + chunkSize-1;

        if low > remain.high {
          break;
        } else if high > remain.high {
          high = remain.high;
          moreWork = false;
        }

        const current:rType = remain(low .. high);

        if high >= low then {
          if debugAdvancedIters then
            writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", unDensify(current,c), " yielded as ", current);
          yield (current,);
        }
      }
    }
  }
}

// Follower
pragma "no doc"
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator

/*
   The serial version of the guided iterator.

   Equivalent to serial iteration over the range ``c``.
*/
iter guided(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

/*

  :arg c: The range to iterate over. Must have a length greater than zero.
  :type c: range(?)

  :arg numTasks: The number of tasks to use. Must be >= zero. If this argument
                 has the value 0, it will use the value indicated by
                 ``dataParTasksPerLocale``.
  :type numTasks: int

  :yields: Indices in the range ``c``.

  This iterator is equivalent to the guided policy of OpenMP: Given an input
  range ``c``, each task is assigned chunks of variable size, until there are
  no remaining iterations in ``c``. The size of each chunk is the number of
  unassigned iterations divided by the number of tasks, ``numTasks``. The size
  decreases approximately exponentially to 1. The splitting strategy is
  therefore adaptive.

*/
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=min(c.length, defaultNumTasks(numTasks));
  type rType=c.type;
  var remain:rType = densify(c,c);
  // If the number of tasks is insufficient, yield in serial
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Guided Iterator: serial execution because there is not enoguh work");
    yield (remain,);
  }

  else {
    var undone=true;
    const factor=nTasks;
    var lock : vlock;

    coforall tid in 0..#nTasks with (ref remain, ref undone, ref lock) do {
      while undone do {
        // There is local work in remain(tid)
        const current:rType=adaptSplit(remain, factor, undone, lock);
        if current.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", unDensify(current,c), " yielded as ", current);
          yield (current,);
        }
      }
    }
  }
}

// Follower
pragma "no doc"
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
/*
   The serial version of the adaptive iterator.

   Equivalent to serial iteration over the range ``c``.
*/
iter adaptive(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator
/*
The enum used to represent adaptive methods.

- ``Whole``
  Each task without work tries to steal from its neighbor range
  until it exhausts that range. Then the task continues with the next
  neighbor range, and so on until there is no more work. This is the default
  policy.

- ``RoundRobin``
  Each task without work tries to steal once from its neighbor range, next
  from the following neighbor range and so on in a round-robin way until
  there is no more work.

- ``WholeTail``
  Similar to the ``Whole`` method, but now the splitting in the victim
  range is performed from its tail.
*/
enum Method {
  Whole = 0,
  RoundRobin = 1,
  WholeTail = 2
};

/*
  Used to select the adaptive stealing method. Defaults to ``Whole``.
  See :data:`Method` for more information.
*/
config param methodStealing = Method.Whole;

/*

  :arg c: The range to iterate over. Must have a length greater than zero.
  :type c: range(?)

  :arg numTasks: The number of tasks to use. Must be >= zero. If this argument
                 has the value 0, it will use the value indicated by
                 ``dataParTasksPerLocale``.
  :type numTasks: int

  :yields: Indices in the range ``c``.

  This iterator implements a naive adaptive binary splitting work-stealing
  strategy: Initially the leader iterator distributes the range to split, ``c``,
  evenly among the ``numTasks`` tasks.

  Then, each task performs adaptive splitting on its local sub-range's iterations.
  When a task exhausts its local iterations, it steals and splits from the
  range of another task (the victim). The splitting method on the local range
  and on the victim range is binary: i.e. the size of each chunk is computed as
  the number of unassigned iterations divided by 2. There are three stealing
  strategies that can be selected at compile time using the config param
  :param:`methodStealing`.
*/
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  /*if (methodStealing != Method.Whole || methodStealing != Method.WholeTail) then
    compilerError("methodStealing value must be between 0 and 2");*/

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=min(c.length, defaultNumTasks(numTasks));
  type rType=c.type;

  // If the number of tasks is insufficient, yield in serial
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    yield (densify(c,c),);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var locks:[SpaceThreads] vlock; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrier : atomic int;

    // Start the parallel work
    coforall tid in 0..#nTasks with (ref localWork, ref moreLocalWork, ref moreWork, ref r) do {

      // Step 1: Initial range per Thread/Task

      // Initial Local range in localWork[tid]
      const chunkSize = c.length/nTasks;
      localWork[tid]=
      if tid==nTasks-1 then
        r#(chunkSize*(nTasks-1)-r.length)
      else
        (r+tid*chunkSize)#chunkSize;
      barrier.add(1);

      if debugAdvancedIters then
        writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier.waitFor(nTasks);

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
        // There is local work
        // The current range we get after splitting locally
        const zeroBasedIters:rType=adaptSplit(localWork[tid], factorSteal, moreLocalWork[tid], locks[tid]);
        if zeroBasedIters.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
          yield (zeroBasedIters,);
        }
      }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
        if debugAdvancedIters then
          writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

        // Perform the spliting at the victim remaining range

        if methodStealing == Method.RoundRobin {
          if moreLocalWork[victim] then {
            // There is work in victim
            const zeroBasedIters2:rType=adaptSplit(localWork[victim], factorSteal, moreLocalWork[victim], locks[victim]);
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          } else {
            stealFailed=true;
          }
        } else {
          while moreLocalWork[victim] do {
            // There is work in victim
            const zeroBasedIters2:rType=adaptSplit(localWork[victim], factorSteal, moreLocalWork[victim], locks[victim], methodStealing==Method.WholeTail);
                                          //after splitting from a victim range
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          }
        }

        // If here, then it can have failed the stealing intent at the victim (method 1),
        // or we have exhausted the victim range (methods 0, 2)

        if (methodStealing==Method.Whole || methodStealing==Method.WholeTail || (methodStealing==Method.RoundRobin && stealFailed)) then {
          nVisitedVictims += 1; // Signal that there is no more work in victim
          stealFailed=false;
          if debugAdvancedIters then
            writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        }
        // Check if there is no more work
        if nVisitedVictims >= nTasks-1 then
          moreWork=false; // Signal that there is no more work in any victim
        else {  // There can be still work in other victim
          victim=(victim+1) % nTasks; // New victim to steal
          if methodStealing==Method.RoundRobin && victim==tid then
            victim=(victim+1) % nTasks;
        }
      }
    }
  }
}

// Follower
pragma "no doc"
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
private proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.maxTaskPar;
      else dnTasks=dataParTasksPerLocale;
  } else if nTasks<0 then {
    halt("'numTasks' is negative");
  }
  return dnTasks;
}

private proc adaptSplit(ref rangeToSplit:range(?), splitFactor:int, ref itLeft:bool, ref lock : vlock, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  const profThreshold=1;

  lock.lock();
  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  const direction = if splitTail then -1 else 1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  lock.unlock();
  return firstRange;
}

}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  This module contains several iterators that can be used to drive a `forall`
  loop by performing dynamic and adaptive splitting of a range's iterations.

  For more information, see *User-Defined Parallel Zippered Iterators
  in Chapel*. Bradford L. Chamberlain, Sung-Eun Choi, Steven J. Deitz,
  Angeles Navarro. *PGAS 2011: Fifth Conference on Partitioned Global
  Address Space Programming Models*, October 2011.
*/
module AdvancedIters {

/*
   An atomic test-and-set lock.
*/
pragma "no doc"
record vlock {
  var l: atomic bool;
  proc lock() {
    on this do while l.testAndSet() != false do chpl_task_yield();
  }
  proc unlock() {
    l.write(false);
  }
}

/*
   Toggle debugging output.
*/
config param debugAdvancedIters:bool=false;

//************************* Dynamic iterator

/*
   The serial version of the dynamic iterator.

   Equivalent to serial iteration over the range ``c``.
*/
iter dynamic(c:range(?), chunkSize:int, numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial dynamic Iterator. Working with range ", c);

  for i in c do yield i;
}

// Parallel iterator

/*

  :arg c: The range to iterate over. The length of the range must be greater
          than zero.
  :type c: range(?)

  :arg chunkSize: The size of chunks to be yielded to each thread. Must be
                  greater than zero.
  :type chunkSize: int

  :arg numTasks: The number of tasks to use. Must be >= zero. If this argument
                 has the value 0, it will use the value indicated by
                 ``dataParTasksPerLocale``.
  :type numTasks: int

  :yields: Indices in the range ``c``.

  This iterator is equivalent to the dynamic scheduling approach of OpenMP.

  Given an input range ``c``, each task is assigned chunks of size
  ``chunkSize`` from ``c`` (or the remaining iterations if there are fewer
  than ``chunkSize``). This continues until there are no remaining iterations
  in ``c``.

*/
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int=0)
where tag == iterKind.leader
{
  assert(chunkSize > 0); // caller's responsibility

  // # of tasks the range can fill. (fast) ceil so all work is represented
  const chunkTasks = divceilpos(c.length, chunkSize): int;

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks = min(chunkTasks, defaultNumTasks(numTasks));

  type rType=c.type;

  // We're going to have to densify at some point, might as well
  // do it early and make range slicing easier.
  const remain:rType=densify(c,c);

  // If the number of tasks is insufficient, yield in serial
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Dynamic Iterator: serial execution because there is not enough work");
    yield (remain,);
  } else {
    var moreWork : bool =true;
    var curIndex : atomic remain.low.type;
    curIndex.write(remain.low);

    coforall tid in 0..#nTasks with (ref remain, ref moreWork, ref curIndex) do {
      while moreWork do {
        // There is local work in remain
        const low = curIndex.fetchAdd(chunkSize);
        var high = low + chunkSize-1;

        if low > remain.high {
          break;
        } else if high > remain.high {
          high = remain.high;
          moreWork = false;
        }

        const current:rType = remain(low .. high);

        if high >= low then {
          if debugAdvancedIters then
            writeln("Parallel dynamic Iterator. Working at tid ", tid, " with range ", unDensify(current,c), " yielded as ", current);
          yield (current,);
        }
      }
    }
  }
}

// Follower
pragma "no doc"
iter dynamic(param tag:iterKind, c:range(?), chunkSize:int, numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Guided iterator

/*
   The serial version of the guided iterator.

   Equivalent to serial iteration over the range ``c``.
*/
iter guided(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial guided Iterator. Working with range ", c);

  for i in c do yield i;
}

/*

  :arg c: The range to iterate over. Must have a length greater than zero.
  :type c: range(?)

  :arg numTasks: The number of tasks to use. Must be >= zero. If this argument
                 has the value 0, it will use the value indicated by
                 ``dataParTasksPerLocale``.
  :type numTasks: int

  :yields: Indices in the range ``c``.

  This iterator is equivalent to the guided policy of OpenMP: Given an input
  range ``c``, each task is assigned chunks of variable size, until there are
  no remaining iterations in ``c``. The size of each chunk is the number of
  unassigned iterations divided by the number of tasks, ``numTasks``. The size
  decreases approximately exponentially to 1. The splitting strategy is
  therefore adaptive.

*/
iter guided(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader
{
  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=min(c.length, defaultNumTasks(numTasks));
  type rType=c.type;
  var remain:rType = densify(c,c);
  // If the number of tasks is insufficient, yield in serial
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Guided Iterator: serial execution because there is not enoguh work");
    yield (remain,);
  }

  else {
    var undone=true;
    const factor=nTasks;
    var lock : vlock;

    coforall tid in 0..#nTasks with (ref remain, ref undone, ref lock) do {
      while undone do {
        // There is local work in remain(tid)
        const current:rType=adaptSplit(remain, factor, undone, lock);
        if current.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel guided Iterator. Working at tid ", tid, " with range ", unDensify(current,c), " yielded as ", current);
          yield (current,);
        }
      }
    }
  }
}

// Follower
pragma "no doc"
iter guided(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower

{
  type rType=c.type;
  const current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Adaptive work-stealing iterator
/*
   The serial version of the adaptive iterator.

   Equivalent to serial iteration over the range ``c``.
*/
iter adaptive(c:range(?), numTasks:int=0) {

  if debugAdvancedIters then
    writeln("Serial adaptive work-stealing Iterator. Working with range ", c);

  for i in c do yield i;

}

// Parallel iterator
/*
The enum used to represent adaptive methods.

- ``Whole``
  Each task without work tries to steal from its neighbor range
  until it exhausts that range. Then the task continues with the next
  neighbor range, and so on until there is no more work. This is the default
  policy.

- ``RoundRobin``
  Each task without work tries to steal once from its neighbor range, next
  from the following neighbor range and so on in a round-robin way until
  there is no more work.

- ``WholeTail``
  Similar to the ``Whole`` method, but now the splitting in the victim
  range is performed from its tail.
*/
enum Method {
  Whole = 0,
  RoundRobin = 1,
  WholeTail = 2
};

/*
  Used to select the adaptive stealing method. Defaults to ``Whole``.
  See :data:`Method` for more information.
*/
config param methodStealing = Method.Whole;

/*

  :arg c: The range to iterate over. Must have a length greater than zero.
  :type c: range(?)

  :arg numTasks: The number of tasks to use. Must be >= zero. If this argument
                 has the value 0, it will use the value indicated by
                 ``dataParTasksPerLocale``.
  :type numTasks: int

  :yields: Indices in the range ``c``.

  This iterator implements a naive adaptive binary splitting work-stealing
  strategy: Initially the leader iterator distributes the range to split, ``c``,
  evenly among the ``numTasks`` tasks.

  Then, each task performs adaptive splitting on its local sub-range's iterations.
  When a task exhausts its local iterations, it steals and splits from the
  range of another task (the victim). The splitting method on the local range
  and on the victim range is binary: i.e. the size of each chunk is computed as
  the number of unassigned iterations divided by 2. There are three stealing
  strategies that can be selected at compile time using the config param
  :param:`methodStealing`.
*/
iter adaptive(param tag:iterKind, c:range(?), numTasks:int=0)
where tag == iterKind.leader

{
  /*if (methodStealing != Method.Whole || methodStealing != Method.WholeTail) then
    compilerError("methodStealing value must be between 0 and 2");*/

  // Check if the number of tasks is 0, in that case it returns a default value
  const nTasks=min(c.length, defaultNumTasks(numTasks));
  type rType=c.type;

  // If the number of tasks is insufficient, yield in serial
  if c.length == 0 then halt("The range is empty");
  if nTasks == 1 then {
    if debugAdvancedIters then
      writeln("Adaptive work-stealing Iterator: serial execution because there is not enough work");
    yield (densify(c,c),);

  }
  else {
    const r:rType=densify(c,c);
    const SpaceThreads:domain(1)=0..#nTasks;
    var localWork:[SpaceThreads] rType; // The remaining range to split on each Thread
    var moreLocalWork:[SpaceThreads] bool=true; // bool var to signal there is still work on each local range
    var locks:[SpaceThreads] vlock; // sync var to control the splitting on each Thread
    const factor:int=2;

    const factorSteal:int=2;
    var moreWork:bool=true; // A global var to control the termination

    // Variables to put a barrier to ensure the initial range is computed on each Thread
    var barrier : atomic int;

    // Start the parallel work
    coforall tid in 0..#nTasks with (ref localWork, ref moreLocalWork, ref moreWork, ref r) do {

      // Step 1: Initial range per Thread/Task

      // Initial Local range in localWork[tid]
      const chunkSize = c.length/nTasks;
      localWork[tid]=
      if tid==nTasks-1 then
        r#(chunkSize*(nTasks-1)-r.length)
      else
        (r+tid*chunkSize)#chunkSize;
      barrier.add(1);

      if debugAdvancedIters then
        writeln("Parallel adaptive work-stealing Iterator. Working at tid ", tid, " with initial range ", localWork[tid]);

      // A barrier waiting for each thread to finish the initial assignment
      barrier.waitFor(nTasks);

      // Step 2: While there is work at tid, do splitting

      while moreLocalWork[tid] do {
        // There is local work
        // The current range we get after splitting locally
        const zeroBasedIters:rType=adaptSplit(localWork[tid], factorSteal, moreLocalWork[tid], locks[tid]);
        if zeroBasedIters.length !=0 then {
          if debugAdvancedIters then
            writeln("Parallel adaptive Iterator. Working locally at tid ", tid, " with range yielded as ", zeroBasedIters);
          yield (zeroBasedIters,);
        }
      }

      // Step3: Task tid finished its work, so it will try to steal from a neighbor

      var nVisitedVictims:int=0;
      var victim=(tid+1) % nTasks;
      var stealFailed:bool=false;

      while moreWork do {
        if debugAdvancedIters then
          writeln("Entering at Stealing phase in tid ", tid," with victim ", victim, " using method of Stealing ", methodStealing);

        // Perform the spliting at the victim remaining range

        if methodStealing == Method.RoundRobin {
          if moreLocalWork[victim] then {
            // There is work in victim
            const zeroBasedIters2:rType=adaptSplit(localWork[victim], factorSteal, moreLocalWork[victim], locks[victim]);
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          } else {
            stealFailed=true;
          }
        } else {
          while moreLocalWork[victim] do {
            // There is work in victim
            const zeroBasedIters2:rType=adaptSplit(localWork[victim], factorSteal, moreLocalWork[victim], locks[victim], methodStealing==Method.WholeTail);
                                          //after splitting from a victim range
            if zeroBasedIters2.length !=0 then {
              if debugAdvancedIters then
                writeln("Range stealed at victim ", victim," yielded as ", zeroBasedIters2," by tid ", tid);
              yield (zeroBasedIters2,);
            }
          }
        }

        // If here, then it can have failed the stealing intent at the victim (method 1),
        // or we have exhausted the victim range (methods 0, 2)

        if (methodStealing==Method.Whole || methodStealing==Method.WholeTail || (methodStealing==Method.RoundRobin && stealFailed)) then {
          nVisitedVictims += 1; // Signal that there is no more work in victim
          stealFailed=false;
          if debugAdvancedIters then
            writeln("Failed Stealing intent at tid ", tid," with victim ", victim, " and total no. of visited victims ", nVisitedVictims);
        }
        // Check if there is no more work
        if nVisitedVictims >= nTasks-1 then
          moreWork=false; // Signal that there is no more work in any victim
        else {  // There can be still work in other victim
          victim=(victim+1) % nTasks; // New victim to steal
          if methodStealing==Method.RoundRobin && victim==tid then
            victim=(victim+1) % nTasks;
        }
      }
    }
  }
}

// Follower
pragma "no doc"
iter adaptive(param tag:iterKind, c:range(?), numTasks:int, followThis)
where tag == iterKind.follower
{
  type rType=c.type;
  var current:rType=unDensify(followThis(1),c);
  if debugAdvancedIters then
    writeln("Follower received range ", followThis, " ; shifting to ", current);
  for i in current do {
    yield i;
  }
}

//************************* Helper functions
pragma "no doc"
private proc defaultNumTasks(nTasks:int)
{
  var dnTasks=nTasks;
  if nTasks==0 then {
    if dataParTasksPerLocale==0 then dnTasks=here.maxTaskPar;
      else dnTasks=dataParTasksPerLocale;
  } else if nTasks<0 then {
    halt("'numTasks' is negative");
  }
  return dnTasks;
}

pragma "no doc"
private proc adaptSplit(ref rangeToSplit:range(?), splitFactor:int, ref itLeft:bool, ref lock : vlock, splitTail:bool=false)
{
  type rType=rangeToSplit.type;
  type lenType=rangeToSplit.length.type;
  var totLen, size:lenType;
  const profThreshold=1;

  lock.lock();
  totLen=rangeToSplit.length;
  if totLen > profThreshold then
    size=max(totLen/splitFactor, profThreshold);
  else {
    size=totLen;
    itLeft=false;}
  const direction = if splitTail then -1 else 1;
  const firstRange:rType=rangeToSplit#(direction*size);
  rangeToSplit=rangeToSplit#(direction*(size-totLen));
  lock.unlock();
  return firstRange;
}

}


Correctly (safely) increment atomic int in solution 21.

/*
 * Amicable numbers
 */

use MathFunctions;

config const maxNum = 10000,
  printPairs = false;

proc main() {
  var allSums: [{1..maxNum-1}] int,
    sum: atomic int;

  // Calculate all the sumDivisors first.
  forall n in allSums.domain {
    allSums[n] = sumDivisors(n);
  }

  // Iterate over all pairs where a < b. This ensures that duplicates are not
  // added to sum. For example, without this, both 220,284 and 284,220 would be
  // considered.
  forall a in 1..maxNum-1 {
    forall b in a+1..maxNum-1 {
      if allSums[a] == b && allSums[b] == a {
        sum.fetchAdd(a + b);

        if printPairs {
          writef("d(%n) = %n\nd(%n) = %n\n", a, b, b, a);
        }
      }
    }
  }

  writeln(sum.read());
}

// Returns true if a and b are amicable numbers. If sumDivisors(a) == b,
// sumDivisors(b) == a, and a != b, then a and b are amicable.
proc amicable(a, b) {
  return a != b && sumDivisors(a) == b && sumDivisors(b) == a;
}

/*
 * Amicable numbers
 */

use MathFunctions;

config const maxNum = 10000,
  printPairs = false;

proc main() {
  var allSums: [{1..maxNum-1}] int,
    sum: atomic int;

  // Calculate all the sumDivisors first.
  forall n in allSums.domain {
    allSums[n] = sumDivisors(n);
  }

  // Iterate over all pairs where a < b. This ensures that duplicates are not
  // added to sum. For example, without this, both 220,284 and 284,220 would be
  // considered.
  forall a in 1..maxNum-1 {
    forall b in a+1..maxNum-1 {
      if allSums[a] == b && allSums[b] == a {
        sum.write(sum.read() + a + b);

        if printPairs {
          writef("d(%n) = %n\nd(%n) = %n\n", a, b, b, a);
        }
      }
    }
  }

  writeln(sum.read());
}

// Returns true if a and b are amicable numbers. If sumDivisors(a) == b,
// sumDivisors(b) == a, and a != b, then a and b are amicable.
proc amicable(a, b) {
  return a != b && sumDivisors(a) == b && sumDivisors(b) == a;
}