Changing prettyPrint format to use brackets instead of curly braces. This is done to be more consistent with array-literal syntax

/*

  Associative Primer

  This is a tutorial on Chapel's associative domains and arrays.

  It may be beneficial to read the 'arrays.chpl' and 'domains.chpl'
  primers before proceeding if you're not already familiar with Chapel's
  domains and arrays.

*/

//
// Associative domains are commonly used to represent arbitrary sets, or to
// represent the set of keys for a dictionary/hash table.
//
// The current implementation supports most Chapel types, and will eventually
// support any language-defined type.
{
  var A : domain(int);     // a domain of integers
  var B : domain(string);  // a domain of strings
  var C : domain(real);    // a domain of reals

  class X {
    var x : int;
  }

  //
  // A domain whose indices are classes of type 'X'. The indices are hashed on
  // identity.
  //
  var D : domain(X);

  record Y {
    var y : real;
  }

  //
  // A domain whose indices are records of type 'Y'. The indices are hashed on
  // value.
  //
  var E : domain(Y);
}

//
// Below, 'Names' is an associative domain of strings.  Associative domains
// start out empty (no indices).
//
var Names: domain(string);
writeln("An empty associative domain: ", Names);
writeln();

//
// We can also use the domain-literal syntax to create an associative domain.
//
// Below, 'Days' is an associative domain of strings.
//
var Days = {"Sunday", "Wednesday", "Saturday"};

//
// The '+=' operator can be used to add indices to an associative domain.
//
Names += "Alice";
Names += "Bob";
Names += "Clark";
Names += "Dana";

//
// The 'add' method is equivalent to '+='
//
// The order in which indices are printed is not guaranteed for associative
// domains.
//
Names.add("Ellen");
Names.add("Frank");
writeln("A populated associative domain of strings: ", Names);
writeln();

//
// Below we can add a range of integers with the '+=' or 'add' operators
// because of what is called 'promotion'. Those functions take integer
// arguments, which the Chapel compiler recognizes and then 'promotes' the code
// to handle iteration and invokation of the function.
//
// This promotion will work with anything that can be iterated over in a
// for loop.
//
var DaysInJune : domain(int);
DaysInJune += 1..30;

//
// The '-=' operator is used to remove indices from an associative domain.
// The 'remove' method performs the same function.
//
Names -= "Bob";
Names += "Robert";
Names.remove("Frank");
writeln("Removed 'Bob' and 'Frank', added 'Robert': ", Names);
writeln();

//
// If we want to check that "Bob" and "Frank" were indeed removed we can use the
// 'member' function, which returns a boolean.
//
if Names.member("Bob") then
  halt("Error: Bob should have been removed!");
if Names.member("Frank") then
  halt("Error: Frank should have been removed!");

//
// Like all other domains, we can use associative domains to define an array.
// 'Scores' is the Chapel-equivalent of a "map", "dictionary", or "hash table"
// found in other languages. Where 'Names' provides the keys, 'Scores' provides
// the values.
//
// Scores: an associative array of strings to integers.
//
var Scores : [Names] int;

//
// We could also use the array-literal syntax to create an associative
// array.
//
// Below, 'Months' is an associative array of strings to integers.
//
var Months = ["June" => 30, "January" => 31, "September" => 30];

//
// All array elements are initialized to the default value for that type.
//
writeln("Our first associative array: ", Scores);

//
// Printing an array only prints its values. Let's write a function that
// prints our associative array in a more human-readable format.
//

//
// Given an array, print in the following format:
// [ idx => val, ... ]
//
// We'll use the 'sorted' iterator to print in a consistent order. Otherwise,
// the order in which indices are yielded is nondeterministic.
//
proc prettyPrint(arr : [?dom]) {
  write("[ ");
  var first = true;
  for k in dom.sorted() {
    if !first {
      write(", ", k, " => ", arr[k]);
    } else {
      write(k, " => ", arr[k]);
      first = false;
    }
  }
  writeln(" ]");
}

//
// Using our new routine, print our array.
//
write("Our first array, pretty-printed: ");
prettyPrint(Scores);
writeln();

//
// Let's initialize our 'Scores' array.
//
// Traditional array operations like whole-array assignment and indexing
// are supported for associative arrays.
//
Scores = 100;
Scores["Robert"] = 85;
Scores["Alice"] = 75;
Scores["Dana"] = 91;
write("Our 'Scores' array initialized with some values: ");
prettyPrint(Scores);
writeln();

//
// Like other Chapel arrays, we can use reductions.
//
writeln("The average score is ", (+ reduce Scores) / Scores.size);
writeln();

//
// Because Chapel arrays are updated when their domain is changed, we can
// add indices after the array is declared. When we add an index to the
// defining domain, the value in the array is initialized to the default
// value for that type.
//
Names += "John";
if Scores["John"] != 0 then
  halt("Scores[\"John\"] should have been initialized to 0.");

//
// Once added to the 'Names' domain, we can use it like any other index in
// the array.
//
Scores["John"] = 76;

//
// We can also implicitly add indices to an associative array if it is the
// only array defined over its domain.
//
// Here, the string "Gary" is added as an index, and the corresponding
// array element is initialized to 10.
//
// The first line of code below is equivalent to the following:
//   Names += "Gary";
//   Scores["Gary"] = 10;
//
Scores["Gary"] = 10;
write("Our updated 'Scores' array after adding some more names: ");
prettyPrint(Scores);
writeln();

//
// We can also verify that the underlying 'Names' domain was modified:
//
if !Names.member("Gary") then
  halt("Error: 'Gary' should be present in the 'Names' domain");

//
// If we were to define another array with 'Names' as the backing domain,
// then we could no longer use this convenient syntax.
//
var Ages : [Names] int;

//
// Here, 'Ages and Scores' are backed by the same associative domain: 'Names'.
// We can no longer write something like
//   Ages["Mark"] = 33;
// or
//   Score["Sam"] = 42;
// if "Mark" or "Sam" are not valid indices in the 'Names' domain.
//

//
// This restriction exists because Chapel binds arrays to their domains for
// their lifetime to support semantic reasoning by the compiler and users.
// Extending the array via indexing will modify its domain, which would in
// turn modify the keys and values of any other arrays sharing that domain.
//
// We consider this behavior to be too surprising to support ("I modified
// 'Ages', why did 'Scores' change as well?"). Thus, to extend an array which
// shares its domain, the user is required to modify the domain directly.
// By modifying such a domain, it should be less surprising that arrays
// bound to that domain are affected.
//
Names.add("Mark");
Ages["Mark"] = 33;
Scores["Mark"] = 81;

//
// The program will still halt if one tries to access an index
// not in the domain:
//   writeln(Scores["Sally"]);
//

//
// Set operations are available only on associative domains and arrays.
//
// The supported set operations are:
//   Union (| or +)
//   Intersection (&)
//   Difference (-)
//   Symmetric Difference (^)
//
// For other array and domain types, these operators are promoted unless
// otherwise stated.
//

//
// TODO: Better examples that aren't duplicating the abilities of ranges.
//
// We'll use domains of integers for convenience, but
// these set operations work for any domain type.
//

config const n = 20;

var odds, evens, lowerHalf : domain(int);
odds += 1..n by 2;
evens += 1..n by 2 align 2;
lowerHalf += 1..(n/2);

//
// Note: When printing the result of an iterator in this way, the results are
// coalesced into a Default Rectangular array of the indices.
//
writeln("Integers from 1..n composed of odds | evens: ", (odds | evens).sorted());
writeln("Odd integers below n/2: odds & lowerHalf = ", (odds & lowerHalf).sorted());
writeln("Another way to get the same result: lowerHalf - evens = ", (lowerHalf - evens).sorted());
writeln("Even below n/2, odd above: lowerHalf ^ odds = ", (lowerHalf ^ odds).sorted());
writeln();

//
// Let's use the array-literal syntax to create a couple of arrays to use
// as examples for set operations.
//
var A = ["a" => true, "b" => false, "c" => true];
var B = ["b" => true, "c" => false, "d" => true];
writeln("Some simple associative arrays:");
write("A = ");
prettyPrint(A);
write("B = ");
prettyPrint(B);

//
// When performing a union between two arrays with overlapping indices, the
// values of the second array take precedence.
//
var C = A | B;
write("C is the union of A and B; C = A | B : ");
prettyPrint(C);

//
// The op= variants are supported for associative arrays:
//
// Union: +=, |=
// Intersection: &=
// Difference: -=
// Symmetric Difference: ^=
//
// However, this is only permitted when the array's domain is not shared with
// other arrays.
//
A |= B;
write("A becomes the union of A and B; A |= B: ");
prettyPrint(A);

/*

  Associative Primer

  This is a tutorial on Chapel's associative domains and arrays.

  It may be beneficial to read the 'arrays.chpl' and 'domains.chpl'
  primers before proceeding if you're not already familiar with Chapel's
  domains and arrays.

*/

//
// Associative domains are commonly used to represent arbitrary sets, or to
// represent the set of keys for a dictionary/hash table.
//
// The current implementation supports most Chapel types, and will eventually
// support any language-defined type.
{
  var A : domain(int);     // a domain of integers
  var B : domain(string);  // a domain of strings
  var C : domain(real);    // a domain of reals

  class X {
    var x : int;
  }

  //
  // A domain whose indices are classes of type 'X'. The indices are hashed on
  // identity.
  //
  var D : domain(X);

  record Y {
    var y : real;
  }

  //
  // A domain whose indices are records of type 'Y'. The indices are hashed on
  // value.
  //
  var E : domain(Y);
}

//
// Below, 'Names' is an associative domain of strings.  Associative domains
// start out empty (no indices).
//
var Names: domain(string);
writeln("An empty associative domain: ", Names);
writeln();

//
// We can also use the domain-literal syntax to create an associative domain.
//
// Below, 'Days' is an associative domain of strings.
//
var Days = {"Sunday", "Wednesday", "Saturday"};

//
// The '+=' operator can be used to add indices to an associative domain.
//
Names += "Alice";
Names += "Bob";
Names += "Clark";
Names += "Dana";

//
// The 'add' method is equivalent to '+='
//
// The order in which indices are printed is not guaranteed for associative
// domains.
//
Names.add("Ellen");
Names.add("Frank");
writeln("A populated associative domain of strings: ", Names);
writeln();

//
// Below we can add a range of integers with the '+=' or 'add' operators
// because of what is called 'promotion'. Those functions take integer
// arguments, which the Chapel compiler recognizes and then 'promotes' the code
// to handle iteration and invokation of the function.
//
// This promotion will work with anything that can be iterated over in a
// for loop.
//
var DaysInJune : domain(int);
DaysInJune += 1..30;

//
// The '-=' operator is used to remove indices from an associative domain.
// The 'remove' method performs the same function.
//
Names -= "Bob";
Names += "Robert";
Names.remove("Frank");
writeln("Removed 'Bob' and 'Frank', added 'Robert': ", Names);
writeln();

//
// If we want to check that "Bob" and "Frank" were indeed removed we can use the
// 'member' function, which returns a boolean.
//
if Names.member("Bob") then
  halt("Error: Bob should have been removed!");
if Names.member("Frank") then
  halt("Error: Frank should have been removed!");

//
// Like all other domains, we can use associative domains to define an array.
// 'Scores' is the Chapel-equivalent of a "map", "dictionary", or "hash table"
// found in other languages. Where 'Names' provides the keys, 'Scores' provides
// the values.
//
// Scores: an associative array of strings to integers.
//
var Scores : [Names] int;

//
// We could also use the array-literal syntax to create an associative
// array.
//
// Below, 'Months' is an associative array of strings to integers.
//
var Months = ["June" => 30, "January" => 31, "September" => 30];

//
// All array elements are initialized to the default value for that type.
//
writeln("Our first associative array: ", Scores);

//
// Printing an array only prints its values. Let's write a function that
// prints our associative array in a more human-readable format.
//

//
// Given an array, print in the following format:
// { idx => val, ... }
//
// We'll use the 'sorted' iterator to print in a consistent order. Otherwise,
// the order in which indices are yielded is nondeterministic.
//
proc prettyPrint(arr : [?dom]) {
  write("{ ");
  var first = true;
  for k in dom.sorted() {
    if !first {
      write(", ", k, " => ", arr[k]);
    } else {
      write(k, " => ", arr[k]);
      first = false;
    }
  }
  writeln(" }");
}

//
// Using our new routine, print our array.
//
write("Our first array, pretty-printed: ");
prettyPrint(Scores);
writeln();

//
// Let's initialize our 'Scores' array.
//
// Traditional array operations like whole-array assignment and indexing
// are supported for associative arrays.
//
Scores = 100;
Scores["Robert"] = 85;
Scores["Alice"] = 75;
Scores["Dana"] = 91;
write("Our 'Scores' array initialized with some values: ");
prettyPrint(Scores);
writeln();

//
// Like other Chapel arrays, we can use reductions.
//
writeln("The average score is ", (+ reduce Scores) / Scores.size);
writeln();

//
// Because Chapel arrays are updated when their domain is changed, we can
// add indices after the array is declared. When we add an index to the
// defining domain, the value in the array is initialized to the default
// value for that type.
//
Names += "John";
if Scores["John"] != 0 then
  halt("Scores[\"John\"] should have been initialized to 0.");

//
// Once added to the 'Names' domain, we can use it like any other index in
// the array.
//
Scores["John"] = 76;

//
// We can also implicitly add indices to an associative array if it is the
// only array defined over its domain.
//
// Here, the string "Gary" is added as an index, and the corresponding
// array element is initialized to 10.
//
// The first line of code below is equivalent to the following:
//   Names += "Gary";
//   Scores["Gary"] = 10;
//
Scores["Gary"] = 10;
write("Our updated 'Scores' array after adding some more names: ");
prettyPrint(Scores);
writeln();

//
// We can also verify that the underlying 'Names' domain was modified:
//
if !Names.member("Gary") then
  halt("Error: 'Gary' should be present in the 'Names' domain");

//
// If we were to define another array with 'Names' as the backing domain,
// then we could no longer use this convenient syntax.
//
var Ages : [Names] int;

//
// Here, 'Ages and Scores' are backed by the same associative domain: 'Names'.
// We can no longer write something like
//   Ages["Mark"] = 33;
// or
//   Score["Sam"] = 42;
// if "Mark" or "Sam" are not valid indices in the 'Names' domain.
//

//
// This restriction exists because Chapel binds arrays to their domains for
// their lifetime to support semantic reasoning by the compiler and users.
// Extending the array via indexing will modify its domain, which would in
// turn modify the keys and values of any other arrays sharing that domain.
//
// We consider this behavior to be too surprising to support ("I modified
// 'Ages', why did 'Scores' change as well?"). Thus, to extend an array which
// shares its domain, the user is required to modify the domain directly.
// By modifying such a domain, it should be less surprising that arrays
// bound to that domain are affected.
//
Names.add("Mark");
Ages["Mark"] = 33;
Scores["Mark"] = 81;

//
// The program will still halt if one tries to access an index
// not in the domain:
//   writeln(Scores["Sally"]);
//

//
// Set operations are available only on associative domains and arrays.
//
// The supported set operations are:
//   Union (| or +)
//   Intersection (&)
//   Difference (-)
//   Symmetric Difference (^)
//
// For other array and domain types, these operators are promoted unless
// otherwise stated.
//

//
// TODO: Better examples that aren't duplicating the abilities of ranges.
//
// We'll use domains of integers for convenience, but
// these set operations work for any domain type.
//

config const n = 20;

var odds, evens, lowerHalf : domain(int);
odds += 1..n by 2;
evens += 1..n by 2 align 2;
lowerHalf += 1..(n/2);

//
// Note: When printing the result of an iterator in this way, the results are
// coalesced into a Default Rectangular array of the indices.
//
writeln("Integers from 1..n composed of odds | evens: ", (odds | evens).sorted());
writeln("Odd integers below n/2: odds & lowerHalf = ", (odds & lowerHalf).sorted());
writeln("Another way to get the same result: lowerHalf - evens = ", (lowerHalf - evens).sorted());
writeln("Even below n/2, odd above: lowerHalf ^ odds = ", (lowerHalf ^ odds).sorted());
writeln();

//
// Let's use the array-literal syntax to create a couple of arrays to use
// as examples for set operations.
//
var A = ["a" => true, "b" => false, "c" => true];
var B = ["b" => true, "c" => false, "d" => true];
writeln("Some simple associative arrays:");
write("A = ");
prettyPrint(A);
write("B = ");
prettyPrint(B);

//
// When performing a union between two arrays with overlapping indices, the
// values of the second array take precedence.
//
var C = A | B;
write("C is the union of A and B; C = A | B : ");
prettyPrint(C);

//
// The op= variants are supported for associative arrays:
//
// Union: +=, |=
// Intersection: &=
// Difference: -=
// Symmetric Difference: ^=
//
// However, this is only permitted when the array's domain is not shared with
// other arrays.
//
A |= B;
write("A becomes the union of A and B; A |= B: ");
prettyPrint(A);


Leak the origRootLocale.

Leak the origRootLocale.

At least the block distribution attempts to access the origRootLocale after it has been
deleted.  This is due to a mismatch between the module initialization order and the
assumption that shared objects will remain live until after their last use.

Locale objects are class objects, so their MM is under program (as opposed to compiler)
control.  The block distribution (for example) stores an array of locales (targetLocales),
but the locales themselves are stored elsewhere.  When running on a single locale, the
origRootLocale is never replaced.

It would seem that the distribution ProblemDist created in the user code should be deleted
before the origRootLocale.  This might go back to a problem in how the autoDestroyGlobals
is populated.

Anyway, the workaround (of not deleting origRootLocale) allows this and two other test
cases in the same directory to pass.

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// ChapelLocale.chpl
//
module ChapelLocale {

  use LocaleModel;
  use AutoPtr;

  //
  // Node and sublocale types and special sublocale values.
  //
  type chpl_nodeID_t = int(32);
  type chpl_sublocID_t = int(32);

  extern const c_sublocid_none: chpl_sublocID_t;
  extern const c_sublocid_any: chpl_sublocID_t;

  //
  // An abstract class. Specifies the required locale interface.
  // Each locale implementation must inherit from this class.
  //
  class locale {
    //- Constructor
    proc locale() { }

    //------------------------------------------------------------------------{
    //- Fields and accessors defined for all locale types (not overridable)
    //-

    // Every locale has a parent, except for the root locale.
    // The parent of the root locale is nil (by definition).
    const parent : locale;

    // To be removed from the required interface once legacy code is adjusted.
    // Modified in RootLocale.init().
    var numCores: int;

    var maxTaskPar: int; // max parallelism tasking layer expects to deliver

    proc id : int return chpl_id();  // just the node part
    proc localeid : chpl_localeID_t return chpl_localeid(); // full locale id
    proc name return chpl_name() : c_string;

    // This many tasks are running on this locale.
    //
    // Note handling runningTaskCounter <= 0 in runningTaskCnt().  The
    // references to it are done via here.runningTasks(), and currently
    // "here" means the requested locale, not the actual or concrete
    // one.  But this may be "any", and the running task count in it may
    // be zero, which in turn leads to overindexing the locales array
    // after the subtraction of 1 in locale.runningTasks(), below.  In
    // the long run we probably need to go through all the "here" refs
    // in the module code and differentiate more carefully which need
    // the requested locale and which need the concrete one.  But for
    // now, we make the assumption that all requests for the number of
    // running tasks want the count from the locale the calling task is
    // running on, so the minimum possible value must be 1.
    var runningTaskCounter : atomic int;

    inline proc runningTaskCntSet(val : int) {
      runningTaskCounter.write(val, memory_order_relaxed);
    }

    inline proc runningTaskCntAdd(val : int) {
      runningTaskCounter.add(val, memory_order_relaxed);
    }

    inline proc runningTaskCntSub(val : int) {
      runningTaskCounter.sub(val, memory_order_relaxed);
    }

    inline proc runningTaskCnt() {
      var rtc = runningTaskCounter.read(memory_order_relaxed);
      return if (rtc <= 0) then 1 else rtc;
    }
    //------------------------------------------------------------------------}

    //------------------------------------------------------------------------{
    //- User Interface Methods (overridable)
    //-

    // These are dynamically dispatched, so they can be overridden in
    // concrete classes.
    proc chpl_id() : int {
      _throwPVFCError();
      return -1;
    }

    proc chpl_localeid() : chpl_localeID_t {
      _throwPVFCError();
      return chpl_buildLocaleID(-1:chpl_nodeID_t, c_sublocid_none);
    }

    proc chpl_name() : c_string {
      _throwPVFCError();
      return "";
    }

    // A useful default definition is provided (not pure virtual).
    proc readWriteThis(f) {
      f <~> name;
    }

    proc getChildCount() : int {
      _throwPVFCError();
      return 0;
    }

// Part of the required locale interface.
// Commented out because presently iterators are statically bound.
//    iter getChildIndices() : int {
//      for idx in this.getChildSpace do
//        yield idx;
//    }

    proc addChild(loc:locale)
    {
      _throwPVFCError();
    }

    proc getChild(idx:int) : locale {
      _throwPVFCError();
      return this;
    }

// Part of the required locale interface.
// Commented out because presently iterators are statically bound.
//    iter getChildren() : locale  {
//    _throwPVFCError();
//      yield 0;
//    }

    //------------------------------------------------------------------------}
  }

  class AbstractLocaleModel : locale {
    // This will be used for interfaces that will be common to all
    // (non-RootLocale) locale models
  }

  // rootLocale is declared to be of type locale rather than
  // RootLocale because doing so would make locales depend on arrays
  // which depend on locales, etc.
  //
  // If we had a way to express domain and array return types abstractly,
  // that problem would go away. <hilde>
  //
  // The rootLocale is private to each locale.  It cannot be
  // initialized until LocaleModel is initialized.  To disable this
  // replication, set replicateRootLocale to false.
  pragma "private" var rootLocale : locale = nil;
  config param replicateRootLocale = true;

  // The rootLocale needs to be initalized on all locales prior to
  // initializing the Locales array.  Unfortunately, the rootLocale
  // cannot be properly replicated until DefaultRectangular can be
  // initialized (as the private copies of the defaultDist are
  // needed).  We remedy this by initializing a rootLocale on locale 0
  // (called origRootLocale), and setting all locales' rootLocale to
  // origRootLocale.  Later, after DefaultRectangular can be
  // initialized, we create local copies of the rootLocale (and the
  // Locales array).  This is being done in the InitPrivateGlobals
  // module.
  //
  var origRootLocale : locale = nil;
// origRootLocale is shared too much, and used after this module is destroyed,
// so releasing its memory with an autopointer turns out to be premature.
// Until someone has the time to analyze the lifetime of origRootLocale, we are
// obliged to leak it. :(
//  var origRootLocaleWrapper = new auto_ptr(origRootLocale);

  class AbstractRootLocale : locale {
    // These functions are used to establish values for Locales[] and
    // LocaleSpace -- an array of locales and its correponding domain
    // which are used as the default set of targetLocales in many
    // distributions.
    proc getDefaultLocaleSpace() {
      _throwPVFCError();
      const emptyLocaleSpace: domain(1) = {1..0};
      return emptyLocaleSpace;
    }

    proc getDefaultLocaleArray() {
      _throwPVFCError();
      const emptyLocaleSpace: domain(1) = {1..0};
      const emptyLocales: [emptyLocaleSpace] locale;
      return emptyLocales;
    }

    proc localeIDtoLocale(id : chpl_localeID_t) : locale {
      _throwPVFCError();
      return this;
    }

    // These iterators are to be used by RootLocale:init() to
    // initialize the LocaleModel.  The calling loop body cannot
    // contain any non-local code, since the rootLocale is not yet
    // initialized.
    iter chpl_initOnLocales() {
      if numLocales > 1 then
        halt("The locales must be initialized in parallel");
      for locIdx in (origRootLocale:RootLocale).getDefaultLocaleSpace() {
        yield locIdx;
        rootLocale = origRootLocale;
      }
    }

    // Since we are going on to all the locales here, we use the
    // opportunity to initialize any global private variables we
    // either need (e.g., defaultDist) or can do at this point in
    // initialization (e.g., rootLocale).
    iter chpl_initOnLocales(param tag: iterKind)
      where tag==iterKind.standalone {
      // Simple locales barrier, see implementation below for notes
      var b: localesBarrier;
      var flags: [1..#numLocales-1] localesSignal;
      coforall locIdx in 0..#numLocales /*ref(b)*/ {
        on __primitive("chpl_on_locale_num",
                       chpl_buildLocaleID(locIdx:chpl_nodeID_t,
                                          c_sublocid_any)) {
          chpl_defaultDistInitPrivate();
          yield locIdx;
          b.wait(locIdx, flags);
          chpl_rootLocaleInitPrivate(locIdx);
        }
      }
    }
  }

  //
  // Simple locales barrier
  //
  // This sits outside of the abstract root locale definition above
  // because the compiler cannot resolve the constructor (known issues
  // with nested classes/records).  In addition, we cannot have the
  // flags array in the record, because the initCopy function needs
  // the dataPar* configs declared in ChapelDistribution.
  //
  // Each non-0 locale increments the count, and then waits on a
  // *local* atomic.  This is done by creating an array of type
  // class localesSignal, one per locale, and allocating each locale's
  // copy before updating the count.  Locale 0 waits for the others to
  // arrive and then set the signals to true.  We can't do anything
  // too complicated this early on, so we are using a for loop to
  // broadcast that we are done.
  class localesSignal {
    var s: atomic bool;
  }
  record localesBarrier {
    proc wait(locIdx, flags) {
      if locIdx==0 {
        // locale 0 has nothing else to do, so check flags
        while (true) {
          // This fence ensures that writes to the count variables
          // are available to this task. (Note that they aren't
          // atomic if they're 128-bit writes though - so we
          // have some risk of getting part of a wide pointer).
          // Without this fence, there is a race condition on
          // a weakly-ordered architecture.
          atomic_fence();
          var count = 0;
          for f in flags do
            if f then count += 1;
          if count==numLocales-1 then break;
          // Give time to other tasks/threads/processes
          // like we do in waitFor
          chpl_task_yield();
        }
        // Let the others go
        for f in flags do
          f.s.testAndSet();
      } else {
        var f = new localesSignal();
        // expose my flag to locale 0
        flags[locIdx] = f;
        // wait (locally) for locale 0 to set my flag
        f.s.waitFor(true);
        // clean up
        delete f;
      }
    }
  }

  // This function is called in the LocaleArray module to initialize
  // the rootLocale.  It sets up the origRootLocale and also includes
  // set up of the each locale's LocaleModel via RootLocale:init().
  //
  // The init() function must use the chpl_initOnLocales() iterator above
  // to iterate in parallel over the locales to set up the LocaleModel
  // object.
  proc chpl_init_rootLocale() {
    origRootLocale = new RootLocale();
    (origRootLocale:RootLocale).init();
    //    origRootLocaleWrapper.ptr = origRootLocale;
  }

  // This function sets up a private copy of rootLocale by replicating
  // origRootLocale and resets the Locales array to point to the local
  // copy on all but locale 0 (which is done in LocalesArray.chpl as
  // part of the declaration).
  proc chpl_rootLocaleInitPrivate(locIdx) {
    // Even when not replicating the rootLocale, we must temporarily
    // set the rootLocale to the original version on locale 0, because
    // the initialization below needs to get/set locale ids.
    rootLocale = origRootLocale;
    if replicateRootLocale && locIdx!=0 {
      // Create a new local rootLocale
      var newRootLocale = new RootLocale();
      // We don't want to be doing unnecessary ref count updates here
      // as they require additional tasks.  We know we don't need them
      // so tell the compiler to not insert them.
      pragma "no copy" pragma "no auto destroy"
      const origLocales => (origRootLocale:RootLocale).getDefaultLocaleArray();
      var origRL = origLocales._value.theData;
      var newRL = newRootLocale.getDefaultLocaleArray()._value.theData;
      // We must directly implement a bulk copy here, as the mechanisms
      // for doing so via a whole array assignment are not initialized
      // yet and copying element-by-element via a for loop is is costly.
      __primitive("chpl_comm_get",
                  __primitive("array_get", newRL, 0),
                  0 /* locale 0 */,
                  __primitive("array_get", origRL, 0), numLocales);
      // Set the rootLocale to the local copy
      rootLocale = newRootLocale;
    }
    if locIdx!=0 {
      // We mimic a private Locales array alias by using the move
      // primitive.
      __primitive("move", Locales,
                  (rootLocale:RootLocale).getDefaultLocaleArray());
    }
  }

  // We need a temporary value for "here" before the architecture is defined.
  // This is due to the fact that "here" is used for memory and task control
  // in setting up the architecture itself.
  // Its type should probably be renamed dummyLocale or something
  // representative.
  // The dummy locale provides system-default tasking and memory management.
  const dummyLocale = new locale();

  extern proc chpl_task_getRequestedSubloc(): chpl_sublocID_t;

  pragma "insert line file info"
  export
  proc chpl_getLocaleID(ref localeID: chpl_localeID_t) {
    localeID = here_id;
  }

  // Return the locale ID of the current locale
  inline proc here_id {
     if localeModelHasSublocales then
      return chpl_rt_buildLocaleID(chpl_nodeID, chpl_task_getRequestedSubloc());
    else
      return chpl_rt_buildLocaleID(chpl_nodeID, c_sublocid_any);
  }
  // Return the current locale
  inline proc here {
    return chpl_localeID_to_locale(here_id);
  }

  // Returns a wide pointer to the locale with the given id.
  proc chpl_localeID_to_locale(id : chpl_localeID_t) : locale {
    if rootLocale then
      return (rootLocale:AbstractRootLocale).localeIDtoLocale(id);
    else
      // For code prior to rootLocale initialization
      return dummyLocale;
  }

  pragma "insert line file info"
  extern proc chpl_memhook_malloc_pre(number:int, size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_malloc_post(ptr:opaque, number:int,
                                       size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_realloc_pre(ptr:object, size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_realloc_post(newPtr:opaque, ptr:object,
                                        size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_free_pre(ptr:opaque): void;
  extern proc chpl_memhook_md_num(): int(16);

  proc chpl_getPrivatizedCopy(type objectType, objectPid:int): objectType
    return __primitive("chpl_getPrivatizedClass", nil:objectType, objectPid);

//########################################################################{
//# Locale diagnostics
//#

  proc locale.totalThreads() {
    var totalThreads: int;
    extern proc chpl_task_getNumThreads() : uint(32);
    on this do totalThreads = chpl_task_getNumThreads();
    return totalThreads;
  }

  proc locale.idleThreads() {
    var idleThreads: int;
    extern proc chpl_task_getNumIdleThreads() : uint(32);
    on this do idleThreads = chpl_task_getNumIdleThreads();
    return idleThreads;
  }

  proc locale.queuedTasks() {
    var queuedTasks: int;
    extern proc chpl_task_getNumQueuedTasks() : uint(32);
    on this do queuedTasks = chpl_task_getNumQueuedTasks();
    return queuedTasks;
  }

  proc locale.runningTasks() {
    return this.runningTaskCnt();
  }

  proc locale.blockedTasks() {
    var blockedTasks: int;
    extern proc chpl_task_getNumBlockedTasks() : int(32);
    on this do blockedTasks = chpl_task_getNumBlockedTasks();
    return blockedTasks;
  }

//########################################################################}

  //
  // Increment and decrement the task count.
  //
  // Elsewhere in the module code we adjust the running task count
  // directly, but at least for now the runtime also needs to be
  // able to do so.  These functions support that.
  //
  pragma "insert line file info"
  export
  proc chpl_taskRunningCntInc() {
    here.runningTaskCntAdd(1);
  }

  pragma "insert line file info"
  export
  proc chpl_taskRunningCntDec() {
    here.runningTaskCntSub(1);
  }
}

/*
 * Copyright 2004-2015 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// ChapelLocale.chpl
//
module ChapelLocale {

  use LocaleModel;
  use AutoPtr;

  //
  // Node and sublocale types and special sublocale values.
  //
  type chpl_nodeID_t = int(32);
  type chpl_sublocID_t = int(32);

  extern const c_sublocid_none: chpl_sublocID_t;
  extern const c_sublocid_any: chpl_sublocID_t;

  //
  // An abstract class. Specifies the required locale interface.
  // Each locale implementation must inherit from this class.
  //
  class locale {
    //- Constructor
    proc locale() { }

    //------------------------------------------------------------------------{
    //- Fields and accessors defined for all locale types (not overridable)
    //-

    // Every locale has a parent, except for the root locale.
    // The parent of the root locale is nil (by definition).
    const parent : locale;

    // To be removed from the required interface once legacy code is adjusted.
    // Modified in RootLocale.init().
    var numCores: int;

    var maxTaskPar: int; // max parallelism tasking layer expects to deliver

    proc id : int return chpl_id();  // just the node part
    proc localeid : chpl_localeID_t return chpl_localeid(); // full locale id
    proc name return chpl_name() : c_string;

    // This many tasks are running on this locale.
    //
    // Note handling runningTaskCounter <= 0 in runningTaskCnt().  The
    // references to it are done via here.runningTasks(), and currently
    // "here" means the requested locale, not the actual or concrete
    // one.  But this may be "any", and the running task count in it may
    // be zero, which in turn leads to overindexing the locales array
    // after the subtraction of 1 in locale.runningTasks(), below.  In
    // the long run we probably need to go through all the "here" refs
    // in the module code and differentiate more carefully which need
    // the requested locale and which need the concrete one.  But for
    // now, we make the assumption that all requests for the number of
    // running tasks want the count from the locale the calling task is
    // running on, so the minimum possible value must be 1.
    var runningTaskCounter : atomic int;

    inline proc runningTaskCntSet(val : int) {
      runningTaskCounter.write(val, memory_order_relaxed);
    }

    inline proc runningTaskCntAdd(val : int) {
      runningTaskCounter.add(val, memory_order_relaxed);
    }

    inline proc runningTaskCntSub(val : int) {
      runningTaskCounter.sub(val, memory_order_relaxed);
    }

    inline proc runningTaskCnt() {
      var rtc = runningTaskCounter.read(memory_order_relaxed);
      return if (rtc <= 0) then 1 else rtc;
    }
    //------------------------------------------------------------------------}

    //------------------------------------------------------------------------{
    //- User Interface Methods (overridable)
    //-

    // These are dynamically dispatched, so they can be overridden in
    // concrete classes.
    proc chpl_id() : int {
      _throwPVFCError();
      return -1;
    }

    proc chpl_localeid() : chpl_localeID_t {
      _throwPVFCError();
      return chpl_buildLocaleID(-1:chpl_nodeID_t, c_sublocid_none);
    }

    proc chpl_name() : c_string {
      _throwPVFCError();
      return "";
    }

    // A useful default definition is provided (not pure virtual).
    proc readWriteThis(f) {
      f <~> name;
    }

    proc getChildCount() : int {
      _throwPVFCError();
      return 0;
    }

// Part of the required locale interface.
// Commented out because presently iterators are statically bound.
//    iter getChildIndices() : int {
//      for idx in this.getChildSpace do
//        yield idx;
//    }

    proc addChild(loc:locale)
    {
      _throwPVFCError();
    }

    proc getChild(idx:int) : locale {
      _throwPVFCError();
      return this;
    }

// Part of the required locale interface.
// Commented out because presently iterators are statically bound.
//    iter getChildren() : locale  {
//    _throwPVFCError();
//      yield 0;
//    }

    //------------------------------------------------------------------------}
  }

  class AbstractLocaleModel : locale {
    // This will be used for interfaces that will be common to all
    // (non-RootLocale) locale models
  }

  // rootLocale is declared to be of type locale rather than
  // RootLocale because doing so would make locales depend on arrays
  // which depend on locales, etc.
  //
  // If we had a way to express domain and array return types abstractly,
  // that problem would go away. <hilde>
  //
  // The rootLocale is private to each locale.  It cannot be
  // initialized until LocaleModel is initialized.  To disable this
  // replication, set replicateRootLocale to false.
  pragma "private" var rootLocale : locale = nil;
  config param replicateRootLocale = true;

  // The rootLocale needs to be initalized on all locales prior to
  // initializing the Locales array.  Unfortunately, the rootLocale
  // cannot be properly replicated until DefaultRectangular can be
  // initialized (as the private copies of the defaultDist are
  // needed).  We remedy this by initializing a rootLocale on locale 0
  // (called origRootLocale), and setting all locales' rootLocale to
  // origRootLocale.  Later, after DefaultRectangular can be
  // initialized, we create local copies of the rootLocale (and the
  // Locales array).  This is being done in the InitPrivateGlobals
  // module.
  //
  var origRootLocale : locale = nil;
  var origRootLocaleWrapper = new auto_ptr(origRootLocale);

  class AbstractRootLocale : locale {
    // These functions are used to establish values for Locales[] and
    // LocaleSpace -- an array of locales and its correponding domain
    // which are used as the default set of targetLocales in many
    // distributions.
    proc getDefaultLocaleSpace() {
      _throwPVFCError();
      const emptyLocaleSpace: domain(1) = {1..0};
      return emptyLocaleSpace;
    }

    proc getDefaultLocaleArray() {
      _throwPVFCError();
      const emptyLocaleSpace: domain(1) = {1..0};
      const emptyLocales: [emptyLocaleSpace] locale;
      return emptyLocales;
    }

    proc localeIDtoLocale(id : chpl_localeID_t) : locale {
      _throwPVFCError();
      return this;
    }

    // These iterators are to be used by RootLocale:init() to
    // initialize the LocaleModel.  The calling loop body cannot
    // contain any non-local code, since the rootLocale is not yet
    // initialized.
    iter chpl_initOnLocales() {
      if numLocales > 1 then
        halt("The locales must be initialized in parallel");
      for locIdx in (origRootLocale:RootLocale).getDefaultLocaleSpace() {
        yield locIdx;
        rootLocale = origRootLocale;
      }
    }

    // Since we are going on to all the locales here, we use the
    // opportunity to initialize any global private variables we
    // either need (e.g., defaultDist) or can do at this point in
    // initialization (e.g., rootLocale).
    iter chpl_initOnLocales(param tag: iterKind)
      where tag==iterKind.standalone {
      // Simple locales barrier, see implementation below for notes
      var b: localesBarrier;
      var flags: [1..#numLocales-1] localesSignal;
      coforall locIdx in 0..#numLocales /*ref(b)*/ {
        on __primitive("chpl_on_locale_num",
                       chpl_buildLocaleID(locIdx:chpl_nodeID_t,
                                          c_sublocid_any)) {
          chpl_defaultDistInitPrivate();
          yield locIdx;
          b.wait(locIdx, flags);
          chpl_rootLocaleInitPrivate(locIdx);
        }
      }
    }
  }

  //
  // Simple locales barrier
  //
  // This sits outside of the abstract root locale definition above
  // because the compiler cannot resolve the constructor (known issues
  // with nested classes/records).  In addition, we cannot have the
  // flags array in the record, because the initCopy function needs
  // the dataPar* configs declared in ChapelDistribution.
  //
  // Each non-0 locale increments the count, and then waits on a
  // *local* atomic.  This is done by creating an array of type
  // class localesSignal, one per locale, and allocating each locale's
  // copy before updating the count.  Locale 0 waits for the others to
  // arrive and then set the signals to true.  We can't do anything
  // too complicated this early on, so we are using a for loop to
  // broadcast that we are done.
  class localesSignal {
    var s: atomic bool;
  }
  record localesBarrier {
    proc wait(locIdx, flags) {
      if locIdx==0 {
        // locale 0 has nothing else to do, so check flags
        while (true) {
          // This fence ensures that writes to the count variables
          // are available to this task. (Note that they aren't
          // atomic if they're 128-bit writes though - so we
          // have some risk of getting part of a wide pointer).
          // Without this fence, there is a race condition on
          // a weakly-ordered architecture.
          atomic_fence();
          var count = 0;
          for f in flags do
            if f then count += 1;
          if count==numLocales-1 then break;
          // Give time to other tasks/threads/processes
          // like we do in waitFor
          chpl_task_yield();
        }
        // Let the others go
        for f in flags do
          f.s.testAndSet();
      } else {
        var f = new localesSignal();
        // expose my flag to locale 0
        flags[locIdx] = f;
        // wait (locally) for locale 0 to set my flag
        f.s.waitFor(true);
        // clean up
        delete f;
      }
    }
  }

  // This function is called in the LocaleArray module to initialize
  // the rootLocale.  It sets up the origRootLocale and also includes
  // set up of the each locale's LocaleModel via RootLocale:init().
  //
  // The init() function must use the chpl_initOnLocales() iterator above
  // to iterate in parallel over the locales to set up the LocaleModel
  // object.
  proc chpl_init_rootLocale() {
    origRootLocale = new RootLocale();
    (origRootLocale:RootLocale).init();
    origRootLocaleWrapper.ptr = origRootLocale;
  }

  // This function sets up a private copy of rootLocale by replicating
  // origRootLocale and resets the Locales array to point to the local
  // copy on all but locale 0 (which is done in LocalesArray.chpl as
  // part of the declaration).
  proc chpl_rootLocaleInitPrivate(locIdx) {
    // Even when not replicating the rootLocale, we must temporarily
    // set the rootLocale to the original version on locale 0, because
    // the initialization below needs to get/set locale ids.
    rootLocale = origRootLocale;
    if replicateRootLocale && locIdx!=0 {
      // Create a new local rootLocale
      var newRootLocale = new RootLocale();
      // We don't want to be doing unnecessary ref count updates here
      // as they require additional tasks.  We know we don't need them
      // so tell the compiler to not insert them.
      pragma "no copy" pragma "no auto destroy"
      const origLocales => (origRootLocale:RootLocale).getDefaultLocaleArray();
      var origRL = origLocales._value.theData;
      var newRL = newRootLocale.getDefaultLocaleArray()._value.theData;
      // We must directly implement a bulk copy here, as the mechanisms
      // for doing so via a whole array assignment are not initialized
      // yet and copying element-by-element via a for loop is is costly.
      __primitive("chpl_comm_get",
                  __primitive("array_get", newRL, 0),
                  0 /* locale 0 */,
                  __primitive("array_get", origRL, 0), numLocales);
      // Set the rootLocale to the local copy
      rootLocale = newRootLocale;
    }
    if locIdx!=0 {
      // We mimic a private Locales array alias by using the move
      // primitive.
      __primitive("move", Locales,
                  (rootLocale:RootLocale).getDefaultLocaleArray());
    }
  }

  // We need a temporary value for "here" before the architecture is defined.
  // This is due to the fact that "here" is used for memory and task control
  // in setting up the architecture itself.
  // Its type should probably be renamed dummyLocale or something
  // representative.
  // The dummy locale provides system-default tasking and memory management.
  const dummyLocale = new locale();

  extern proc chpl_task_getRequestedSubloc(): chpl_sublocID_t;

  pragma "insert line file info"
  export
  proc chpl_getLocaleID(ref localeID: chpl_localeID_t) {
    localeID = here_id;
  }

  // Return the locale ID of the current locale
  inline proc here_id {
     if localeModelHasSublocales then
      return chpl_rt_buildLocaleID(chpl_nodeID, chpl_task_getRequestedSubloc());
    else
      return chpl_rt_buildLocaleID(chpl_nodeID, c_sublocid_any);
  }
  // Return the current locale
  inline proc here {
    return chpl_localeID_to_locale(here_id);
  }

  // Returns a wide pointer to the locale with the given id.
  proc chpl_localeID_to_locale(id : chpl_localeID_t) : locale {
    if rootLocale then
      return (rootLocale:AbstractRootLocale).localeIDtoLocale(id);
    else
      // For code prior to rootLocale initialization
      return dummyLocale;
  }

  pragma "insert line file info"
  extern proc chpl_memhook_malloc_pre(number:int, size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_malloc_post(ptr:opaque, number:int,
                                       size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_realloc_pre(ptr:object, size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_realloc_post(newPtr:opaque, ptr:object,
                                        size:int, md:int(16)): void;
  pragma "insert line file info"
  extern proc chpl_memhook_free_pre(ptr:opaque): void;
  extern proc chpl_memhook_md_num(): int(16);

  proc chpl_getPrivatizedCopy(type objectType, objectPid:int): objectType
    return __primitive("chpl_getPrivatizedClass", nil:objectType, objectPid);

//########################################################################{
//# Locale diagnostics
//#

  proc locale.totalThreads() {
    var totalThreads: int;
    extern proc chpl_task_getNumThreads() : uint(32);
    on this do totalThreads = chpl_task_getNumThreads();
    return totalThreads;
  }

  proc locale.idleThreads() {
    var idleThreads: int;
    extern proc chpl_task_getNumIdleThreads() : uint(32);
    on this do idleThreads = chpl_task_getNumIdleThreads();
    return idleThreads;
  }

  proc locale.queuedTasks() {
    var queuedTasks: int;
    extern proc chpl_task_getNumQueuedTasks() : uint(32);
    on this do queuedTasks = chpl_task_getNumQueuedTasks();
    return queuedTasks;
  }

  proc locale.runningTasks() {
    return this.runningTaskCnt();
  }

  proc locale.blockedTasks() {
    var blockedTasks: int;
    extern proc chpl_task_getNumBlockedTasks() : int(32);
    on this do blockedTasks = chpl_task_getNumBlockedTasks();
    return blockedTasks;
  }

//########################################################################}

  //
  // Increment and decrement the task count.
  //
  // Elsewhere in the module code we adjust the running task count
  // directly, but at least for now the runtime also needs to be
  // able to do so.  These functions support that.
  //
  pragma "insert line file info"
  export
  proc chpl_taskRunningCntInc() {
    here.runningTaskCntAdd(1);
  }

  pragma "insert line file info"
  export
  proc chpl_taskRunningCntDec() {
    here.runningTaskCntSub(1);
  }
}