temporarily disable indexer

temporarily disable indexer

use Logging, Memory, LibEv, IO, Indexer, Search, Partitions;

// TODO: port to pure chapel
extern proc initialize_socket(port: c_int): c_int;

// trampolines
extern var c_accept_cb: opaque;

extern proc send(sockfd:c_int, buffer: c_string, len: size_t, flags: c_int);

config var port: c_int = 3033;

// TODO: we need to know which client context this is so that we can maintain parsing context
//       is the fd enough?
export proc handle_received_data(fd: c_int, buffer: c_string, read: size_t, buffer_size: size_t) {
  // writeln("from chpl: " + buffer);
  // accumulate string buffer
  var word = buffer;
  var trimmedWord: string = "";
  writeln("word: ", word);
  for i in 1..word.length {
  	var ch = word.substring(i);
  	if (ch != "\r" && ch != "\n") {
	  	writeln("ch: ", word.substring(i));
	  	trimmedWord += word.substring(i);
  	}
  }
//  writeln("trimmedWord: ", trimmedWord);
	if (trimmedWord == "dump") {
		dumpPartition(partitionForWord("dog"));
	} else {
		writeln("<adding>");
	  indexWord(trimmedWord, 1);
		dumpPostingTableForWord(trimmedWord);
		writeln("</adding>");
	}
  send(fd, buffer, read, 0);
}

proc initIndex() {
	writeln("This program is running on ", numLocales, " locales");
	writeln("It began running on locale #", here.id);
	writeln();

  initPartitions();
  initIndices();

// temporarily disable indexer
//  initIndexer();

 //  writeln("adding 3 entries");
	// indexWord("dog", 1);
	// indexWord("cat", 2);
	// indexWord("cat", 3);

 //  writeln("dumping partition holding word: dog");
	// dumpPartition(partitionForWord("dog"));

 //  writeln("dumping posting table for word: cat");
	// dumpPostingTableForWord("cat");

  var infile = open("words.txt", iomode.r);
  var reader = infile.reader();
  var word: string;
  var docId: DocId = 1;
  while (reader.readln(word)) {
    //enqueueIndexRequest(word, docId);
    indexWord(word, docId);
    docId = (docId + 1) % 1000 + 1; // fake different docs
  }

  //waitForIndexer();

  dumpPostingTableForWord("the");

  // TODO: build execution Tree w/ conj / disj. (operator) nodes
  // test basic boolean operators
  writeln("conjunction");
  var conj = conjunction(["the", "dog"]);
  writeln(conj);

  writeln("disjunction");
  var disj = disjunction(["the", "dog"]);
  writeln(disj);
}

proc conjunction(words: [] string): domain(DocId) {
  writeln("finding conjunction of: ", words);
  var doms: [1..words.size] domain(DocId);

  for j in 1..words.size {
    var word = words[j];
    var entry = entryForWord(word);
    if (entry != nil) {
      on entry {
        for docId in documentIdsForEntry(entry) {
          doms[j] += docId;
        }
      }
    }
  }

  writeln("applying intersection");

  for j in 2..words.size {
    doms[1] = doms[1] & doms[j];
  }

  return doms[1];
}

proc disjunction(words: [] string): domain(DocId) {
  writeln("finding disjunction of: ", words);
  var doms: [1..words.size] domain(DocId);

  for j in 1..words.size {
    var word = words[j];
    var entry = entryForWord(word);
    if (entry != nil) {
      on entry {
        for docId in documentIdsForEntry(entry) {
          doms[j] += docId;
        }
      }
    }
  }

  writeln("applying disjunction");

  for j in 2..words.size {
    doms[1] = doms[1] | doms[j];
  }

  return doms[1];
}

proc writeLocInfo(loc: locale) {
  on loc {
    writeln("locale #", here.id, "...");
    writeln("  ...is named: ", here.name);
    writeln("  ...has ", here.numCores, " processor cores");
    writeln("  ...has ", here.physicalMemory(unit=MemUnits.GB, retType=real), " GB of memory");
    writeln("  ...has ", here.maxTaskPar, " maximum parallelism");
  }
}

proc main() {

	// writeln("creating socket...");
	// var sd: ev_fd = initialize_socket(port);
	// writeln("socket id = ", sd);
	// if (sd == -1) {
	// 	writeln("socket error");
	// 	return -1;
	// }

  writeln("initializing index");
  initIndex();

	// writeln("initializing event loop...");

 //  // port c_accept_cb, c_read_cb
	// var w_accept: ev_io = new ev_io();
	// ev_io_init(w_accept, c_accept_cb, sd, EV_READ);
	// ev_io_start(EV_DEFAULT, w_accept);

	// while (1) {
	// 	ev_loop_fn(EV_DEFAULT, 0);
	// }
}

use Logging, Memory, LibEv, IO, Indexer, Search, Partitions;

// TODO: port to pure chapel
extern proc initialize_socket(port: c_int): c_int;

// trampolines
extern var c_accept_cb: opaque;

extern proc send(sockfd:c_int, buffer: c_string, len: size_t, flags: c_int);

config var port: c_int = 3033;

// TODO: we need to know which client context this is so that we can maintain parsing context
//       is the fd enough?
export proc handle_received_data(fd: c_int, buffer: c_string, read: size_t, buffer_size: size_t) {
  // writeln("from chpl: " + buffer);
  // accumulate string buffer
  var word = buffer;
  var trimmedWord: string = "";
  writeln("word: ", word);
  for i in 1..word.length {
  	var ch = word.substring(i);
  	if (ch != "\r" && ch != "\n") {
	  	writeln("ch: ", word.substring(i));
	  	trimmedWord += word.substring(i);
  	}
  }
//  writeln("trimmedWord: ", trimmedWord);
	if (trimmedWord == "dump") {
		dumpPartition(partitionForWord("dog"));
	} else {
		writeln("<adding>");
	  indexWord(trimmedWord, 1);
		dumpPostingTableForWord(trimmedWord);
		writeln("</adding>");
	}
  send(fd, buffer, read, 0);
}

proc initIndex() {
	writeln("This program is running on ", numLocales, " locales");
	writeln("It began running on locale #", here.id);
	writeln();

  initPartitions();
  initIndices();

  initIndexer();

 //  writeln("adding 3 entries");
	// indexWord("dog", 1);
	// indexWord("cat", 2);
	// indexWord("cat", 3);

 //  writeln("dumping partition holding word: dog");
	// dumpPartition(partitionForWord("dog"));

 //  writeln("dumping posting table for word: cat");
	// dumpPostingTableForWord("cat");

  var infile = open("words.txt", iomode.r);
  var reader = infile.reader();
  var word: string;
  var docId: DocId = 1;
  while (reader.readln(word)) {
    enqueueIndexRequest(word, docId);
    docId = (docId + 1) % 1000 + 1; // fake different docs
  }

  waitForIndexer();

  dumpPostingTableForWord("the");

  // TODO: build execution Tree w/ conj / disj. (operator) nodes
  // test basic boolean operators
  writeln("conjunction");
  var conj = conjunction(["the", "dog"]);
  writeln(conj);

  writeln("disjunction");
  var disj = disjunction(["the", "dog"]);
  writeln(disj);
}

proc conjunction(words: [] string): domain(DocId) {
  writeln("finding conjunction of: ", words);
  var doms: [1..words.size] domain(DocId);

  for j in 1..words.size {
    var word = words[j];
    var entry = entryForWord(word);
    if (entry != nil) {
      on entry {
        for docId in documentIdsForEntry(entry) {
          doms[j] += docId;
        }
      }
    }
  }

  writeln("applying intersection");

  for j in 2..words.size {
    doms[1] = doms[1] & doms[j];
  }

  return doms[1];
}

proc disjunction(words: [] string): domain(DocId) {
  writeln("finding disjunction of: ", words);
  var doms: [1..words.size] domain(DocId);

  for j in 1..words.size {
    var word = words[j];
    var entry = entryForWord(word);
    if (entry != nil) {
      on entry {
        for docId in documentIdsForEntry(entry) {
          doms[j] += docId;
        }
      }
    }
  }

  writeln("applying disjunction");

  for j in 2..words.size {
    doms[1] = doms[1] | doms[j];
  }

  return doms[1];
}

proc writeLocInfo(loc: locale) {
  on loc {
    writeln("locale #", here.id, "...");
    writeln("  ...is named: ", here.name);
    writeln("  ...has ", here.numCores, " processor cores");
    writeln("  ...has ", here.physicalMemory(unit=MemUnits.GB, retType=real), " GB of memory");
    writeln("  ...has ", here.maxTaskPar, " maximum parallelism");
  }
}

proc main() {

	// writeln("creating socket...");
	// var sd: ev_fd = initialize_socket(port);
	// writeln("socket id = ", sd);
	// if (sd == -1) {
	// 	writeln("socket error");
	// 	return -1;
	// }

  writeln("initializing index");
  initIndex();

	// writeln("initializing event loop...");

 //  // port c_accept_cb, c_read_cb
	// var w_accept: ev_io = new ev_io();
	// ev_io_init(w_accept, c_accept_cb, sd, EV_READ);
	// ev_io_start(EV_DEFAULT, w_accept);

	// while (1) {
	// 	ev_loop_fn(EV_DEFAULT, 0);
	// }
}


This change parallelizes radix sort.

This change parallelizes radix sort. Several loops were written in serial,
but using sync variables, we can now run loops in parallel (though with a
larger memory allocation hit).

git-svn-id: 88467cb1fb04b8a755be7e1ee1026be4190196ef@18742 3a8e244f-b0f2-452b-bcba-4c88e055c3ca

use Time, Random;

// radix_sort
// ----------
// Values is the 0-based array to be sorted.
// radix and nbits toggles how many keys are generated and processed per pass.

// assert indices for Values starts at 0
proc radix_sort(Values, Permute, radix:int(64), nbits:int(64)): void {

    var nelem:int(64) = Values.numElements;
    var nbuckets:int(64) = 1 << radix;   // Number of keys in counting sort

    var D1 = [0..(nelem-1)];
    var D2 = [0..(nbuckets-1)];

    var Permute_old: [D1] int(64) = Permute; // Old permutation array

    var sortTime: real;                           // overall timing
    const sortStartTime = getCurrentTime();       // capture the start time

    var npasses:int(64) = (nbits / radix);
    if ( (npasses * radix) < nbits ) then npasses = npasses + 1;

    for pass in (0..(npasses-1)) {
      var Index: [D1] int(64);           // Index
      var Offsets$: [D2] sync int(64);    // Starting offset of
                                         // histogrammed values
      var Offsets_old: [D2] int(64);     // Old offsets
      var Histogram$: [D2] sync int(64);  // Histogram for counting sort
      var All_counts: [D1] int(64);      // Storage for counting sort tallies

      var phaseTime: real;                        // timing of pass
      const phaseStartTime = getCurrentTime();    // capture the pass start

      // Mask of bits sorted on this pass
      var mask:int(64) = (nbuckets - 1) << (pass * radix);

      Histogram$.writeEF(0);
      forall i in D1 {
        var r:int(64) = ((mask & Values[Permute_old[i]]) >> (pass * radix));
        Histogram$[r] += 1;
      }

// write this loop with an aggregate array operation
      Offsets$[0].writeXF(0);
      for i in (1..(nbuckets-1)) {
        Offsets$[i].writeXF(Offsets$[(i-1)].readXX() + Histogram$[(i-1)].readXX());
      }
      Offsets_old = Offsets$.readXX();

      forall i in D1 {
        var r:int(64) = ((mask & Values[Permute_old[i]]) >> (pass * radix));
        var loc:int(64) = Offsets$[r];
        Offsets$[r] = loc + 1;
        Permute[loc] = Permute_old[i];
        Index[loc] = i;
      }

      for b in D2 {
        var base:int(64) = Offsets_old[b];
        var Count$: [base..(nelem-1)] sync int(64) = All_counts.localSlice(base..(nelem-1));

        forall i in (0..(Histogram$[b].readXX()-1)) {
          Count$[(i+base)].writeXF(0);
          Permute_old[(i+base)] = Permute[(i+base)];
        }

        for i in (1..(Histogram$[b].readXX()-1)) {
          forall j in (0..(i-1)) {
            if (Index[(i+base)] < Index[(j+base)]) {
              Count$[(j+base)] += 1;
            } else {
              Count$[(i+base)] += 1;
            }
          }
        }

        forall i in (0..(Histogram$[b].readXX()-1)) {
          Permute[(Count$[(i+base)].readXX()+base)] = Permute_old[(i+base)];
        }
      }

      var Tmp: [D1] int(64) = Permute_old;
      Permute_old = Permute;
      Permute = Tmp;

      phaseTime = getCurrentTime() - phaseStartTime; // store the elapsed pass
//      writeln("Completed pass ", pass, " in ", phaseTime, " sec    mask=", mask);

    }

    Permute = Permute_old;

    sortTime = getCurrentTime() - sortStartTime;  // store the elapsed time
//    writeln(npasses, " passes in ", sortTime, " secs");

    return;
}

config const npoints:int(64) = 8;
writeln("Using ", npoints, " points");

var D = [0..(npoints-1)];
var F: [D] real(64);
var F_prime: [D] int(64);
var G: [D] int(64);
var Permute_F: [D] int(64);
var Permute_G: [D] int(64);

var rngTime: real;
writeln("Generating random numbers...");
const rngStartTime = getCurrentTime();
fillRandom(F, 65535);
F_prime = (F * 9223372036854775808):int(64);
rngTime = getCurrentTime() - rngStartTime;
//writeln("Finished generating numbers in ", rngTime, " sec");

forall i in D {
  Permute_F[i] = i;
  Permute_G[i] = i;
  G[i] = i % 10;
}

var mwTime: real;
const mwStartTime = getCurrentTime();

radix_sort(F_prime, Permute_F, 16, 64);
radix_sort(G, Permute_G, 5, 5);

mwTime = getCurrentTime() - mwStartTime;
//writeln("----- Total Time = ", mwTime, " sec -----------------------------------------");

var selfcheckTime: real;
const selfcheckStartTime = getCurrentTime();
var nerr_f: int(64) = 0;
var nerr_g: int(64) = 0;

writeln("0: ", G[Permute_G[0]], " - ", F_prime[Permute_F[0]]);
for i in (1..(npoints-1)) {
  writeln(i, ": ", G[Permute_G[i]], " - ", F_prime[Permute_F[i]]);
  if (F_prime[Permute_F[i]] <= F_prime[Permute_F[(i-1)]]) then nerr_f = nerr_f + 1;
  if (G[Permute_G[i]] < G[Permute_G[(i-1)]]) then nerr_g = nerr_g + 1;
}

selfcheckTime = getCurrentTime() - selfcheckStartTime;
writeln("Number of errors: f=", nerr_f, "  g=", nerr_g);
//writeln("Checked in ", selfcheckTime, " sec");

use Time, Random;

// radix_sort
// ----------
// Values is the 0-based array to be sorted.
// radix and nbits toggles how many keys are generated and processed per pass.

// assert indices for Values starts at 0
proc radix_sort(Values, Permute, radix:int(64), nbits:int(64)): void {

    var nelem:int(64) = Values.numElements;
    var nbuckets:int(64) = 1 << radix;   // Number of keys in counting sort

    var D1 = [0..(nelem-1)];
    var D2 = [0..(nbuckets-1)];

    var Permute_old: [D1] int(64) = Permute; // Old permutation array

    var sortTime: real;                           // overall timing
    const sortStartTime = getCurrentTime();       // capture the start time

    var npasses:int(64) = (nbits / radix);
    if ( (npasses * radix) < nbits ) then npasses = npasses + 1;

    for pass in (0..(npasses-1)) {
      var Index: [D1] int(64);           // Index
      var Offsets: [D2] int(64);         // Starting offset of
                                         // histogrammed values
      var Offsets_old: [D2] int(64);     // Old offsets
      var Histogram: [D2] int(64);       // Histogram for counting sort
      var All_counts: [D1] int(64);      // Storage for counting sort tallies

      var phaseTime: real;                        // timing of pass
      const phaseStartTime = getCurrentTime();    // capture the pass start

      // Mask of bits sorted on this pass
      var mask:int(64) = (nbuckets - 1) << (pass * radix);

      for i in D1 { // was forall
        var r:int(64) = ((mask & Values[Permute_old[i]]) >> (pass * radix));
        Histogram[r] += 1;
      }

// write this loop with an aggregate array operation
      for i in (1..(nbuckets-1)) {
        Offsets[i] = Offsets[(i-1)] + Histogram[(i-1)];
      }
      Offsets_old = Offsets;

      for i in D1 { // was forall
        var r:int(64) = ((mask & Values[Permute_old[i]]) >> (pass * radix));
        var loc:int(64) = Offsets[r];
        Offsets[r] += 1;
        Permute[loc] = Permute_old[i];
        Index[loc] = i;
      }

      for b in D2 {
        var base:int(64) = Offsets_old[b];
        var Count = All_counts.localSlice(base..(nelem-1));

        for i in (0..(Histogram[b]-1)) { // was forall
          Count[(i+base)] = 0;
          Permute_old[(i+base)] = Permute[(i+base)];
        }

        for i in (1..(Histogram[b]-1)) {
          for j in (0..(i-1)) { // was forall
            if (Index[(i+base)] < Index[(j+base)]) {
              Count[(j+base)] += 1;
            } else {
              Count[(i+base)] += 1;
            }
          }
        }

        for i in (0..(Histogram[b]-1)) { //was forall
          Permute[(Count[(i+base)]+base)] = Permute_old[(i+base)];
        }
      }

      var Tmp: [D1] int(64) = Permute_old;
      Permute_old = Permute;
      Permute = Tmp;

      phaseTime = getCurrentTime() - phaseStartTime; // store the elapsed pass
//      writeln("Completed pass ", pass, " in ", phaseTime, " sec    mask=", mask);

    }

    Permute = Permute_old;

    sortTime = getCurrentTime() - sortStartTime;  // store the elapsed time
//    writeln(npasses, " passes in ", sortTime, " secs");

    return;
}

config const npoints:int(64) = 8;
writeln("Using ", npoints, " points");

var D = [0..(npoints-1)];
var F: [D] real(64);
var F_prime: [D] int(64);
var G: [D] int(64);
var Permute_F: [D] int(64);
var Permute_G: [D] int(64);

var rngTime: real;
writeln("Generating random numbers...");
const rngStartTime = getCurrentTime();
fillRandom(F, 65535);
F_prime = (F * 9223372036854775808):int(64);
rngTime = getCurrentTime() - rngStartTime;
//writeln("Finished generating numbers in ", rngTime, " sec");

forall i in D {
  Permute_F[i] = i;
  Permute_G[i] = i;
  G[i] = i % 10;
}

var mwTime: real;
const mwStartTime = getCurrentTime();

radix_sort(F_prime, Permute_F, 16, 64);
radix_sort(G, Permute_G, 5, 5);

mwTime = getCurrentTime() - mwStartTime;
//writeln("----- Total Time = ", mwTime, " sec -----------------------------------------");

var selfcheckTime: real;
const selfcheckStartTime = getCurrentTime();
var nerr_f: int(64) = 0;
var nerr_g: int(64) = 0;

writeln("0: ", G[Permute_G[0]], " - ", F_prime[Permute_F[0]]);
for i in (1..(npoints-1)) {
  writeln(i, ": ", G[Permute_G[i]], " - ", F_prime[Permute_F[i]]);
  if (F_prime[Permute_F[i]] <= F_prime[Permute_F[(i-1)]]) then nerr_f = nerr_f + 1;
  if (G[Permute_G[i]] < G[Permute_G[(i-1)]]) then nerr_g = nerr_g + 1;
}

selfcheckTime = getCurrentTime() - selfcheckStartTime;
writeln("Number of errors: f=", nerr_f, "  g=", nerr_g);
//writeln("Checked in ", selfcheckTime, " sec");