signature:
def spDiff ( SP1 , SP2 ) :
    """Function that compares two spatial pooler instances . Compares the
 static variables between the two poolers to make sure that they are equivalent .
 Parameters
 SP1 first spatial pooler to be compared
 SP2 second spatial pooler to be compared
 To establish equality , this function does the following :
 1 . Compares the connected synapse matrices for each coincidence
 2 . Compare the potential synapse matrices for each coincidence
 3 . Compare the permanence matrices for each coincidence
 4 . Compare the firing boosts between the two poolers .
 5 . Compare the duty cycles before and after inhibition for both poolers"""

implementation:
if ( len ( SP1 . _masterConnectedM ) != len ( SP2 . _masterConnectedM ) ) :
    print "Connected synapse matrices are different sizes"
    return False
if ( len ( SP1 . _masterPotentialM ) != len ( SP2 . _masterPotentialM ) ) :
    print "Potential synapse matrices are different sizes"
    return False
if ( len ( SP1 . _masterPermanenceM ) != len ( SP2 . _masterPermanenceM ) ) :
    print "Permanence matrices are different sizes"
    return False
# iterate over cells
for i in range ( 0 , len ( SP1 . _masterConnectedM ) ) : # grab the Coincidence Matrices and compare them
    connected1 = SP1 . _masterConnectedM [ i ]
    connected2 = SP2 . _masterConnectedM [ i ]
    if ( connected1 != connected2 ) :
        print "Connected Matrices for cell %d different" % ( i )
        return False
    # grab permanence Matrices and compare them
    permanences1 = SP1 . _masterPermanenceM [ i ] ;
    permanences2 = SP2 . _masterPermanenceM [ i ] ;
    if ( permanences1 != permanences2 ) :
        print "Permanence Matrices for cell %d different" % ( i )
        return False
    # grab the potential connection Matrices and compare them
    potential1 = SP1 . _masterPotentialM [ i ] ;
    potential2 = SP2 . _masterPotentialM [ i ] ;
    if ( potential1 != potential2 ) :
        print "Potential Matrices for cell %d different" % ( i )
        return False
# Check firing boosts
if ( not numpy . array_equal ( SP1 . _firingBoostFactors , SP2 . _firingBoostFactors ) ) :
    print "Firing boost factors are different between spatial poolers"
    return False
# Check duty cycles after inhibiton
if ( not numpy . array_equal ( SP1 . _dutyCycleAfterInh , SP2 . _dutyCycleAfterInh ) ) :
    print "Duty cycles after inhibition are different between spatial poolers"
    return False
# Check duty cycles before inhibition
if ( not numpy . array_equal ( SP1 . _dutyCycleBeforeInh , SP2 . _dutyCycleBeforeInh ) ) :
    print "Duty cycles before inhibition are different between spatial poolers"
    return False
print ( "Spatial Poolers are equivalent" )
return True


signature:
def removeSeqStarts ( vectors , resets , numSteps = 1 ) :
    """Convert a list of sequences of pattern indices , and a pattern lookup table
 into a an array of patterns
 Parameters :
 vectors : the data vectors . Row 0 contains the outputs from time
 step 0 , row 1 from time step 1 , etc .
 resets : the reset signal . This is a vector of booleans
 the same length as the number of rows in ' vectors ' . It
 has a 1 where a sequence started and a 0 otherwise . The
 first ' numSteps ' rows of ' vectors ' of each sequence will
 not be included in the return result .
 numSteps Number of samples to remove from the start of each sequence
 retval : copy of vectors , with the first ' numSteps ' samples at the
 start of each sequence removed ."""

implementation:
# Do nothing if numSteps is 0
if numSteps == 0 :
    return vectors
resetIndices = resets . nonzero ( ) [ 0 ]
removeRows = resetIndices
for i in range ( numSteps - 1 ) :
    removeRows = numpy . hstack ( ( removeRows , resetIndices + i + 1 ) )
return numpy . delete ( vectors , removeRows , axis = 0 )


signature:
def _accumulateFrequencyCounts ( values , freqCounts = None ) :
    """Accumulate a list of values ' values ' into the frequency counts ' freqCounts ' ,
 and return the updated frequency counts
 For example , if values contained the following : [ 1,1,3,5,1,3,5 ] , and the initial
 freqCounts was None , then the return value would be :
 [0,3,0,2,0,2]
 which corresponds to how many of each value we saw in the input , i . e . there
 were 0 0 ' s , 3 1 ' s , 0 2 ' s , 2 3 ' s , 0 4 ' s , and 2 5 ' s .
 If freqCounts is not None , the values will be added to the existing counts and
 the length of the frequency Counts will be automatically extended as necessary
 Parameters :
 values : The values to accumulate into the frequency counts
 freqCounts : Accumulated frequency counts so far , or none"""

implementation:
# How big does our freqCounts vector need to be ?
values = numpy . array ( values )
numEntries = values . max ( ) + 1
if freqCounts is not None :
    numEntries = max ( numEntries , freqCounts . size )
# Where do we accumulate the results ?
if freqCounts is not None :
    if freqCounts . size != numEntries :
        newCounts = numpy . zeros ( numEntries , dtype = 'int32' )
        newCounts [ 0 : freqCounts . size ] = freqCounts
    else :
        newCounts = freqCounts
else :
    newCounts = numpy . zeros ( numEntries , dtype = 'int32' )
# Accumulate the new values
for v in values :
    newCounts [ v ] += 1
return newCounts


signature:
def _listOfOnTimesInVec ( vector ) :
    """Returns 3 things for a vector :
 * the total on time
 * the number of runs
 * a list of the durations of each run .
 Parameters :
 input stream : 1110000011000001111100000
 return value : ( 11 , 3 , [ 3 , 2 , 6 ] )"""

implementation:
# init counters
durations = [ ]
numOnTimes = 0
totalOnTime = 0
# Find where the nonzeros are
nonzeros = numpy . array ( vector ) . nonzero ( ) [ 0 ]
# Nothing to do if vector is empty
if len ( nonzeros ) == 0 :
    return ( 0 , 0 , [ ] )
# Special case of only 1 on bit
if len ( nonzeros ) == 1 :
    return ( 1 , 1 , [ 1 ] )
# Count the consecutive non - zeros
prev = nonzeros [ 0 ]
onTime = 1
endIdx = nonzeros [ - 1 ]
for idx in nonzeros [ 1 : ] :
    if idx != prev + 1 :
        totalOnTime += onTime
        numOnTimes += 1
        durations . append ( onTime )
        onTime = 1
    else :
        onTime += 1
    prev = idx
# Add in the last one
totalOnTime += onTime
numOnTimes += 1
durations . append ( onTime )
return ( totalOnTime , numOnTimes , durations )


signature:
def _fillInOnTimes ( vector , durations ) :
    """Helper function used by averageOnTimePerTimestep . ' durations ' is a vector
 which must be the same len as vector . For each " on " in vector , it fills in
 the corresponding element of duration with the duration of that " on " signal
 up until that time
 Parameters :
 vector : vector of output values over time
 durations : vector same length as ' vector ' , initialized to 0 ' s .
 This is filled in with the durations of each ' on " signal .
 Example :
 vector : 1110000011000001111100000
 durations : 12300000120000012345600000"""

implementation:
# Find where the nonzeros are
nonzeros = numpy . array ( vector ) . nonzero ( ) [ 0 ]
# Nothing to do if vector is empty
if len ( nonzeros ) == 0 :
    return
# Special case of only 1 on bit
if len ( nonzeros ) == 1 :
    durations [ nonzeros [ 0 ] ] = 1
    return
# Count the consecutive non - zeros
prev = nonzeros [ 0 ]
onTime = 1
onStartIdx = prev
endIdx = nonzeros [ - 1 ]
for idx in nonzeros [ 1 : ] :
    if idx != prev + 1 : # Fill in the durations
        durations [ onStartIdx : onStartIdx + onTime ] = range ( 1 , onTime + 1 )
        onTime = 1
        onStartIdx = idx
    else :
        onTime += 1
    prev = idx
# Fill in the last one
durations [ onStartIdx : onStartIdx + onTime ] = range ( 1 , onTime + 1 )


signature:
def averageOnTimePerTimestep ( vectors , numSamples = None ) :
    """Computes the average on - time of the outputs that are on at each time step , and
 then averages this over all time steps .
 This metric is resiliant to the number of outputs that are on at each time
 step . That is , if time step 0 has many more outputs on than time step 100 , it
 won ' t skew the results . This is particularly useful when measuring the
 average on - time of things like the temporal memory output where you might
 have many columns bursting at the start of a sequence - you don ' t want those
 start of sequence bursts to over - influence the calculated average on - time .
 Parameters :
 vectors : the vectors for which the onTime is calculated . Row 0
 contains the outputs from time step 0 , row 1 from time step
 1 , etc .
 numSamples : the number of elements for which on - time is calculated .
 If not specified , then all elements are looked at .
 Returns ( scalar average on - time over all time steps ,
 list containing frequency counts of each encountered on - time )"""

implementation:
# Special case given a 1 dimensional vector : it represents a single column
if vectors . ndim == 1 :
    vectors . shape = ( - 1 , 1 )
numTimeSteps = len ( vectors )
numElements = len ( vectors [ 0 ] )
# How many samples will we look at ?
if numSamples is not None :
    import pdb ;
    pdb . set_trace ( )
    # Test this . . . .
    countOn = numpy . random . randint ( 0 , numElements , numSamples )
    vectors = vectors [ : , countOn ]
# Fill in each non - zero of vectors with the on - time that that output was
# on for .
durations = numpy . zeros ( vectors . shape , dtype = 'int32' )
for col in xrange ( vectors . shape [ 1 ] ) :
    _fillInOnTimes ( vectors [ : , col ] , durations [ : , col ] )
# Compute the average on time for each time step
sums = vectors . sum ( axis = 1 )
sums . clip ( min = 1 , max = numpy . inf , out = sums )
avgDurations = durations . sum ( axis = 1 , dtype = 'float64' ) / sums
avgOnTime = avgDurations . sum ( ) / ( avgDurations > 0 ) . sum ( )
# Generate the frequency counts for each duration
freqCounts = _accumulateFrequencyCounts ( avgDurations )
return ( avgOnTime , freqCounts )


signature:
def averageOnTime ( vectors , numSamples = None ) :
    """Returns the average on - time , averaged over all on - time runs .
 Parameters :
 vectors : the vectors for which the onTime is calculated . Row 0
 contains the outputs from time step 0 , row 1 from time step
 1 , etc .
 numSamples : the number of elements for which on - time is calculated .
 If not specified , then all elements are looked at .
 Returns : ( scalar average on - time of all outputs ,
 list containing frequency counts of each encountered on - time )"""

implementation:
# Special case given a 1 dimensional vector : it represents a single column
if vectors . ndim == 1 :
    vectors . shape = ( - 1 , 1 )
numTimeSteps = len ( vectors )
numElements = len ( vectors [ 0 ] )
# How many samples will we look at ?
if numSamples is None :
    numSamples = numElements
    countOn = range ( numElements )
else :
    countOn = numpy . random . randint ( 0 , numElements , numSamples )
# Compute the on - times and accumulate the frequency counts of each on - time
# encountered
sumOfLengths = 0.0
onTimeFreqCounts = None
n = 0
for i in countOn :
    ( onTime , segments , durations ) = _listOfOnTimesInVec ( vectors [ : , i ] )
    if onTime != 0.0 :
        sumOfLengths += onTime
        n += segments
        onTimeFreqCounts = _accumulateFrequencyCounts ( durations , onTimeFreqCounts )
# Return the average on time of each element that was on .
if n > 0 :
    return ( sumOfLengths / n , onTimeFreqCounts )
else :
    return ( 0.0 , onTimeFreqCounts )


signature:
def plotOutputsOverTime ( vectors , buVectors = None , title = 'On-times' ) :
    """Generate a figure that shows each output over time . Time goes left to right ,
 and each output is plotted on a different line , allowing you to see the overlap
 in the outputs , when they turn on / off , etc .
 Parameters :
 vectors : the vectors to plot
 buVectors : These are normally specified when plotting the pooling
 outputs of the temporal memory over time . The ' buVectors '
 are the sequence outputs and the ' vectors ' are the
 pooling outputs . The buVector ( sequence ) outputs will be drawn
 in a darker color than the vector ( pooling ) outputs to
 distinguish where the cell is outputting due to pooling vs .
 sequence memory .
 title : title for the plot
 avgOnTime : The average on - time measurement . If not supplied ,
 then it will be calculated from the passed in vectors ."""

implementation:
# Produce the plot
import pylab
pylab . ion ( )
pylab . figure ( )
imData = vectors . transpose ( )
if buVectors is not None :
    assert ( buVectors . shape == vectors . shape )
    imData = imData . copy ( )
    imData [ buVectors . transpose ( ) . astype ( 'bool' ) ] = 2
pylab . imshow ( imData , aspect = 'auto' , cmap = pylab . cm . gray_r , interpolation = 'nearest' )
pylab . title ( title )


signature:
def plotHistogram ( freqCounts , title = 'On-Times Histogram' , xLabel = 'On-Time' ) :
    """This is usually used to display a histogram of the on - times encountered
 in a particular output .
 The freqCounts is a vector containg the frequency counts of each on - time
 ( starting at an on - time of 0 and going to an on - time = len ( freqCounts ) - 1)
 The freqCounts are typically generated from the averageOnTimePerTimestep
 or averageOnTime methods of this module .
 Parameters :
 freqCounts : The frequency counts to plot
 title : Title of the plot"""

implementation:
import pylab
pylab . ion ( )
pylab . figure ( )
pylab . bar ( numpy . arange ( len ( freqCounts ) ) - 0.5 , freqCounts )
pylab . title ( title )
pylab . xlabel ( xLabel )


signature:
def populationStability ( vectors , numSamples = None ) :
    """Returns the stability for the population averaged over multiple time steps
 Parameters :
 vectors : the vectors for which the stability is calculated
 numSamples the number of time steps where stability is counted
 At each time step , count the fraction of the active elements which are stable
 from the previous step
 Average all the fraction"""

implementation:
# Calculate the stability
numVectors = len ( vectors )
if numSamples is None :
    numSamples = numVectors - 1
    countOn = range ( numVectors - 1 )
else :
    countOn = numpy . random . randint ( 0 , numVectors - 1 , numSamples )
sigmap = 0.0
for i in countOn :
    match = checkMatch ( vectors [ i ] , vectors [ i + 1 ] , sparse = False )
    # Ignore reset vectors ( all 0 ' s )
    if match [ 1 ] != 0 :
        sigmap += float ( match [ 0 ] ) / match [ 1 ]
return sigmap / numSamples


signature:
def percentOutputsStableOverNTimeSteps ( vectors , numSamples = None ) :
    """Returns the percent of the outputs that remain completely stable over
 N time steps .
 Parameters :
 vectors : the vectors for which the stability is calculated
 numSamples : the number of time steps where stability is counted
 For each window of numSamples , count how many outputs are active during
 the entire window ."""

implementation:
# Calculate the stability
totalSamples = len ( vectors )
windowSize = numSamples
# Process each window
numWindows = 0
pctStable = 0
for wStart in range ( 0 , totalSamples - windowSize + 1 ) : # Count how many elements are active for the entire time
    data = vectors [ wStart : wStart + windowSize ]
    outputSums = data . sum ( axis = 0 )
    stableOutputs = ( outputSums == windowSize ) . sum ( )
    # Accumulated
    samplePctStable = float ( stableOutputs ) / data [ 0 ] . sum ( )
    print samplePctStable
    pctStable += samplePctStable
    numWindows += 1
# Return percent average over all possible windows
return float ( pctStable ) / numWindows


signature:
def computeSaturationLevels ( outputs , outputsShape , sparseForm = False ) :
    """Compute the saturation for a continuous level . This breaks the level into
 multiple regions and computes the saturation level for each region .
 Parameters :
 outputs : output of the level . If sparseForm is True , this is a list of
 the non - zeros . If sparseForm is False , it is the dense
 representation
 outputsShape : The shape of the outputs of the level ( height , width )
 retval : ( sat , innerSat ) :
 sat : list of the saturation levels of each non - empty
 region of the level ( each 0 - > 1.0)
 innerSat : list of the saturation level of each non - empty region
 that is not near an edge ( each 0 - > 1.0)"""

implementation:
# Get the outputs into a SparseBinaryMatrix
if not sparseForm :
    outputs = outputs . reshape ( outputsShape )
    spOut = SM32 ( outputs )
else :
    if len ( outputs ) > 0 :
        assert ( outputs . max ( ) < outputsShape [ 0 ] * outputsShape [ 1 ] )
    spOut = SM32 ( 1 , outputsShape [ 0 ] * outputsShape [ 1 ] )
    spOut . setRowFromSparse ( 0 , outputs , [ 1 ] * len ( outputs ) )
    spOut . reshape ( outputsShape [ 0 ] , outputsShape [ 1 ] )
# Get the activity in each local region using the nNonZerosPerBox method
# This method takes a list of the end row indices and a list of the end
# column indices .
# We will use regions that are 15x15 , which give us about a 1/225 ( . 4 % ) resolution
# on saturation .
regionSize = 15
rows = xrange ( regionSize + 1 , outputsShape [ 0 ] + 1 , regionSize )
cols = xrange ( regionSize + 1 , outputsShape [ 1 ] + 1 , regionSize )
regionSums = spOut . nNonZerosPerBox ( rows , cols )
# Get all the nonzeros out - those are our saturation sums
( locations , values ) = regionSums . tolist ( )
values /= float ( regionSize * regionSize )
sat = list ( values )
# Now , to compute which are the inner regions , we will only take the ones that
# are surrounded by activity above , below , left and right
innerSat = [ ]
locationSet = set ( locations )
for ( location , value ) in itertools . izip ( locations , values ) :
    ( row , col ) = location
    if ( row - 1 , col ) in locationSet and ( row , col - 1 ) in locationSet and ( row + 1 , col ) in locationSet and ( row , col + 1 ) in locationSet :
        innerSat . append ( value )
return ( sat , innerSat )


signature:
def checkMatch ( input , prediction , sparse = True , verbosity = 0 ) :
    """Compares the actual input with the predicted input and returns results
 Parameters :
 input : The actual input
 prediction : the predicted input
 verbosity : If > 0 , print debugging messages
 sparse : If true , they are in sparse form ( list of
 active indices )
 retval ( foundInInput , totalActiveInInput , missingFromInput ,
 totalActiveInPrediction )
 foundInInput : The number of predicted active elements that were
 found in the actual input
 totalActiveInInput : The total number of active elements in the input .
 missingFromInput : The number of predicted active elements that were not
 found in the actual input
 totalActiveInPrediction : The total number of active elements in the prediction"""

implementation:
if sparse :
    activeElementsInInput = set ( input )
    activeElementsInPrediction = set ( prediction )
else :
    activeElementsInInput = set ( input . nonzero ( ) [ 0 ] )
    activeElementsInPrediction = set ( prediction . nonzero ( ) [ 0 ] )
totalActiveInPrediction = len ( activeElementsInPrediction )
totalActiveInInput = len ( activeElementsInInput )
foundInInput = len ( activeElementsInPrediction . intersection ( activeElementsInInput ) )
missingFromInput = len ( activeElementsInPrediction . difference ( activeElementsInInput ) )
missingFromPrediction = len ( activeElementsInInput . difference ( activeElementsInPrediction ) )
if verbosity >= 1 :
    print "preds. found in input:" , foundInInput , "out of" , totalActiveInPrediction ,
    print "; preds. missing from input:" , missingFromInput , "out of" , totalActiveInPrediction ,
    print "; unexpected active in input:" , missingFromPrediction , "out of" , totalActiveInInput
return ( foundInInput , totalActiveInInput , missingFromInput , totalActiveInPrediction )


signature:
def predictionExtent ( inputs , resets , outputs , minOverlapPct = 100.0 ) :
    """Computes the predictive ability of a temporal memory ( TM ) . This routine returns
 a value which is the average number of time steps of prediction provided
 by the TM . It accepts as input the inputs , outputs , and resets provided to
 the TM as well as a ' minOverlapPct ' used to evalulate whether or not a
 prediction is a good enough match to the actual input .
 The ' outputs ' are the pooling outputs of the TM . This routine treats each output
 as a " manifold " that includes the active columns that should be present in the
 next N inputs . It then looks at each successive input and sees if it ' s active
 columns are within the manifold . For each output sample , it computes how
 many time steps it can go forward on the input before the input overlap with
 the manifold is less then ' minOverlapPct ' . It returns the average number of
 time steps calculated for each output .
 Parameters :
 inputs : The inputs to the TM . Row 0 contains the inputs from time
 step 0 , row 1 from time step 1 , etc .
 resets : The reset input to the TM . Element 0 contains the reset from
 time step 0 , element 1 from time step 1 , etc .
 outputs : The pooling outputs from the TM . Row 0 contains the outputs
 from time step 0 , row 1 from time step 1 , etc .
 minOverlapPct : How much each input ' s columns must overlap with the pooling
 output ' s columns to be considered a valid prediction .
 retval : ( Average number of time steps of prediction over all output
 samples ,
 Average number of time steps of prediction when we aren ' t
 cut short by the end of the sequence ,
 List containing frequency counts of each encountered
 prediction time )"""

implementation:
# List of how many times we encountered each prediction amount . Element 0
# is how many times we successfully predicted 0 steps in advance , element 1
# is how many times we predicted 1 step in advance , etc .
predCounts = None
# Total steps of prediction over all samples
predTotal = 0
# Total number of samples
nSamples = len ( outputs )
# Total steps of prediction for samples at the start of the sequence , or
# for samples whose prediction runs aren ' t cut short by the end of the
# sequence .
predTotalNotLimited = 0
nSamplesNotLimited = 0
# Compute how many cells / column we have
nCols = len ( inputs [ 0 ] )
nCellsPerCol = len ( outputs [ 0 ] ) // nCols
# Evalulate prediction for each output sample
for idx in xrange ( nSamples ) : # What are the active columns for this output ?
    activeCols = outputs [ idx ] . reshape ( nCols , nCellsPerCol ) . max ( axis = 1 )
    # How many steps of prediction do we have ?
    steps = 0
    while ( idx + steps + 1 < nSamples ) and ( resets [ idx + steps + 1 ] == 0 ) :
        overlap = numpy . logical_and ( inputs [ idx + steps + 1 ] , activeCols )
        overlapPct = 100.0 * float ( overlap . sum ( ) ) / inputs [ idx + steps + 1 ] . sum ( )
        if overlapPct >= minOverlapPct :
            steps += 1
        else :
            break
    # print " idx : " , idx , " steps : " , steps
    # Accumulate into our total
    predCounts = _accumulateFrequencyCounts ( [ steps ] , predCounts )
    predTotal += steps
    # If this sample was not cut short by the end of the sequence , include
    # it into the " NotLimited " runs
    if resets [ idx ] or ( ( idx + steps + 1 < nSamples ) and ( not resets [ idx + steps + 1 ] ) ) :
        predTotalNotLimited += steps
        nSamplesNotLimited += 1
# Return results
return ( float ( predTotal ) / nSamples , float ( predTotalNotLimited ) / nSamplesNotLimited , predCounts )


signature:
def getCentreAndSpreadOffsets ( spaceShape , spreadShape , stepSize = 1 ) :
    """Generates centre offsets and spread offsets for block - mode based training
 regimes - star , cross , block .
 Parameters :
 spaceShape : The ( height , width ) of the 2 - D space to explore . This
 sets the number of center - points .
 spreadShape : The shape ( height , width ) of the area around each center - point
 to explore .
 stepSize : The step size . How big each step is , in pixels . This controls
 * both * the spacing of the center - points within the block and the
 points we explore around each center - point
 retval : ( centreOffsets , spreadOffsets )"""

implementation:
from nupic . math . cross import cross
# Init data structures
# What is the range on the X and Y offsets of the center points ?
shape = spaceShape
# If the shape is ( 1,1 ) , special case of just 1 center point
if shape [ 0 ] == 1 and shape [ 1 ] == 1 :
    centerOffsets = [ ( 0 , 0 ) ]
else :
    xMin = - 1 * ( shape [ 1 ] // 2 )
    xMax = xMin + shape [ 1 ] - 1
    xPositions = range ( stepSize * xMin , stepSize * xMax + 1 , stepSize )
    yMin = - 1 * ( shape [ 0 ] // 2 )
    yMax = yMin + shape [ 0 ] - 1
    yPositions = range ( stepSize * yMin , stepSize * yMax + 1 , stepSize )
    centerOffsets = list ( cross ( yPositions , xPositions ) )
numCenterOffsets = len ( centerOffsets )
print "centerOffsets:" , centerOffsets
# What is the range on the X and Y offsets of the spread points ?
shape = spreadShape
# If the shape is ( 1,1 ) , special case of no spreading around each center
# point
if shape [ 0 ] == 1 and shape [ 1 ] == 1 :
    spreadOffsets = [ ( 0 , 0 ) ]
else :
    xMin = - 1 * ( shape [ 1 ] // 2 )
    xMax = xMin + shape [ 1 ] - 1
    xPositions = range ( stepSize * xMin , stepSize * xMax + 1 , stepSize )
    yMin = - 1 * ( shape [ 0 ] // 2 )
    yMax = yMin + shape [ 0 ] - 1
    yPositions = range ( stepSize * yMin , stepSize * yMax + 1 , stepSize )
    spreadOffsets = list ( cross ( yPositions , xPositions ) )
    # Put the ( 0,0 ) entry first
    spreadOffsets . remove ( ( 0 , 0 ) )
    spreadOffsets . insert ( 0 , ( 0 , 0 ) )
numSpreadOffsets = len ( spreadOffsets )
print "spreadOffsets:" , spreadOffsets
return centerOffsets , spreadOffsets


implementation:
if outputCloningHeight < 0 :
    outputCloningHeight = outputCloningWidth
columnsHeight , columnsWidth = columnsShape
numDistinctMasters = outputCloningWidth * outputCloningHeight
a = numpy . empty ( ( columnsHeight , columnsWidth ) , 'uint32' )
for row in xrange ( columnsHeight ) :
    for col in xrange ( columnsWidth ) :
        a [ row , col ] = ( col % outputCloningWidth ) + ( row % outputCloningHeight ) * outputCloningWidth
return a , numDistinctMasters


signature:
def numpyStr ( array , format = '%f' , includeIndices = False , includeZeros = True ) :
    """Pretty print a numpy matrix using the given format string for each
 value . Return the string representation
 Parameters :
 array : The numpy array to print . This can be either a 1D vector or 2D matrix
 format : The format string to use for each value
 includeIndices : If true , include [ row , col ] label for each value
 includeZeros : Can only be set to False if includeIndices is on .
 If True , include 0 values in the print - out
 If False , exclude 0 values from the print - out ."""

implementation:
shape = array . shape
assert ( len ( shape ) <= 2 )
items = [ '[' ]
if len ( shape ) == 1 :
    if includeIndices :
        format = '%d:' + format
        if includeZeros :
            rowItems = [ format % ( c , x ) for ( c , x ) in enumerate ( array ) ]
        else :
            rowItems = [ format % ( c , x ) for ( c , x ) in enumerate ( array ) if x != 0 ]
    else :
        rowItems = [ format % ( x ) for x in array ]
    items . extend ( rowItems )
else :
    ( rows , cols ) = shape
    if includeIndices :
        format = '%d,%d:' + format
    for r in xrange ( rows ) :
        if includeIndices :
            rowItems = [ format % ( r , c , x ) for c , x in enumerate ( array [ r ] ) ]
        else :
            rowItems = [ format % ( x ) for x in array [ r ] ]
        if r > 0 :
            items . append ( '' )
        items . append ( '[' )
        items . extend ( rowItems )
        if r < rows - 1 :
            items . append ( ']\n' )
        else :
            items . append ( ']' )
items . append ( ']' )
return ' ' . join ( items )


signature:
def sample ( self , rgen ) :
    """Generates a random sample from the discrete probability distribution
 and returns its value and the log of the probability of sampling that value ."""

implementation:
rf = rgen . uniform ( 0 , self . sum )
index = bisect . bisect ( self . cdf , rf )
return self . keys [ index ] , numpy . log ( self . pmf [ index ] )


signature:
def logProbability ( self , distn ) :
    """Form of distribution must be an array of counts in order of self . keys ."""

implementation:
x = numpy . asarray ( distn )
n = x . sum ( )
return ( logFactorial ( n ) - numpy . sum ( [ logFactorial ( k ) for k in x ] ) + numpy . sum ( x * numpy . log ( self . dist . pmf ) ) )


signature:
def sample ( self , rgen ) :
    """Generates a random sample from the Poisson probability distribution and
 returns its value and the log of the probability of sampling that value ."""

implementation:
x = rgen . poisson ( self . lambdaParameter )
return x , self . logDensity ( x )


signature:
def createDataOutLink ( network , sensorRegionName , regionName ) :
    """Link sensor region to other region so that it can pass it data ."""

implementation: network . link ( sensorRegionName , regionName , "UniformLink" , "" , srcOutput = "dataOut" , destInput = "bottomUpIn" )


signature:
def createFeedForwardLink ( network , regionName1 , regionName2 ) :
    """Create a feed - forward link between 2 regions : regionName1 - > regionName2"""

implementation: network . link ( regionName1 , regionName2 , "UniformLink" , "" , srcOutput = "bottomUpOut" , destInput = "bottomUpIn" )


signature:
def createResetLink ( network , sensorRegionName , regionName ) :
    """Create a reset link from a sensor region : sensorRegionName - > regionName"""

implementation: network . link ( sensorRegionName , regionName , "UniformLink" , "" , srcOutput = "resetOut" , destInput = "resetIn" )


signature:
def createSensorToClassifierLinks ( network , sensorRegionName , classifierRegionName ) :
    """Create required links from a sensor region to a classifier region ."""

implementation:
network . link ( sensorRegionName , classifierRegionName , "UniformLink" , "" , srcOutput = "bucketIdxOut" , destInput = "bucketIdxIn" )
network . link ( sensorRegionName , classifierRegionName , "UniformLink" , "" , srcOutput = "actValueOut" , destInput = "actValueIn" )
network . link ( sensorRegionName , classifierRegionName , "UniformLink" , "" , srcOutput = "categoryOut" , destInput = "categoryIn" )


signature:
def createNetwork ( dataSource ) :
    """Create and initialize a network ."""

implementation:
with open ( _PARAMS_PATH , "r" ) as f :
    modelParams = yaml . safe_load ( f ) [ "modelParams" ]
# Create a network that will hold the regions .
network = Network ( )
# Add a sensor region .
network . addRegion ( "sensor" , "py.RecordSensor" , '{}' )
# Set the encoder and data source of the sensor region .
sensorRegion = network . regions [ "sensor" ] . getSelf ( )
sensorRegion . encoder = createEncoder ( modelParams [ "sensorParams" ] [ "encoders" ] )
sensorRegion . dataSource = dataSource
# Make sure the SP input width matches the sensor region output width .
modelParams [ "spParams" ] [ "inputWidth" ] = sensorRegion . encoder . getWidth ( )
# Add SP and TM regions .
network . addRegion ( "SP" , "py.SPRegion" , json . dumps ( modelParams [ "spParams" ] ) )
network . addRegion ( "TM" , "py.TMRegion" , json . dumps ( modelParams [ "tmParams" ] ) )
# Add a classifier region .
clName = "py.%s" % modelParams [ "clParams" ] . pop ( "regionName" )
network . addRegion ( "classifier" , clName , json . dumps ( modelParams [ "clParams" ] ) )
# Add all links
createSensorToClassifierLinks ( network , "sensor" , "classifier" )
createDataOutLink ( network , "sensor" , "SP" )
createFeedForwardLink ( network , "SP" , "TM" )
createFeedForwardLink ( network , "TM" , "classifier" )
# Reset links are optional , since the sensor region does not send resets .
createResetLink ( network , "sensor" , "SP" )
createResetLink ( network , "sensor" , "TM" )
# Make sure all objects are initialized .
network . initialize ( )
return network


signature:
def getPredictionResults ( network , clRegionName ) :
    """Get prediction results for all prediction steps ."""

implementation:
classifierRegion = network . regions [ clRegionName ]
actualValues = classifierRegion . getOutputData ( "actualValues" )
probabilities = classifierRegion . getOutputData ( "probabilities" )
steps = classifierRegion . getSelf ( ) . stepsList
N = classifierRegion . getSelf ( ) . maxCategoryCount
results = { step : { } for step in steps }
for i in range ( len ( steps ) ) : # stepProbabilities are probabilities for this prediction step only .
    stepProbabilities = probabilities [ i * N : ( i + 1 ) * N - 1 ]
    mostLikelyCategoryIdx = stepProbabilities . argmax ( )
    predictedValue = actualValues [ mostLikelyCategoryIdx ]
    predictionConfidence = stepProbabilities [ mostLikelyCategoryIdx ]
    results [ steps [ i ] ] [ "predictedValue" ] = predictedValue
    results [ steps [ i ] ] [ "predictionConfidence" ] = predictionConfidence
return results


signature:
def runHotgym ( numRecords ) :
    """Run the Hot Gym example ."""

implementation:
# Create a data source for the network .
dataSource = FileRecordStream ( streamID = _INPUT_FILE_PATH )
numRecords = min ( numRecords , dataSource . getDataRowCount ( ) )
network = createNetwork ( dataSource )
# Set predicted field
network . regions [ "sensor" ] . setParameter ( "predictedField" , "consumption" )
# Enable learning for all regions .
network . regions [ "SP" ] . setParameter ( "learningMode" , 1 )
network . regions [ "TM" ] . setParameter ( "learningMode" , 1 )
network . regions [ "classifier" ] . setParameter ( "learningMode" , 1 )
# Enable inference for all regions .
network . regions [ "SP" ] . setParameter ( "inferenceMode" , 1 )
network . regions [ "TM" ] . setParameter ( "inferenceMode" , 1 )
network . regions [ "classifier" ] . setParameter ( "inferenceMode" , 1 )
results = [ ]
N = 1
# Run the network , N iterations at a time .
for iteration in range ( 0 , numRecords , N ) :
    network . run ( N )
    predictionResults = getPredictionResults ( network , "classifier" )
    oneStep = predictionResults [ 1 ] [ "predictedValue" ]
    oneStepConfidence = predictionResults [ 1 ] [ "predictionConfidence" ]
    fiveStep = predictionResults [ 5 ] [ "predictedValue" ]
    fiveStepConfidence = predictionResults [ 5 ] [ "predictionConfidence" ]
    result = ( oneStep , oneStepConfidence * 100 , fiveStep , fiveStepConfidence * 100 )
    print "1-step: {:16} ({:4.4}%)\t 5-step: {:16} ({:4.4}%)" . format ( * result )
    results . append ( result )
return results


signature:
def _loadDummyModelParameters ( self , params ) :
    """Loads all the parameters for this dummy model . For any paramters
 specified as lists , read the appropriate value for this model using the model
 index"""

implementation:
for key , value in params . iteritems ( ) :
    if type ( value ) == list :
        index = self . modelIndex % len ( params [ key ] )
        self . _params [ key ] = params [ key ] [ index ]
    else :
        self . _params [ key ] = params [ key ]


signature:
def _computModelDelay ( self ) :
    """Computes the amount of time ( if any ) to delay the run of this model .
 This can be determined by two mutually exclusive parameters :
 delay and sleepModelRange .
 ' delay ' specifies the number of seconds a model should be delayed . If a list
 is specified , the appropriate amount of delay is determined by using the
 model ' s modelIndex property .
 However , this doesn ' t work when testing orphaned models , because the
 modelIndex will be the same for every recovery attempt . Therefore , every
 recovery attempt will also be delayed and potentially orphaned .
 ' sleepModelRange ' doesn ' t use the modelIndex property for a model , but rather
 sees which order the model is in the database , and uses that to determine
 whether or not a model should be delayed ."""

implementation:
# ' delay ' and ' sleepModelRange ' are mutually exclusive
if self . _params [ 'delay' ] is not None and self . _params [ 'sleepModelRange' ] is not None :
    raise RuntimeError ( "Only one of 'delay' or " "'sleepModelRange' may be specified" )
# Get the sleepModel range
if self . _sleepModelRange is not None :
    range , delay = self . _sleepModelRange . split ( ':' )
    delay = float ( delay )
    range = map ( int , range . split ( ',' ) )
    modelIDs = self . _jobsDAO . jobGetModelIDs ( self . _jobID )
    modelIDs . sort ( )
    range [ 1 ] = min ( range [ 1 ] , len ( modelIDs ) )
    # If the model is in range , add the delay
    if self . _modelID in modelIDs [ range [ 0 ] : range [ 1 ] ] :
        self . _delay = delay
else :
    self . _delay = self . _params [ 'delay' ]


signature:
def _getMetrics ( self ) :
    """Protected function that can be overridden by subclasses . Its main purpose
 is to allow the the OPFDummyModelRunner to override this with deterministic
 values
 Returns : All the metrics being computed for this model"""

implementation:
metric = None
if self . metrics is not None :
    metric = self . metrics ( self . _currentRecordIndex + 1 )
elif self . metricValue is not None :
    metric = self . metricValue
else :
    raise RuntimeError ( 'No metrics or metric value specified for dummy model' )
return { self . _optimizeKeyPattern : metric }


signature:
def run ( self ) :
    """Runs the given OPF task against the given Model instance"""

implementation:
self . _logger . debug ( "Starting Dummy Model: modelID=%s;" % ( self . _modelID ) )
# Initialize periodic activities ( e . g . , for model result updates )
periodic = self . _initPeriodicActivities ( )
self . _optimizedMetricLabel = self . _optimizeKeyPattern
self . _reportMetricLabels = [ self . _optimizeKeyPattern ]
# Create our top - level loop - control iterator
if self . _iterations >= 0 :
    iterTracker = iter ( xrange ( self . _iterations ) )
else :
    iterTracker = iter ( itertools . count ( ) )
# This gets set in the unit tests . It tells the worker to sys exit
# the first N models . This is how we generate orphaned models
doSysExit = False
if self . _sysExitModelRange is not None :
    modelAndCounters = self . _jobsDAO . modelsGetUpdateCounters ( self . _jobID )
    modelIDs = [ x [ 0 ] for x in modelAndCounters ]
    modelIDs . sort ( )
    ( beg , end ) = self . _sysExitModelRange
    if self . _modelID in modelIDs [ int ( beg ) : int ( end ) ] :
        doSysExit = True
if self . _delayModelRange is not None :
    modelAndCounters = self . _jobsDAO . modelsGetUpdateCounters ( self . _jobID )
    modelIDs = [ x [ 0 ] for x in modelAndCounters ]
    modelIDs . sort ( )
    ( beg , end ) = self . _delayModelRange
    if self . _modelID in modelIDs [ int ( beg ) : int ( end ) ] :
        time . sleep ( 10 )
    # DEBUG ! ! ! ! infinite wait if we have 50 models
    # if len ( modelIDs ) > = 50:
    # jobCancel = self . _ jobsDAO . jobGetFields ( self . _ jobID , [ ' cancel ' ] ) [ 0]
    # while not jobCancel :
    # time . sleep ( 1)
    # jobCancel = self . _ jobsDAO . jobGetFields ( self . _ jobID , [ ' cancel ' ] ) [ 0]
if self . _errModelRange is not None :
    modelAndCounters = self . _jobsDAO . modelsGetUpdateCounters ( self . _jobID )
    modelIDs = [ x [ 0 ] for x in modelAndCounters ]
    modelIDs . sort ( )
    ( beg , end ) = self . _errModelRange
    if self . _modelID in modelIDs [ int ( beg ) : int ( end ) ] :
        raise RuntimeError ( "Exiting with error due to errModelRange parameter" )
# Delay , if necessary
if self . _delay is not None :
    time . sleep ( self . _delay )
# Run it !
self . _currentRecordIndex = 0
while True : # Check if the model should be stopped
# If killed by a terminator , stop running
    if self . _isKilled :
        break
    # If job stops or hypersearch ends , stop running
    if self . _isCanceled :
        break
    # If model is mature , stop running ONLY IF we are not the best model
    # for the job . Otherwise , keep running so we can keep returning
    # predictions to the user
    if self . _isMature :
        if not self . _isBestModel :
            self . _cmpReason = self . _jobsDAO . CMPL_REASON_STOPPED
            break
        else :
            self . _cmpReason = self . _jobsDAO . CMPL_REASON_EOF
    # Get the the next record , and " write it "
    try :
        self . _currentRecordIndex = next ( iterTracker )
    except StopIteration :
        break
    # " Write " a dummy output value . This is used to test that the batched
    # writing works properly
    self . _writePrediction ( ModelResult ( None , None , None , None ) )
    periodic . tick ( )
    # Compute wait times . See if model should exit
    if self . __shouldSysExit ( self . _currentRecordIndex ) :
        sys . exit ( 1 )
    # Simulate computation time
    if self . _busyWaitTime is not None :
        time . sleep ( self . _busyWaitTime )
        self . __computeWaitTime ( )
    # Asked to abort after so many iterations ?
    if doSysExit :
        sys . exit ( 1 )
    # Asked to raise a jobFailException ?
    if self . _jobFailErr :
        raise utils . JobFailException ( "E10000" , "dummyModel's jobFailErr was True." )
# Handle final operations
if self . _doFinalize :
    if not self . _makeCheckpoint :
        self . _model = None
    # Delay finalization operation
    if self . _finalDelay is not None :
        time . sleep ( self . _finalDelay )
    self . _finalize ( )
self . _logger . info ( "Finished: modelID=%r " % ( self . _modelID ) )
return ( self . _cmpReason , None )


signature:
def _createPredictionLogger ( self ) :
    """Creates the model ' s PredictionLogger object , which is an interface to write
 model results to a permanent storage location"""

implementation:
class DummyLogger :
    def writeRecord ( self , record ) :
        pass
    def writeRecords ( self , records , progressCB ) :
        pass
    def close ( self ) :
        pass
self . _predictionLogger = DummyLogger ( )


signature:
def __shouldSysExit ( self , iteration ) :
    """Checks to see if the model should exit based on the exitAfter dummy
 parameter"""

implementation:
if self . _exitAfter is None or iteration < self . _exitAfter :
    return False
results = self . _jobsDAO . modelsGetFieldsForJob ( self . _jobID , [ 'params' ] )
modelIDs = [ e [ 0 ] for e in results ]
modelNums = [ json . loads ( e [ 1 ] [ 0 ] ) [ 'structuredParams' ] [ '__model_num' ] for e in results ]
sameModelNumbers = filter ( lambda x : x [ 1 ] == self . modelIndex , zip ( modelIDs , modelNums ) )
firstModelID = min ( zip ( * sameModelNumbers ) [ 0 ] )
return firstModelID == self . _modelID


signature:
def getDescription ( self ) :
    """Returns a description of the dataset"""

implementation:
description = { 'name' : self . name , 'fields' : [ f . name for f in self . fields ] , 'numRecords by field' : [ f . numRecords for f in self . fields ] }
return description


signature:
def setSeed ( self , seed ) :
    """Set the random seed and the numpy seed
 Parameters :
 seed : random seed"""

implementation:
rand . seed ( seed )
np . random . seed ( seed )


signature:
def addField ( self , name , fieldParams , encoderParams ) :
    """Add a single field to the dataset .
 Parameters :
 name : The user - specified name of the field
 fieldSpec : A list of one or more dictionaries specifying parameters
 to be used for dataClass initialization . Each dict must
 contain the key ' type ' that specifies a distribution for
 the values in this field
 encoderParams : Parameters for the field encoder"""

implementation:
assert fieldParams is not None and 'type' in fieldParams
dataClassName = fieldParams . pop ( 'type' )
try :
    dataClass = eval ( dataClassName ) ( fieldParams )
except TypeError , e :
    print ( "#### Error in constructing %s class object. Possibly missing " "some required constructor parameters. Parameters " "that were provided are: %s" % ( dataClass , fieldParams ) )
    raise
encoderParams [ 'dataClass' ] = dataClass
encoderParams [ 'dataClassName' ] = dataClassName
fieldIndex = self . defineField ( name , encoderParams )


signature:
def addMultipleFields ( self , fieldsInfo ) :
    """Add multiple fields to the dataset .
 Parameters :
 fieldsInfo : A list of dictionaries , containing a field name , specs for
 the data classes and encoder params for the corresponding
 field ."""

implementation:
assert all ( x in field for x in [ 'name' , 'fieldSpec' , 'encoderParams' ] for field in fieldsInfo )
for spec in fieldsInfo :
    self . addField ( spec . pop ( 'name' ) , spec . pop ( 'fieldSpec' ) , spec . pop ( 'encoderParams' ) )


signature:
def defineField ( self , name , encoderParams = None ) :
    """Initialize field using relevant encoder parameters .
 Parameters :
 name : Field name
 encoderParams : Parameters for the encoder .
 Returns the index of the field"""

implementation:
self . fields . append ( _field ( name , encoderParams ) )
return len ( self . fields ) - 1


signature:
def setFlag ( self , index , flag ) :
    """Set flag for field at index . Flags are special characters such as ' S ' for
 sequence or ' T ' for timestamp .
 Parameters :
 index : index of field whose flag is being set
 flag : special character"""

implementation:
assert len ( self . fields ) > index
self . fields [ index ] . flag = flag


signature:
def generateRecord ( self , record ) :
    """Generate a record . Each value is stored in its respective field .
 Parameters :
 record : A 1 - D array containing as many values as the number of fields
 fields : An object of the class field that specifies the characteristics
 of each value in the record
 Assertion :
 len ( record ) = = len ( fields ) : A value for each field must be specified .
 Replace missing values of any type by
 SENTINEL _ VALUE _ FOR _ MISSING _ DATA
 This method supports external classes but not combination of classes ."""

implementation:
assert ( len ( record ) == len ( self . fields ) )
if record is not None :
    for x in range ( len ( self . fields ) ) :
        self . fields [ x ] . addValue ( record [ x ] )
else :
    for field in self . fields :
        field . addValue ( field . dataClass . getNext ( ) )


signature:
def generateRecords ( self , records ) :
    """Generate multiple records . Refer to definition for generateRecord"""

implementation:
if self . verbosity > 0 :
    print 'Generating' , len ( records ) , 'records...'
for record in records :
    self . generateRecord ( record )


signature:
def getRecord ( self , n = None ) :
    """Returns the nth record"""

implementation:
if n is None :
    assert len ( self . fields ) > 0
    n = self . fields [ 0 ] . numRecords - 1
assert ( all ( field . numRecords > n for field in self . fields ) )
record = [ field . values [ n ] for field in self . fields ]
return record


signature:
def getAllRecords ( self ) :
    """Returns all the records"""

implementation:
values = [ ]
numRecords = self . fields [ 0 ] . numRecords
assert ( all ( field . numRecords == numRecords for field in self . fields ) )
for x in range ( numRecords ) :
    values . append ( self . getRecord ( x ) )
return values


signature:
def encodeRecord ( self , record , toBeAdded = True ) :
    """Encode a record as a sparse distributed representation
 Parameters :
 record : Record to be encoded
 toBeAdded : Whether the encodings corresponding to the record are added to
 the corresponding fields"""

implementation:
encoding = [ self . fields [ i ] . encodeValue ( record [ i ] , toBeAdded ) for i in xrange ( len ( self . fields ) ) ]
return encoding


signature:
def encodeAllRecords ( self , records = None , toBeAdded = True ) :
    """Encodes a list of records .
 Parameters :
 records : One or more records . ( i , j ) th element of this 2D array
 specifies the value at field j of record i .
 If unspecified , records previously generated and stored are
 used .
 toBeAdded : Whether the encodings corresponding to the record are added to
 the corresponding fields"""

implementation:
if records is None :
    records = self . getAllRecords ( )
if self . verbosity > 0 :
    print 'Encoding' , len ( records ) , 'records.'
encodings = [ self . encodeRecord ( record , toBeAdded ) for record in records ]
return encodings


signature:
def addValueToField ( self , i , value = None ) :
    """Add ' value ' to the field i .
 Parameters :
 value : value to be added
 i : value is added to field i"""

implementation:
assert ( len ( self . fields ) > i )
if value is None :
    value = self . fields [ i ] . dataClass . getNext ( )
    self . fields [ i ] . addValue ( value )
    return value
else :
    self . fields [ i ] . addValue ( value )


signature:
def addValuesToField ( self , i , numValues ) :
    """Add values to the field i ."""

implementation:
assert ( len ( self . fields ) > i )
values = [ self . addValueToField ( i ) for n in range ( numValues ) ]
return values


signature:
def getSDRforValue ( self , i , j ) :
    """Returns the sdr for jth value at column i"""

implementation:
assert len ( self . fields ) > i
assert self . fields [ i ] . numRecords > j
encoding = self . fields [ i ] . encodings [ j ]
return encoding


signature:
def getZeroedOutEncoding ( self , n ) :
    """Returns the nth encoding with the predictedField zeroed out"""

implementation:
assert all ( field . numRecords > n for field in self . fields )
encoding = np . concatenate ( [ field . encoder . encode ( SENTINEL_VALUE_FOR_MISSING_DATA ) if field . isPredictedField else field . encodings [ n ] for field in self . fields ] )
return encoding


signature:
def getTotaln ( self ) :
    """Returns the cumulative n for all the fields in the dataset"""

implementation:
n = sum ( [ field . n for field in self . fields ] )
return n


signature:
def getTotalw ( self ) :
    """Returns the cumulative w for all the fields in the dataset"""

implementation:
w = sum ( [ field . w for field in self . fields ] )
return w


signature:
def getEncoding ( self , n ) :
    """Returns the nth encoding"""

implementation:
assert ( all ( field . numEncodings > n for field in self . fields ) )
encoding = np . concatenate ( [ field . encodings [ n ] for field in self . fields ] )
return encoding


signature:
def getAllEncodings ( self ) :
    """Returns encodings for all the records"""

implementation:
numEncodings = self . fields [ 0 ] . numEncodings
assert ( all ( field . numEncodings == numEncodings for field in self . fields ) )
encodings = [ self . getEncoding ( index ) for index in range ( numEncodings ) ]
return encodings


signature:
def saveRecords ( self , path = 'myOutput' ) :
    """Export all the records into a csv file in numenta format .
 Example header format :
 fieldName1 fieldName2 fieldName3
 date string float
 T S
 Parameters :
 path : Relative path of the file to which the records are to be exported"""

implementation:
numRecords = self . fields [ 0 ] . numRecords
assert ( all ( field . numRecords == numRecords for field in self . fields ) )
import csv
with open ( path + '.csv' , 'wb' ) as f :
    writer = csv . writer ( f )
    writer . writerow ( self . getAllFieldNames ( ) )
    writer . writerow ( self . getAllDataTypes ( ) )
    writer . writerow ( self . getAllFlags ( ) )
    writer . writerows ( self . getAllRecords ( ) )
if self . verbosity > 0 :
    print '******' , numRecords , 'records exported in numenta format to file:' , path , '******\n'


signature:
def removeAllRecords ( self ) :
    """Deletes all the values in the dataset"""

implementation:
for field in self . fields :
    field . encodings , field . values = [ ] , [ ]
    field . numRecords , field . numEncodings = ( 0 , 0 )


signature:
def encodeValue ( self , value , toBeAdded = True ) :
    """Value is encoded as a sdr using the encoding parameters of the Field"""

implementation:
encodedValue = np . array ( self . encoder . encode ( value ) , dtype = realDType )
if toBeAdded :
    self . encodings . append ( encodedValue )
    self . numEncodings += 1
return encodedValue


signature:
def _setTypes ( self , encoderSpec ) :
    """Set up the dataTypes and initialize encoders"""

implementation:
if self . encoderType is None :
    if self . dataType in [ 'int' , 'float' ] :
        self . encoderType = 'adaptiveScalar'
    elif self . dataType == 'string' :
        self . encoderType = 'category'
    elif self . dataType in [ 'date' , 'datetime' ] :
        self . encoderType = 'date'
if self . dataType is None :
    if self . encoderType in [ 'scalar' , 'adaptiveScalar' ] :
        self . dataType = 'float'
    elif self . encoderType in [ 'category' , 'enumeration' ] :
        self . dataType = 'string'
    elif self . encoderType in [ 'date' , 'datetime' ] :
        self . dataType = 'datetime'


signature:
def _initializeEncoders ( self , encoderSpec ) :
    """Initialize the encoders"""

implementation:
# Initializing scalar encoder
if self . encoderType in [ 'adaptiveScalar' , 'scalar' ] :
    if 'minval' in encoderSpec :
        self . minval = encoderSpec . pop ( 'minval' )
    else :
        self . minval = None
    if 'maxval' in encoderSpec :
        self . maxval = encoderSpec . pop ( 'maxval' )
    else :
        self . maxval = None
    self . encoder = adaptive_scalar . AdaptiveScalarEncoder ( name = 'AdaptiveScalarEncoder' , w = self . w , n = self . n , minval = self . minval , maxval = self . maxval , periodic = False , forced = True )
# Initializing category encoder
elif self . encoderType == 'category' :
    self . encoder = sdr_category . SDRCategoryEncoder ( name = 'categoryEncoder' , w = self . w , n = self . n )
# Initializing date encoder
elif self . encoderType in [ 'date' , 'datetime' ] :
    self . encoder = date . DateEncoder ( name = 'dateEncoder' )
else :
    raise RuntimeError ( 'Error in constructing class object. Either encoder type' 'or dataType must be specified' )


signature:
def getScalars ( self , input ) :
    """See method description in base . py"""

implementation:
if input == SENTINEL_VALUE_FOR_MISSING_DATA :
    return numpy . array ( [ None ] )
else :
    return numpy . array ( [ self . categoryToIndex . get ( input , 0 ) ] )


signature:
def getBucketIndices ( self , input ) :
    """See method description in base . py"""

implementation:
# Get the bucket index from the underlying scalar encoder
if input == SENTINEL_VALUE_FOR_MISSING_DATA :
    return [ None ]
else :
    return self . encoder . getBucketIndices ( self . categoryToIndex . get ( input , 0 ) )


signature:
def decode ( self , encoded , parentFieldName = '' ) :
    """See the function description in base . py"""

implementation:
# Get the scalar values from the underlying scalar encoder
( fieldsDict , fieldNames ) = self . encoder . decode ( encoded )
if len ( fieldsDict ) == 0 :
    return ( fieldsDict , fieldNames )
# Expect only 1 field
assert ( len ( fieldsDict ) == 1 )
# Get the list of categories the scalar values correspond to and
# generate the description from the category name ( s ) .
( inRanges , inDesc ) = fieldsDict . values ( ) [ 0 ]
outRanges = [ ]
desc = ""
for ( minV , maxV ) in inRanges :
    minV = int ( round ( minV ) )
    maxV = int ( round ( maxV ) )
    outRanges . append ( ( minV , maxV ) )
    while minV <= maxV :
        if len ( desc ) > 0 :
            desc += ", "
        desc += self . indexToCategory [ minV ]
        minV += 1
# Return result
if parentFieldName != '' :
    fieldName = "%s.%s" % ( parentFieldName , self . name )
else :
    fieldName = self . name
return ( { fieldName : ( outRanges , desc ) } , [ fieldName ] )


signature:
def closenessScores ( self , expValues , actValues , fractional = True , ) :
    """See the function description in base . py
 kwargs will have the keyword " fractional " , which is ignored by this encoder"""

implementation:
expValue = expValues [ 0 ]
actValue = actValues [ 0 ]
if expValue == actValue :
    closeness = 1.0
else :
    closeness = 0.0
if not fractional :
    closeness = 1.0 - closeness
return numpy . array ( [ closeness ] )


signature:
def getBucketValues ( self ) :
    """See the function description in base . py"""

implementation:
if self . _bucketValues is None :
    numBuckets = len ( self . encoder . getBucketValues ( ) )
    self . _bucketValues = [ ]
    for bucketIndex in range ( numBuckets ) :
        self . _bucketValues . append ( self . getBucketInfo ( [ bucketIndex ] ) [ 0 ] . value )
return self . _bucketValues


signature:
def getBucketInfo ( self , buckets ) :
    """See the function description in base . py"""

implementation:
# For the category encoder , the bucket index is the category index
bucketInfo = self . encoder . getBucketInfo ( buckets ) [ 0 ]
categoryIndex = int ( round ( bucketInfo . value ) )
category = self . indexToCategory [ categoryIndex ]
return [ EncoderResult ( value = category , scalar = categoryIndex , encoding = bucketInfo . encoding ) ]


signature:
def topDownCompute ( self , encoded ) :
    """See the function description in base . py"""

implementation:
encoderResult = self . encoder . topDownCompute ( encoded ) [ 0 ]
value = encoderResult . value
categoryIndex = int ( round ( value ) )
category = self . indexToCategory [ categoryIndex ]
return EncoderResult ( value = category , scalar = categoryIndex , encoding = encoderResult . encoding )


signature:
def loadExperiment ( path ) :
    """Loads the experiment description file from the path .
 : param path : ( string ) The path to a directory containing a description . py file
 or the file itself .
 : returns : ( config , control )"""

implementation:
if not os . path . isdir ( path ) :
    path = os . path . dirname ( path )
descriptionPyModule = loadExperimentDescriptionScriptFromDir ( path )
expIface = getExperimentDescriptionInterfaceFromModule ( descriptionPyModule )
return expIface . getModelDescription ( ) , expIface . getModelControl ( )


signature:
def loadExperimentDescriptionScriptFromDir ( experimentDir ) :
    """Loads the experiment description python script from the given experiment
 directory .
 : param experimentDir : ( string ) experiment directory path
 : returns : module of the loaded experiment description scripts"""

implementation:
descriptionScriptPath = os . path . join ( experimentDir , "description.py" )
module = _loadDescriptionFile ( descriptionScriptPath )
return module


signature:
def getExperimentDescriptionInterfaceFromModule ( module ) :
    """: param module : imported description . py module
 : returns : ( : class : ` nupic . frameworks . opf . exp _ description _ api . DescriptionIface ` )
 represents the experiment description"""

implementation:
result = module . descriptionInterface
assert isinstance ( result , exp_description_api . DescriptionIface ) , "expected DescriptionIface-based instance, but got %s" % type ( result )
return result


signature:
def _loadDescriptionFile ( descriptionPyPath ) :
    """Loads a description file and returns it as a module .
 descriptionPyPath : path of description . py file to load"""

implementation:
global g_descriptionImportCount
if not os . path . isfile ( descriptionPyPath ) :
    raise RuntimeError ( ( "Experiment description file %s does not exist or " + "is not a file" ) % ( descriptionPyPath , ) )
mod = imp . load_source ( "pf_description%d" % g_descriptionImportCount , descriptionPyPath )
g_descriptionImportCount += 1
if not hasattr ( mod , "descriptionInterface" ) :
    raise RuntimeError ( "Experiment description file %s does not define %s" % ( descriptionPyPath , "descriptionInterface" ) )
if not isinstance ( mod . descriptionInterface , exp_description_api . DescriptionIface ) :
    raise RuntimeError ( ( "Experiment description file %s defines %s but it " + "is not DescriptionIface-based" ) % ( descriptionPyPath , name ) )
return mod


signature:
def update ( self , modelID , modelParams , modelParamsHash , metricResult , completed , completionReason , matured , numRecords ) :
    """Insert a new entry or update an existing one . If this is an update
 of an existing entry , then modelParams will be None
 Parameters :
 modelID : globally unique modelID of this model
 modelParams : params dict for this model , or None if this is just an update
 of a model that it already previously reported on .
 See the comments for the createModels ( ) method for
 a description of this dict .
 modelParamsHash : hash of the modelParams dict , generated by the worker
 that put it into the model database .
 metricResult : value on the optimizeMetric for this model .
 May be None if we have no results yet .
 completed : True if the model has completed evaluation , False if it
 is still running ( and these are online results )
 completionReason : One of the ClientJobsDAO . CMPL _ REASON _ XXX equates
 matured : True if this model has matured
 numRecords : Number of records that have been processed so far by this
 model .
 retval : Canonicalized result on the optimize metric"""

implementation:
# The modelParamsHash must always be provided - it can change after a
# model is inserted into the models table if it got detected as an
# orphan
assert ( modelParamsHash is not None )
# We consider a model metricResult as " final " if it has completed or
# matured . By default , assume anything that has completed has matured
if completed :
    matured = True
# Get the canonicalized optimize metric results . For this metric , lower
# is always better
if metricResult is not None and matured and completionReason in [ ClientJobsDAO . CMPL_REASON_EOF , ClientJobsDAO . CMPL_REASON_STOPPED ] : # Canonicalize the error score so that lower is better
    if self . _hsObj . _maximize :
        errScore = - 1 * metricResult
    else :
        errScore = metricResult
    if errScore < self . _bestResult :
        self . _bestResult = errScore
        self . _bestModelID = modelID
        self . _hsObj . logger . info ( "New best model after %d evaluations: errScore " "%g on model %s" % ( len ( self . _allResults ) , self . _bestResult , self . _bestModelID ) )
else :
    errScore = numpy . inf
# If this model completed with an unacceptable completion reason , set the
# errScore to infinite and essentially make this model invisible to
# further queries
if completed and completionReason in [ ClientJobsDAO . CMPL_REASON_ORPHAN ] :
    errScore = numpy . inf
    hidden = True
else :
    hidden = False
# Update our set of erred models and completed models . These are used
# to determine if we should abort the search because of too many errors
if completed :
    self . _completedModels . add ( modelID )
    self . _numCompletedModels = len ( self . _completedModels )
    if completionReason == ClientJobsDAO . CMPL_REASON_ERROR :
        self . _errModels . add ( modelID )
        self . _numErrModels = len ( self . _errModels )
# Are we creating a new entry ?
wasHidden = False
if modelID not in self . _modelIDToIdx :
    assert ( modelParams is not None )
    entry = dict ( modelID = modelID , modelParams = modelParams , modelParamsHash = modelParamsHash , errScore = errScore , completed = completed , matured = matured , numRecords = numRecords , hidden = hidden )
    self . _allResults . append ( entry )
    entryIdx = len ( self . _allResults ) - 1
    self . _modelIDToIdx [ modelID ] = entryIdx
    self . _paramsHashToIndexes [ modelParamsHash ] = entryIdx
    swarmId = modelParams [ 'particleState' ] [ 'swarmId' ]
    if not hidden : # Update the list of particles in each swarm
        if swarmId in self . _swarmIdToIndexes :
            self . _swarmIdToIndexes [ swarmId ] . append ( entryIdx )
        else :
            self . _swarmIdToIndexes [ swarmId ] = [ entryIdx ]
        # Update number of particles at each generation in this swarm
        genIdx = modelParams [ 'particleState' ] [ 'genIdx' ]
        numPsEntry = self . _swarmNumParticlesPerGeneration . get ( swarmId , [ 0 ] )
        while genIdx >= len ( numPsEntry ) :
            numPsEntry . append ( 0 )
        numPsEntry [ genIdx ] += 1
        self . _swarmNumParticlesPerGeneration [ swarmId ] = numPsEntry
# Replacing an existing one
else :
    entryIdx = self . _modelIDToIdx . get ( modelID , None )
    assert ( entryIdx is not None )
    entry = self . _allResults [ entryIdx ]
    wasHidden = entry [ 'hidden' ]
    # If the paramsHash changed , note that . This can happen for orphaned
    # models
    if entry [ 'modelParamsHash' ] != modelParamsHash :
        self . _paramsHashToIndexes . pop ( entry [ 'modelParamsHash' ] )
        self . _paramsHashToIndexes [ modelParamsHash ] = entryIdx
        entry [ 'modelParamsHash' ] = modelParamsHash
    # Get the model params , swarmId , and genIdx
    modelParams = entry [ 'modelParams' ]
    swarmId = modelParams [ 'particleState' ] [ 'swarmId' ]
    genIdx = modelParams [ 'particleState' ] [ 'genIdx' ]
    # If this particle just became hidden , remove it from our swarm counts
    if hidden and not wasHidden :
        assert ( entryIdx in self . _swarmIdToIndexes [ swarmId ] )
        self . _swarmIdToIndexes [ swarmId ] . remove ( entryIdx )
        self . _swarmNumParticlesPerGeneration [ swarmId ] [ genIdx ] -= 1
    # Update the entry for the latest info
    entry [ 'errScore' ] = errScore
    entry [ 'completed' ] = completed
    entry [ 'matured' ] = matured
    entry [ 'numRecords' ] = numRecords
    entry [ 'hidden' ] = hidden
# Update the particle best errScore
particleId = modelParams [ 'particleState' ] [ 'id' ]
genIdx = modelParams [ 'particleState' ] [ 'genIdx' ]
if matured and not hidden :
    ( oldResult , pos ) = self . _particleBest . get ( particleId , ( numpy . inf , None ) )
    if errScore < oldResult :
        pos = Particle . getPositionFromState ( modelParams [ 'particleState' ] )
        self . _particleBest [ particleId ] = ( errScore , pos )
# Update the particle latest generation index
prevGenIdx = self . _particleLatestGenIdx . get ( particleId , - 1 )
if not hidden and genIdx > prevGenIdx :
    self . _particleLatestGenIdx [ particleId ] = genIdx
elif hidden and not wasHidden and genIdx == prevGenIdx :
    self . _particleLatestGenIdx [ particleId ] = genIdx - 1
# Update the swarm best score
if not hidden :
    swarmId = modelParams [ 'particleState' ] [ 'swarmId' ]
    if not swarmId in self . _swarmBestOverall :
        self . _swarmBestOverall [ swarmId ] = [ ]
    bestScores = self . _swarmBestOverall [ swarmId ]
    while genIdx >= len ( bestScores ) :
        bestScores . append ( ( None , numpy . inf ) )
    if errScore < bestScores [ genIdx ] [ 1 ] :
        bestScores [ genIdx ] = ( modelID , errScore )
# Update the self . _ modifiedSwarmGens flags to support the
# getMaturedSwarmGenerations ( ) call .
if not hidden :
    key = ( swarmId , genIdx )
    if not key in self . _maturedSwarmGens :
        self . _modifiedSwarmGens . add ( key )
return errScore


signature:
def getModelIDFromParamsHash ( self , paramsHash ) :
    """Return the modelID of the model with the given paramsHash , or
 None if not found .
 Parameters :
 paramsHash : paramsHash to look for
 retval : modelId , or None if not found"""

implementation:
entryIdx = self . _paramsHashToIndexes . get ( paramsHash , None )
if entryIdx is not None :
    return self . _allResults [ entryIdx ] [ 'modelID' ]
else :
    return None


signature:
def numModels ( self , swarmId = None , includeHidden = False ) :
    """Return the total # of models we have in our database ( if swarmId is
 None ) or in a specific swarm .
 Parameters :
 swarmId : A string representation of the sorted list of encoders
 in this swarm . For example ' _ _ address _ encoder . _ _ gym _ encoder '
 includeHidden : If False , this will only return the number of models
 that are not hidden ( i . e . orphanned , etc . )
 retval : numModels"""

implementation:
# Count all models
if includeHidden :
    if swarmId is None :
        return len ( self . _allResults )
    else :
        return len ( self . _swarmIdToIndexes . get ( swarmId , [ ] ) )
# Only count non - hidden models
else :
    if swarmId is None :
        entries = self . _allResults
    else :
        entries = [ self . _allResults [ entryIdx ] for entryIdx in self . _swarmIdToIndexes . get ( swarmId , [ ] ) ]
    return len ( [ entry for entry in entries if not entry [ 'hidden' ] ] )


signature:
def bestModelIdAndErrScore ( self , swarmId = None , genIdx = None ) :
    """Return the model ID of the model with the best result so far and
 it ' s score on the optimize metric . If swarm is None , then it returns
 the global best , otherwise it returns the best for the given swarm
 for all generatons up to and including genIdx .
 Parameters :
 swarmId : A string representation of the sorted list of encoders in this
 swarm . For example ' _ _ address _ encoder . _ _ gym _ encoder '
 genIdx : consider the best in all generations up to and including this
 generation if not None .
 retval : ( modelID , result )"""

implementation:
if swarmId is None :
    return ( self . _bestModelID , self . _bestResult )
else :
    if swarmId not in self . _swarmBestOverall :
        return ( None , numpy . inf )
    # Get the best score , considering the appropriate generations
    genScores = self . _swarmBestOverall [ swarmId ]
    bestModelId = None
    bestScore = numpy . inf
    for ( i , ( modelId , errScore ) ) in enumerate ( genScores ) :
        if genIdx is not None and i > genIdx :
            break
        if errScore < bestScore :
            bestScore = errScore
            bestModelId = modelId
    return ( bestModelId , bestScore )


signature:
def getParticleInfo ( self , modelId ) :
    """Return particle info for a specific modelId .
 Parameters :
 modelId : which model Id
 retval : ( particleState , modelId , errScore , completed , matured )"""

implementation:
entry = self . _allResults [ self . _modelIDToIdx [ modelId ] ]
return ( entry [ 'modelParams' ] [ 'particleState' ] , modelId , entry [ 'errScore' ] , entry [ 'completed' ] , entry [ 'matured' ] )


signature:
def getParticleInfos ( self , swarmId = None , genIdx = None , completed = None , matured = None , lastDescendent = False ) :
    """Return a list of particleStates for all particles we know about in
 the given swarm , their model Ids , and metric results .
 Parameters :
 swarmId : A string representation of the sorted list of encoders in this
 swarm . For example ' _ _ address _ encoder . _ _ gym _ encoder '
 genIdx : If not None , only return particles at this specific generation
 index .
 completed : If not None , only return particles of the given state ( either
 completed if ' completed ' is True , or running if ' completed '
 is false
 matured : If not None , only return particles of the given state ( either
 matured if ' matured ' is True , or not matured if ' matured '
 is false . Note that any model which has completed is also
 considered matured .
 lastDescendent : If True , only return particles that are the last descendent ,
 that is , the highest generation index for a given particle Id
 retval : ( particleStates , modelIds , errScores , completed , matured )
 particleStates : list of particleStates
 modelIds : list of modelIds
 errScores : list of errScores , numpy . inf is plugged in
 if we don ' t have a result yet
 completed : list of completed booleans
 matured : list of matured booleans"""

implementation:
# The indexes of all the models in this swarm . This list excludes hidden
# ( orphaned ) models .
if swarmId is not None :
    entryIdxs = self . _swarmIdToIndexes . get ( swarmId , [ ] )
else :
    entryIdxs = range ( len ( self . _allResults ) )
if len ( entryIdxs ) == 0 :
    return ( [ ] , [ ] , [ ] , [ ] , [ ] )
# Get the particles of interest
particleStates = [ ]
modelIds = [ ]
errScores = [ ]
completedFlags = [ ]
maturedFlags = [ ]
for idx in entryIdxs :
    entry = self . _allResults [ idx ]
    # If this entry is hidden ( i . e . it was an orphaned model ) , it should
    # not be in this list
    if swarmId is not None :
        assert ( not entry [ 'hidden' ] )
    # Get info on this model
    modelParams = entry [ 'modelParams' ]
    isCompleted = entry [ 'completed' ]
    isMatured = entry [ 'matured' ]
    particleState = modelParams [ 'particleState' ]
    particleGenIdx = particleState [ 'genIdx' ]
    particleId = particleState [ 'id' ]
    if genIdx is not None and particleGenIdx != genIdx :
        continue
    if completed is not None and ( completed != isCompleted ) :
        continue
    if matured is not None and ( matured != isMatured ) :
        continue
    if lastDescendent and ( self . _particleLatestGenIdx [ particleId ] != particleGenIdx ) :
        continue
    # Incorporate into return values
    particleStates . append ( particleState )
    modelIds . append ( entry [ 'modelID' ] )
    errScores . append ( entry [ 'errScore' ] )
    completedFlags . append ( isCompleted )
    maturedFlags . append ( isMatured )
return ( particleStates , modelIds , errScores , completedFlags , maturedFlags )


signature:
def getOrphanParticleInfos ( self , swarmId , genIdx ) :
    """Return a list of particleStates for all particles in the given
 swarm generation that have been orphaned .
 Parameters :
 swarmId : A string representation of the sorted list of encoders in this
 swarm . For example ' _ _ address _ encoder . _ _ gym _ encoder '
 genIdx : If not None , only return particles at this specific generation
 index .
 retval : ( particleStates , modelIds , errScores , completed , matured )
 particleStates : list of particleStates
 modelIds : list of modelIds
 errScores : list of errScores , numpy . inf is plugged in
 if we don ' t have a result yet
 completed : list of completed booleans
 matured : list of matured booleans"""

implementation:
entryIdxs = range ( len ( self . _allResults ) )
if len ( entryIdxs ) == 0 :
    return ( [ ] , [ ] , [ ] , [ ] , [ ] )
# Get the particles of interest
particleStates = [ ]
modelIds = [ ]
errScores = [ ]
completedFlags = [ ]
maturedFlags = [ ]
for idx in entryIdxs : # Get info on this model
    entry = self . _allResults [ idx ]
    if not entry [ 'hidden' ] :
        continue
    modelParams = entry [ 'modelParams' ]
    if modelParams [ 'particleState' ] [ 'swarmId' ] != swarmId :
        continue
    isCompleted = entry [ 'completed' ]
    isMatured = entry [ 'matured' ]
    particleState = modelParams [ 'particleState' ]
    particleGenIdx = particleState [ 'genIdx' ]
    particleId = particleState [ 'id' ]
    if genIdx is not None and particleGenIdx != genIdx :
        continue
    # Incorporate into return values
    particleStates . append ( particleState )
    modelIds . append ( entry [ 'modelID' ] )
    errScores . append ( entry [ 'errScore' ] )
    completedFlags . append ( isCompleted )
    maturedFlags . append ( isMatured )
return ( particleStates , modelIds , errScores , completedFlags , maturedFlags )


signature:
def getMaturedSwarmGenerations ( self ) :
    """Return a list of swarm generations that have completed and the
 best ( minimal ) errScore seen for each of them .
 Parameters :
 retval : list of tuples . Each tuple is of the form :
 ( swarmId , genIdx , bestErrScore )"""

implementation:
# Return results go in this list
result = [ ]
# For each of the swarm generations which have had model result updates
# since the last time we were called , see which have completed .
modifiedSwarmGens = sorted ( self . _modifiedSwarmGens )
# Walk through them in order from lowest to highest generation index
for key in modifiedSwarmGens :
    ( swarmId , genIdx ) = key
    # Skip it if we ' ve already reported on it . This should happen rarely , if
    # ever . It means that some worker has started and completed a model in
    # this generation after we ' ve determined that the generation has ended .
    if key in self . _maturedSwarmGens :
        self . _modifiedSwarmGens . remove ( key )
        continue
    # If the previous generation for this swarm is not complete yet , don ' t
    # bother evaluating this one .
    if ( genIdx >= 1 ) and not ( swarmId , genIdx - 1 ) in self . _maturedSwarmGens :
        continue
    # We found a swarm generation that had some results reported since last
    # time , see if it ' s complete or not
    ( _ , _ , errScores , completedFlags , maturedFlags ) = self . getParticleInfos ( swarmId , genIdx )
    maturedFlags = numpy . array ( maturedFlags )
    numMatured = maturedFlags . sum ( )
    if numMatured >= self . _hsObj . _minParticlesPerSwarm and numMatured == len ( maturedFlags ) :
        errScores = numpy . array ( errScores )
        bestScore = errScores . min ( )
        self . _maturedSwarmGens . add ( key )
        self . _modifiedSwarmGens . remove ( key )
        result . append ( ( swarmId , genIdx , bestScore ) )
# Return results
return result


signature:
def firstNonFullGeneration ( self , swarmId , minNumParticles ) :
    """Return the generation index of the first generation in the given
 swarm that does not have numParticles particles in it , either still in the
 running state or completed . This does not include orphaned particles .
 Parameters :
 swarmId : A string representation of the sorted list of encoders in this
 swarm . For example ' _ _ address _ encoder . _ _ gym _ encoder '
 minNumParticles : minium number of partices required for a full
 generation .
 retval : generation index , or None if no particles at all ."""

implementation:
if not swarmId in self . _swarmNumParticlesPerGeneration :
    return None
numPsPerGen = self . _swarmNumParticlesPerGeneration [ swarmId ]
numPsPerGen = numpy . array ( numPsPerGen )
firstNonFull = numpy . where ( numPsPerGen < minNumParticles ) [ 0 ]
if len ( firstNonFull ) == 0 :
    return len ( numPsPerGen )
else :
    return firstNonFull [ 0 ]


signature:
def getResultsPerChoice ( self , swarmId , maxGenIdx , varName ) :
    """Return a dict of the errors obtained on models that were run with
 each value from a PermuteChoice variable .
 For example , if a PermuteChoice variable has the following choices :
 [ ' a ' , ' b ' , ' c ' ]
 The dict will have 3 elements . The keys are the stringified choiceVars ,
 and each value is tuple containing ( choiceVar , errors ) where choiceVar is
 the original form of the choiceVar ( before stringification ) and errors is
 the list of errors received from models that used the specific choice :
 retval :
 [ ' a ' : ( ' a ' , [ 0.1 , 0.2 , 0.3 ] ) , ' b ' : ( ' b ' , [ 0.5 , 0.1 , 0.6 ] ) , ' c ' : ( ' c ' , [ ] ) ]
 Parameters :
 swarmId : swarm Id of the swarm to retrieve info from
 maxGenIdx : max generation index to consider from other models , ignored
 if None
 varName : which variable to retrieve
 retval : list of the errors obtained from each choice ."""

implementation:
results = dict ( )
# Get all the completed particles in this swarm
( allParticles , _ , resultErrs , _ , _ ) = self . getParticleInfos ( swarmId , genIdx = None , matured = True )
for particleState , resultErr in itertools . izip ( allParticles , resultErrs ) : # Consider this generation ?
    if maxGenIdx is not None :
        if particleState [ 'genIdx' ] > maxGenIdx :
            continue
    # Ignore unless this model completed successfully
    if resultErr == numpy . inf :
        continue
    position = Particle . getPositionFromState ( particleState )
    varPosition = position [ varName ]
    varPositionStr = str ( varPosition )
    if varPositionStr in results :
        results [ varPositionStr ] [ 1 ] . append ( resultErr )
    else :
        results [ varPositionStr ] = ( varPosition , [ resultErr ] )
return results


signature:
def _getStreamDef ( self , modelDescription ) :
    """Generate stream definition based on"""

implementation:
# Generate the string containing the aggregation settings .
aggregationPeriod = { 'days' : 0 , 'hours' : 0 , 'microseconds' : 0 , 'milliseconds' : 0 , 'minutes' : 0 , 'months' : 0 , 'seconds' : 0 , 'weeks' : 0 , 'years' : 0 , }
# Honor any overrides provided in the stream definition
aggFunctionsDict = { }
if 'aggregation' in modelDescription [ 'streamDef' ] :
    for key in aggregationPeriod . keys ( ) :
        if key in modelDescription [ 'streamDef' ] [ 'aggregation' ] :
            aggregationPeriod [ key ] = modelDescription [ 'streamDef' ] [ 'aggregation' ] [ key ]
    if 'fields' in modelDescription [ 'streamDef' ] [ 'aggregation' ] :
        for ( fieldName , func ) in modelDescription [ 'streamDef' ] [ 'aggregation' ] [ 'fields' ] :
            aggFunctionsDict [ fieldName ] = str ( func )
# Do we have any aggregation at all ?
hasAggregation = False
for v in aggregationPeriod . values ( ) :
    if v != 0 :
        hasAggregation = True
        break
# Convert the aggFunctionsDict to a list
aggFunctionList = aggFunctionsDict . items ( )
aggregationInfo = dict ( aggregationPeriod )
aggregationInfo [ 'fields' ] = aggFunctionList
streamDef = copy . deepcopy ( modelDescription [ 'streamDef' ] )
streamDef [ 'aggregation' ] = copy . deepcopy ( aggregationInfo )
return streamDef


signature:
def close ( self ) :
    """Deletes temporary system objects / files ."""

implementation:
if self . _tempDir is not None and os . path . isdir ( self . _tempDir ) :
    self . logger . debug ( "Removing temporary directory %r" , self . _tempDir )
    shutil . rmtree ( self . _tempDir )
    self . _tempDir = None
return


signature:
def _readPermutationsFile ( self , filename , modelDescription ) :
    """Read the permutations file and initialize the following member variables :
 _ predictedField : field name of the field we are trying to
 predict
 _ permutations : Dict containing the full permutations dictionary .
 _ flattenedPermutations : Dict containing the flattened version of
 _ permutations . The keys leading to the value in the dict are joined
 with a period to create the new key and permute variables within
 encoders are pulled out of the encoder .
 _ encoderNames : keys from self . _ permutations of only the encoder
 variables .
 _ reportKeys : The ' report ' list from the permutations file .
 This is a list of the items from each experiment ' s pickled
 results file that should be included in the final report . The
 format of each item is a string of key names separated by colons ,
 each key being one level deeper into the experiment results
 dict . For example , ' key1 : key2 ' .
 _ filterFunc : a user - supplied function that can be used to
 filter out specific permutation combinations .
 _ optimizeKey : which report key to optimize for
 _ maximize : True if we should try and maximize the optimizeKey
 metric . False if we should minimize it .
 _ dummyModelParamsFunc : a user - supplied function that can be used to
 artificially generate HTMPredictionModel results . When supplied ,
 the model is not actually run through the OPF , but instead is run
 through a " Dummy Model " ( nupic . swarming . ModelRunner .
 OPFDummyModelRunner ) . This function returns the params dict used
 to control various options in the dummy model ( the returned metric ,
 the execution time , etc . ) . This is used for hypersearch algorithm
 development .
 Parameters :
 filename : Name of permutations file
 retval : None"""

implementation:
# Open and execute the permutations file
vars = { }
permFile = execfile ( filename , globals ( ) , vars )
# Read in misc info .
self . _reportKeys = vars . get ( 'report' , [ ] )
self . _filterFunc = vars . get ( 'permutationFilter' , None )
self . _dummyModelParamsFunc = vars . get ( 'dummyModelParams' , None )
self . _predictedField = None
# default
self . _predictedFieldEncoder = None
# default
self . _fixedFields = None
# default
# The fastSwarm variable , if present , contains the params from a best
# model from a previous swarm . If present , use info from that to seed
# a fast swarm
self . _fastSwarmModelParams = vars . get ( 'fastSwarmModelParams' , None )
if self . _fastSwarmModelParams is not None :
    encoders = self . _fastSwarmModelParams [ 'structuredParams' ] [ 'modelParams' ] [ 'sensorParams' ] [ 'encoders' ]
    self . _fixedFields = [ ]
    for fieldName in encoders :
        if encoders [ fieldName ] is not None :
            self . _fixedFields . append ( fieldName )
if 'fixedFields' in vars :
    self . _fixedFields = vars [ 'fixedFields' ]
# Get min number of particles per swarm from either permutations file or
# config .
self . _minParticlesPerSwarm = vars . get ( 'minParticlesPerSwarm' )
if self . _minParticlesPerSwarm == None :
    self . _minParticlesPerSwarm = Configuration . get ( 'nupic.hypersearch.minParticlesPerSwarm' )
self . _minParticlesPerSwarm = int ( self . _minParticlesPerSwarm )
# Enable logic to kill off speculative swarms when an earlier sprint
# has found that it contains poorly performing field combination ?
self . _killUselessSwarms = vars . get ( 'killUselessSwarms' , True )
# The caller can request that the predicted field ALWAYS be included ( " yes " )
# or optionally include ( " auto " ) . The setting of " no " is N / A and ignored
# because in that case the encoder for the predicted field will not even
# be present in the permutations file .
# When set to " yes " , this will force the first sprint to try the predicted
# field only ( the legacy mode of swarming ) .
# When set to " auto " , the first sprint tries all possible fields ( one at a
# time ) in the first sprint .
self . _inputPredictedField = vars . get ( "inputPredictedField" , "yes" )
# Try all possible 3 - field combinations ? Normally , we start with the best
# 2 - field combination as a base . When this flag is set though , we try
# all possible 3 - field combinations which takes longer but can find a
# better model .
self . _tryAll3FieldCombinations = vars . get ( 'tryAll3FieldCombinations' , False )
# Always include timestamp fields in the 3 - field swarms ?
# This is a less compute intensive version of tryAll3FieldCombinations .
# Instead of trying ALL possible 3 field combinations , it just insures
# that the timestamp fields ( dayOfWeek , timeOfDay , weekend ) are never left
# out when generating the 3 - field swarms .
self . _tryAll3FieldCombinationsWTimestamps = vars . get ( 'tryAll3FieldCombinationsWTimestamps' , False )
# Allow the permutations file to override minFieldContribution . This would
# be set to a negative number for large swarms so that you don ' t disqualify
# a field in an early sprint just because it did poorly there . Sometimes ,
# a field that did poorly in an early sprint could help accuracy when
# added in a later sprint
minFieldContribution = vars . get ( 'minFieldContribution' , None )
if minFieldContribution is not None :
    self . _minFieldContribution = minFieldContribution
# Allow the permutations file to override maxBranching .
maxBranching = vars . get ( 'maxFieldBranching' , None )
if maxBranching is not None :
    self . _maxBranching = maxBranching
# Read in the optimization info .
if 'maximize' in vars :
    self . _optimizeKey = vars [ 'maximize' ]
    self . _maximize = True
elif 'minimize' in vars :
    self . _optimizeKey = vars [ 'minimize' ]
    self . _maximize = False
else :
    raise RuntimeError ( "Permutations file '%s' does not include a maximize" " or minimize metric." )
# The permutations file is the new location for maxModels . The old location ,
# in the jobParams is deprecated .
maxModels = vars . get ( 'maxModels' )
if maxModels is not None :
    if self . _maxModels is None :
        self . _maxModels = maxModels
    else :
        raise RuntimeError ( 'It is an error to specify maxModels both in the job' ' params AND in the permutations file.' )
# Figure out if what kind of search this is :
# If it ' s a temporal prediction search :
# the first sprint has 1 swarm , with just the predicted field
# elif it ' s a spatial prediction search :
# the first sprint has N swarms , each with predicted field + one
# other field .
# elif it ' s a classification search :
# the first sprint has N swarms , each with 1 field
inferenceType = modelDescription [ 'modelParams' ] [ 'inferenceType' ]
if not InferenceType . validate ( inferenceType ) :
    raise ValueError ( "Invalid inference type %s" % inferenceType )
if inferenceType in [ InferenceType . TemporalMultiStep , InferenceType . NontemporalMultiStep ] : # If it does not have a separate encoder for the predicted field that
# goes to the classifier , it is a legacy multi - step network
    classifierOnlyEncoder = None
    for encoder in modelDescription [ "modelParams" ] [ "sensorParams" ] [ "encoders" ] . values ( ) :
        if encoder . get ( "classifierOnly" , False ) and encoder [ "fieldname" ] == vars . get ( 'predictedField' , None ) :
            classifierOnlyEncoder = encoder
            break
    if classifierOnlyEncoder is None or self . _inputPredictedField == "yes" : # If we don ' t have a separate encoder for the classifier ( legacy
    # MultiStep ) or the caller explicitly wants to include the predicted
    # field , then use the legacy temporal search methodology .
        self . _searchType = HsSearchType . legacyTemporal
    else :
        self . _searchType = HsSearchType . temporal
elif inferenceType in [ InferenceType . TemporalNextStep , InferenceType . TemporalAnomaly ] :
    self . _searchType = HsSearchType . legacyTemporal
elif inferenceType in ( InferenceType . TemporalClassification , InferenceType . NontemporalClassification ) :
    self . _searchType = HsSearchType . classification
else :
    raise RuntimeError ( "Unsupported inference type: %s" % inferenceType )
# Get the predicted field . Note that even classification experiments
# have a " predicted " field - which is the field that contains the
# classification value .
self . _predictedField = vars . get ( 'predictedField' , None )
if self . _predictedField is None :
    raise RuntimeError ( "Permutations file '%s' does not have the required" " 'predictedField' variable" % filename )
# Read in and validate the permutations dict
if 'permutations' not in vars :
    raise RuntimeError ( "Permutations file '%s' does not define permutations" % filename )
if not isinstance ( vars [ 'permutations' ] , dict ) :
    raise RuntimeError ( "Permutations file '%s' defines a permutations variable " "but it is not a dict" )
self . _encoderNames = [ ]
self . _permutations = vars [ 'permutations' ]
self . _flattenedPermutations = dict ( )
def _flattenPermutations ( value , keys ) :
    if ':' in keys [ - 1 ] :
        raise RuntimeError ( "The permutation variable '%s' contains a ':' " "character, which is not allowed." )
    flatKey = _flattenKeys ( keys )
    if isinstance ( value , PermuteEncoder ) :
        self . _encoderNames . append ( flatKey )
        # If this is the encoder for the predicted field , save its name .
        if value . fieldName == self . _predictedField :
            self . _predictedFieldEncoder = flatKey
        # Store the flattened representations of the variables within the
        # encoder .
        for encKey , encValue in value . kwArgs . iteritems ( ) :
            if isinstance ( encValue , PermuteVariable ) :
                self . _flattenedPermutations [ '%s:%s' % ( flatKey , encKey ) ] = encValue
    elif isinstance ( value , PermuteVariable ) :
        self . _flattenedPermutations [ flatKey ] = value
    else :
        if isinstance ( value , PermuteVariable ) :
            self . _flattenedPermutations [ key ] = value
rApply ( self . _permutations , _flattenPermutations )


signature:
def _checkForOrphanedModels ( self ) :
    """If there are any models that haven ' t been updated in a while , consider
 them dead , and mark them as hidden in our resultsDB . We also change the
 paramsHash and particleHash of orphaned models so that we can
 re - generate that particle and / or model again if we desire .
 Parameters :
 retval :"""

implementation:
self . logger . debug ( "Checking for orphaned models older than %s" % ( self . _modelOrphanIntervalSecs ) )
while True :
    orphanedModelId = self . _cjDAO . modelAdoptNextOrphan ( self . _jobID , self . _modelOrphanIntervalSecs )
    if orphanedModelId is None :
        return
    self . logger . info ( "Removing orphaned model: %d" % ( orphanedModelId ) )
    # Change the model hash and params hash as stored in the models table so
    # that we can insert a new model with the same paramsHash
    for attempt in range ( 100 ) :
        paramsHash = hashlib . md5 ( "OrphanParams.%d.%d" % ( orphanedModelId , attempt ) ) . digest ( )
        particleHash = hashlib . md5 ( "OrphanParticle.%d.%d" % ( orphanedModelId , attempt ) ) . digest ( )
        try :
            self . _cjDAO . modelSetFields ( orphanedModelId , dict ( engParamsHash = paramsHash , engParticleHash = particleHash ) )
            success = True
        except :
            success = False
        if success :
            break
    if not success :
        raise RuntimeError ( "Unexpected failure to change paramsHash and " "particleHash of orphaned model" )
    # Mark this model as complete , with reason " orphaned "
    self . _cjDAO . modelSetCompleted ( modelID = orphanedModelId , completionReason = ClientJobsDAO . CMPL_REASON_ORPHAN , completionMsg = "Orphaned" )
    # Update our results DB immediately , rather than wait for the worker
    # to inform us . This insures that the getParticleInfos ( ) calls we make
    # below don ' t include this particle . Setting the metricResult to None
    # sets it to worst case
    self . _resultsDB . update ( modelID = orphanedModelId , modelParams = None , modelParamsHash = paramsHash , metricResult = None , completed = True , completionReason = ClientJobsDAO . CMPL_REASON_ORPHAN , matured = True , numRecords = 0 )