signature:
def corruptVector ( v1 , noiseLevel , numActiveCols ) :
    """Corrupts a copy of a binary vector by inverting noiseLevel percent of its bits .
 @ param v1 ( array ) binary vector whose copy will be corrupted
 @ param noiseLevel ( float ) amount of noise to be applied on the new vector
 @ param numActiveCols ( int ) number of sparse columns that represent an input
 @ return v2 ( array ) corrupted binary vector"""

implementation:
size = len ( v1 )
v2 = np . zeros ( size , dtype = "uint32" )
bitsToSwap = int ( noiseLevel * numActiveCols )
# Copy the contents of v1 into v2
for i in range ( size ) :
    v2 [ i ] = v1 [ i ]
for _ in range ( bitsToSwap ) :
    i = random . randrange ( size )
    if v2 [ i ] == 1 :
        v2 [ i ] = 0
    else :
        v2 [ i ] = 1
return v2


signature:
def showPredictions ( ) :
    """Shows predictions of the TM when presented with the characters A , B , C , D , X , and
 Y without any contextual information , that is , not embedded within a sequence ."""

implementation:
for k in range ( 6 ) :
    tm . reset ( )
    print "--- " + "ABCDXY" [ k ] + " ---"
    tm . compute ( set ( seqT [ k ] [ : ] . nonzero ( ) [ 0 ] . tolist ( ) ) , learn = False )
    activeColumnsIndices = [ tm . columnForCell ( i ) for i in tm . getActiveCells ( ) ]
    predictedColumnIndices = [ tm . columnForCell ( i ) for i in tm . getPredictiveCells ( ) ]
    currentColumns = [ 1 if i in activeColumnsIndices else 0 for i in range ( tm . numberOfColumns ( ) ) ]
    predictedColumns = [ 1 if i in predictedColumnIndices else 0 for i in range ( tm . numberOfColumns ( ) ) ]
    print ( "Active cols: " + str ( np . nonzero ( currentColumns ) [ 0 ] ) )
    print ( "Predicted cols: " + str ( np . nonzero ( predictedColumns ) [ 0 ] ) )
    print ""


signature:
def trainTM ( sequence , timeSteps , noiseLevel ) :
    """Trains the TM with given sequence for a given number of time steps and level of input
 corruption
 @ param sequence ( array ) array whose rows are the input characters
 @ param timeSteps ( int ) number of time steps in which the TM will be presented with sequence
 @ param noiseLevel ( float ) amount of noise to be applied on the characters in the sequence"""

implementation:
currentColumns = np . zeros ( tm . numberOfColumns ( ) , dtype = "uint32" )
predictedColumns = np . zeros ( tm . numberOfColumns ( ) , dtype = "uint32" )
ts = 0
for t in range ( timeSteps ) :
    tm . reset ( )
    for k in range ( 4 ) :
        v = corruptVector ( sequence [ k ] [ : ] , noiseLevel , sparseCols )
        tm . compute ( set ( v [ : ] . nonzero ( ) [ 0 ] . tolist ( ) ) , learn = True )
        activeColumnsIndices = [ tm . columnForCell ( i ) for i in tm . getActiveCells ( ) ]
        predictedColumnIndices = [ tm . columnForCell ( i ) for i in tm . getPredictiveCells ( ) ]
        currentColumns = [ 1 if i in activeColumnsIndices else 0 for i in range ( tm . numberOfColumns ( ) ) ]
        acc = accuracy ( currentColumns , predictedColumns )
        x . append ( ts )
        y . append ( acc )
        ts += 1
        predictedColumns = [ 1 if i in predictedColumnIndices else 0 for i in range ( tm . numberOfColumns ( ) ) ]


signature:
def encodeIntoArray ( self , inputVal , outputVal ) :
    """See method description in base . py"""

implementation:
if len ( inputVal ) != len ( outputVal ) :
    raise ValueError ( "Different input (%i) and output (%i) sizes." % ( len ( inputVal ) , len ( outputVal ) ) )
if self . w is not None and sum ( inputVal ) != self . w :
    raise ValueError ( "Input has %i bits but w was set to %i." % ( sum ( inputVal ) , self . w ) )
outputVal [ : ] = inputVal [ : ]
if self . verbosity >= 2 :
    print "input:" , inputVal , "output:" , outputVal
    print "decoded:" , self . decodedToStr ( self . decode ( outputVal ) )


signature:
def decode ( self , encoded , parentFieldName = "" ) :
    """See the function description in base . py"""

implementation:
if parentFieldName != "" :
    fieldName = "%s.%s" % ( parentFieldName , self . name )
else :
    fieldName = self . name
return ( { fieldName : ( [ [ 0 , 0 ] ] , "input" ) } , [ fieldName ] )


signature:
def getBucketInfo ( self , buckets ) :
    """See the function description in base . py"""

implementation: return [ EncoderResult ( value = 0 , scalar = 0 , encoding = numpy . zeros ( self . n ) ) ]


signature:
def topDownCompute ( self , encoded ) :
    """See the function description in base . py"""

implementation: return EncoderResult ( value = 0 , scalar = 0 , encoding = numpy . zeros ( self . n ) )


signature:
def closenessScores ( self , expValues , actValues , ** kwargs ) :
    """Does a bitwise compare of the two bitmaps and returns a fractonal
 value between 0 and 1 of how similar they are .
 - ` ` 1 ` ` = > identical
 - ` ` 0 ` ` = > no overlaping bits
 ` ` kwargs ` ` will have the keyword " fractional " , which is assumed by this
 encoder ."""

implementation:
ratio = 1.0
esum = int ( expValues . sum ( ) )
asum = int ( actValues . sum ( ) )
if asum > esum :
    diff = asum - esum
    if diff < esum :
        ratio = 1 - diff / float ( esum )
    else :
        ratio = 1 / float ( diff )
olap = expValues & actValues
osum = int ( olap . sum ( ) )
if esum == 0 :
    r = 0.0
else :
    r = osum / float ( esum )
r = r * ratio
return numpy . array ( [ r ] )


signature:
def getCallerInfo ( depth = 2 ) :
    """Utility function to get information about function callers
 The information is the tuple ( function / method name , filename , class )
 The class will be None if the caller is just a function and not an object
 method .
 : param depth : ( int ) how far back in the callstack to go to extract the caller
 info"""

implementation:
f = sys . _getframe ( depth )
method_name = f . f_code . co_name
filename = f . f_code . co_filename
arg_class = None
args = inspect . getargvalues ( f )
if len ( args [ 0 ] ) > 0 :
    arg_name = args [ 0 ] [ 0 ]
    # potentially the ' self ' arg if its a method
    arg_class = args [ 3 ] [ arg_name ] . __class__ . __name__
return ( method_name , filename , arg_class )


signature:
def title ( s = None , additional = '' , stream = sys . stdout ) :
    """Utility function to display nice titles
 It automatically extracts the name of the function / method it is called from
 and you can add additional text . title ( ) will then print the name
 of the function / method and the additional text surrounded by tow lines
 of dashes . If you don ' t want the name of the function , you can provide
 alternative text ( regardless of the additional text )
 : param s : ( string ) text to display , uses the function name and arguments by
 default
 : param additional : ( string ) extra text to display ( not needed if s is not
 None )
 : param stream : ( stream ) the stream to print to . Ny default goes to standard
 output
 Examples :
 . . code - block : : python
 def foo ( ) :
 title ( )
 will display :
 . . code - block : : text
 foo
 . . code - block : : python
 def foo ( ) :
 title ( additional = ' ( ) , this is cool ! ! ! ' )
 will display :
 . . code - block : : text
 foo ( ) , this is cool ! ! !
 . . code - block : : python
 def foo ( ) :
 title ( ' No function name here ! ' )
 will display :
 . . code - block : : text
 No function name here !"""

implementation:
if s is None :
    callable_name , file_name , class_name = getCallerInfo ( 2 )
    s = callable_name
    if class_name is not None :
        s = class_name + '.' + callable_name
lines = ( s + additional ) . split ( '\n' )
length = max ( len ( line ) for line in lines )
print >> stream , '-' * length
print >> stream , s + additional
print >> stream , '-' * length


signature:
def getArgumentDescriptions ( f ) :
    """Get the arguments , default values , and argument descriptions for a function .
 Parses the argument descriptions out of the function docstring , using a
 format something lke this :
 [ junk ]
 argument _ name : description . . .
 description . . .
 description . . .
 [ junk ]
 [ more arguments ]
 It will find an argument as long as the exact argument name starts the line .
 It will then strip a trailing colon , if present , then strip the rest of the
 line and use it to start the description . It will then strip and append any
 subsequent lines with a greater indent level than the original argument name .
 : param f : ( function ) to inspect
 : returns : ( list of tuples ) ( ` ` argName ` ` , ` ` argDescription ` ` , ` ` defaultValue ` ` )
 If an argument has no default value , the tuple is only two elements long ( as
 ` ` None ` ` cannot be used , since it could be a default value itself ) ."""

implementation:
# Get the argument names and default values
argspec = inspect . getargspec ( f )
# Scan through the docstring to extract documentation for each argument as
# follows :
# Check the first word of the line , stripping a colon if one is present .
# If it matches an argument name :
# Take the rest of the line , stripping leading whitespeace
# Take each subsequent line if its indentation level is greater than the
# initial indentation level
# Once the indentation level is back to the original level , look for
# another argument
docstring = f . __doc__
descriptions = { }
if docstring :
    lines = docstring . split ( '\n' )
    i = 0
    while i < len ( lines ) :
        stripped = lines [ i ] . lstrip ( )
        if not stripped :
            i += 1
            continue
        # Indentation level is index of the first character
        indentLevel = lines [ i ] . index ( stripped [ 0 ] )
        # Get the first word and remove the colon , if present
        firstWord = stripped . split ( ) [ 0 ]
        if firstWord . endswith ( ':' ) :
            firstWord = firstWord [ : - 1 ]
        if firstWord in argspec . args : # Found an argument
            argName = firstWord
            restOfLine = stripped [ len ( firstWord ) + 1 : ] . strip ( )
            argLines = [ restOfLine ]
            # Take the next lines as long as they are indented more
            i += 1
            while i < len ( lines ) :
                stripped = lines [ i ] . lstrip ( )
                if not stripped : # Empty line - stop
                    break
                if lines [ i ] . index ( stripped [ 0 ] ) <= indentLevel : # No longer indented far enough - stop
                    break
                # This line counts too
                argLines . append ( lines [ i ] . strip ( ) )
                i += 1
            # Store this description
            descriptions [ argName ] = ' ' . join ( argLines )
        else : # Not an argument
            i += 1
# Build the list of ( argName , description , defaultValue )
args = [ ]
if argspec . defaults :
    defaultCount = len ( argspec . defaults )
else :
    defaultCount = 0
nonDefaultArgCount = len ( argspec . args ) - defaultCount
for i , argName in enumerate ( argspec . args ) :
    if i >= nonDefaultArgCount :
        defaultValue = argspec . defaults [ i - nonDefaultArgCount ]
        args . append ( ( argName , descriptions . get ( argName , "" ) , defaultValue ) )
    else :
        args . append ( ( argName , descriptions . get ( argName , "" ) ) )
return args


signature:
def _genLoggingFilePath ( ) :
    """Generate a filepath for the calling app"""

implementation:
appName = os . path . splitext ( os . path . basename ( sys . argv [ 0 ] ) ) [ 0 ] or 'UnknownApp'
appLogDir = os . path . abspath ( os . path . join ( os . environ [ 'NTA_LOG_DIR' ] , 'numenta-logs-%s' % ( os . environ [ 'USER' ] , ) , appName ) )
appLogFileName = '%s-%s-%s.log' % ( appName , long ( time . mktime ( time . gmtime ( ) ) ) , os . getpid ( ) )
return os . path . join ( appLogDir , appLogFileName )


signature:
def aggregationToMonthsSeconds ( interval ) :
    """Return the number of months and seconds from an aggregation dict that
 represents a date and time .
 Interval is a dict that contain one or more of the following keys : ' years ' ,
 ' months ' , ' weeks ' , ' days ' , ' hours ' , ' minutes ' , seconds ' , ' milliseconds ' ,
 ' microseconds ' .
 For example :
 aggregationMicroseconds ( { ' years ' : 1 , ' hours ' : 4 , ' microseconds ' : 42 } ) = =
 { ' months ' : 12 , ' seconds ' : 14400.000042}
 : param interval : ( dict ) The aggregation interval representing a date and time
 : returns : ( dict ) number of months and seconds in the interval :
 ` ` { months ' : XX , ' seconds ' : XX } ` ` . The seconds is
 a floating point that can represent resolutions down to a
 microsecond ."""

implementation:
seconds = interval . get ( 'microseconds' , 0 ) * 0.000001
seconds += interval . get ( 'milliseconds' , 0 ) * 0.001
seconds += interval . get ( 'seconds' , 0 )
seconds += interval . get ( 'minutes' , 0 ) * 60
seconds += interval . get ( 'hours' , 0 ) * 60 * 60
seconds += interval . get ( 'days' , 0 ) * 24 * 60 * 60
seconds += interval . get ( 'weeks' , 0 ) * 7 * 24 * 60 * 60
months = interval . get ( 'months' , 0 )
months += 12 * interval . get ( 'years' , 0 )
return { 'months' : months , 'seconds' : seconds }


signature:
def aggregationDivide ( dividend , divisor ) :
    """Return the result from dividing two dicts that represent date and time .
 Both dividend and divisor are dicts that contain one or more of the following
 keys : ' years ' , ' months ' , ' weeks ' , ' days ' , ' hours ' , ' minutes ' , seconds ' ,
 ' milliseconds ' , ' microseconds ' .
 For example :
 aggregationDivide ( { ' hours ' : 4 } , { ' minutes ' : 15 } ) = = 16
 : param dividend : ( dict ) The numerator , as a dict representing a date and time
 : param divisor : ( dict ) the denominator , as a dict representing a date and time
 : returns : ( float ) number of times divisor goes into dividend"""

implementation:
# Convert each into microseconds
dividendMonthSec = aggregationToMonthsSeconds ( dividend )
divisorMonthSec = aggregationToMonthsSeconds ( divisor )
# It is a usage error to mix both months and seconds in the same operation
if ( dividendMonthSec [ 'months' ] != 0 and divisorMonthSec [ 'seconds' ] != 0 ) or ( dividendMonthSec [ 'seconds' ] != 0 and divisorMonthSec [ 'months' ] != 0 ) :
    raise RuntimeError ( "Aggregation dicts with months/years can only be " "inter-operated with other aggregation dicts that contain " "months/years" )
if dividendMonthSec [ 'months' ] > 0 :
    return float ( dividendMonthSec [ 'months' ] ) / divisor [ 'months' ]
else :
    return float ( dividendMonthSec [ 'seconds' ] ) / divisorMonthSec [ 'seconds' ]


signature:
def validateOpfJsonValue ( value , opfJsonSchemaFilename ) :
    """Validate a python object against an OPF json schema file
 : param value : target python object to validate ( typically a dictionary )
 : param opfJsonSchemaFilename : ( string ) OPF json schema filename containing the
 json schema object . ( e . g . , opfTaskControlSchema . json )
 : raises : jsonhelpers . ValidationError when value fails json validation"""

implementation:
# Create a path by joining the filename with our local json schema root
jsonSchemaPath = os . path . join ( os . path . dirname ( __file__ ) , "jsonschema" , opfJsonSchemaFilename )
# Validate
jsonhelpers . validate ( value , schemaPath = jsonSchemaPath )
return


signature:
def initLogger ( obj ) :
    """Helper function to create a logger object for the current object with
 the standard Numenta prefix .
 : param obj : ( object ) to add a logger to"""

implementation:
if inspect . isclass ( obj ) :
    myClass = obj
else :
    myClass = obj . __class__
logger = logging . getLogger ( "." . join ( [ 'com.numenta' , myClass . __module__ , myClass . __name__ ] ) )
return logger


signature:
def matchPatterns ( patterns , keys ) :
    """Returns a subset of the keys that match any of the given patterns
 : param patterns : ( list ) regular expressions to match
 : param keys : ( list ) keys to search for matches"""

implementation:
results = [ ]
if patterns :
    for pattern in patterns :
        prog = re . compile ( pattern )
        for key in keys :
            if prog . match ( key ) :
                results . append ( key )
else :
    return None
return results


signature:
def _getScaledValue ( self , inpt ) :
    """Convert the input , which is in normal space , into log space"""

implementation:
if inpt == SENTINEL_VALUE_FOR_MISSING_DATA :
    return None
else :
    val = inpt
    if val < self . minval :
        val = self . minval
    elif val > self . maxval :
        val = self . maxval
    scaledVal = math . log10 ( val )
    return scaledVal


signature:
def getBucketIndices ( self , inpt ) :
    """See the function description in base . py"""

implementation:
# Get the scaled value
scaledVal = self . _getScaledValue ( inpt )
if scaledVal is None :
    return [ None ]
else :
    return self . encoder . getBucketIndices ( scaledVal )


signature:
def encodeIntoArray ( self , inpt , output ) :
    """See the function description in base . py"""

implementation:
# Get the scaled value
scaledVal = self . _getScaledValue ( inpt )
if scaledVal is None :
    output [ 0 : ] = 0
else :
    self . encoder . encodeIntoArray ( scaledVal , output )
    if self . verbosity >= 2 :
        print "input:" , inpt , "scaledVal:" , scaledVal , "output:" , output
        print "decoded:" , self . decodedToStr ( self . decode ( output ) )


signature:
def decode ( self , encoded , parentFieldName = '' ) :
    """See the function description in base . py"""

implementation:
# Get the scalar values from the underlying scalar encoder
( fieldsDict , fieldNames ) = self . encoder . decode ( encoded )
if len ( fieldsDict ) == 0 :
    return ( fieldsDict , fieldNames )
# Expect only 1 field
assert ( len ( fieldsDict ) == 1 )
# Convert each range into normal space
( inRanges , inDesc ) = fieldsDict . values ( ) [ 0 ]
outRanges = [ ]
for ( minV , maxV ) in inRanges :
    outRanges . append ( ( math . pow ( 10 , minV ) , math . pow ( 10 , maxV ) ) )
# Generate a text description of the ranges
desc = ""
numRanges = len ( outRanges )
for i in xrange ( numRanges ) :
    if outRanges [ i ] [ 0 ] != outRanges [ i ] [ 1 ] :
        desc += "%.2f-%.2f" % ( outRanges [ i ] [ 0 ] , outRanges [ i ] [ 1 ] )
    else :
        desc += "%.2f" % ( outRanges [ i ] [ 0 ] )
    if i < numRanges - 1 :
        desc += ", "
# Return result
if parentFieldName != '' :
    fieldName = "%s.%s" % ( parentFieldName , self . name )
else :
    fieldName = self . name
return ( { fieldName : ( outRanges , desc ) } , [ fieldName ] )


signature:
def getBucketValues ( self ) :
    """See the function description in base . py"""

implementation:
# Need to re - create ?
if self . _bucketValues is None :
    scaledValues = self . encoder . getBucketValues ( )
    self . _bucketValues = [ ]
    for scaledValue in scaledValues :
        value = math . pow ( 10 , scaledValue )
        self . _bucketValues . append ( value )
return self . _bucketValues


signature:
def getBucketInfo ( self , buckets ) :
    """See the function description in base . py"""

implementation:
scaledResult = self . encoder . getBucketInfo ( buckets ) [ 0 ]
scaledValue = scaledResult . value
value = math . pow ( 10 , scaledValue )
return [ EncoderResult ( value = value , scalar = value , encoding = scaledResult . encoding ) ]


signature:
def topDownCompute ( self , encoded ) :
    """See the function description in base . py"""

implementation:
scaledResult = self . encoder . topDownCompute ( encoded ) [ 0 ]
scaledValue = scaledResult . value
value = math . pow ( 10 , scaledValue )
return EncoderResult ( value = value , scalar = value , encoding = scaledResult . encoding )


signature:
def closenessScores ( self , expValues , actValues , fractional = True ) :
    """See the function description in base . py"""

implementation:
# Compute the percent error in log space
if expValues [ 0 ] > 0 :
    expValue = math . log10 ( expValues [ 0 ] )
else :
    expValue = self . minScaledValue
if actValues [ 0 ] > 0 :
    actValue = math . log10 ( actValues [ 0 ] )
else :
    actValue = self . minScaledValue
if fractional :
    err = abs ( expValue - actValue )
    pctErr = err / ( self . maxScaledValue - self . minScaledValue )
    pctErr = min ( 1.0 , pctErr )
    closeness = 1.0 - pctErr
else :
    err = abs ( expValue - actValue )
    closeness = err
# print " log : : " , " expValue : " , expValues [ 0 ] , " actValue : " , actValues [ 0 ] , \
# " closeness " , closeness
# import pdb ; pdb . set _ trace ( )
return numpy . array ( [ closeness ] )


signature:
def export ( self ) :
    """Exports a network as a networkx MultiDiGraph intermediate representation
 suitable for visualization .
 : return : networkx MultiDiGraph"""

implementation:
graph = nx . MultiDiGraph ( )
# Add regions to graph as nodes , annotated by name
regions = self . network . getRegions ( )
for idx in xrange ( regions . getCount ( ) ) :
    regionPair = regions . getByIndex ( idx )
    regionName = regionPair [ 0 ]
    graph . add_node ( regionName , label = regionName )
# Add links between regions to graph as edges , annotate by input - output
# name pairs
for linkName , link in self . network . getLinks ( ) :
    graph . add_edge ( link . getSrcRegionName ( ) , link . getDestRegionName ( ) , src = link . getSrcOutputName ( ) , dest = link . getDestInputName ( ) )
return graph


signature:
def bitsToString ( arr ) :
    """Returns a string representing a numpy array of 0 ' s and 1 ' s"""

implementation:
s = array ( 'c' , '.' * len ( arr ) )
for i in xrange ( len ( arr ) ) :
    if arr [ i ] == 1 :
        s [ i ] = '*'
return s


signature:
def percentOverlap ( x1 , x2 , size ) :
    """Computes the percentage of overlap between vectors x1 and x2.
 @ param x1 ( array ) binary vector
 @ param x2 ( array ) binary vector
 @ param size ( int ) length of binary vectors
 @ return percentOverlap ( float ) percentage overlap between x1 and x2"""

implementation:
nonZeroX1 = np . count_nonzero ( x1 )
nonZeroX2 = np . count_nonzero ( x2 )
minX1X2 = min ( nonZeroX1 , nonZeroX2 )
percentOverlap = 0
if minX1X2 > 0 :
    percentOverlap = float ( np . dot ( x1 , x2 ) ) / float ( minX1X2 )
return percentOverlap


signature:
def resetVector ( x1 , x2 ) :
    """Copies the contents of vector x1 into vector x2.
 @ param x1 ( array ) binary vector to be copied
 @ param x2 ( array ) binary vector where x1 is copied"""

implementation:
size = len ( x1 )
for i in range ( size ) :
    x2 [ i ] = x1 [ i ]


signature:
def runCPU ( ) :
    """Poll CPU usage , make predictions , and plot the results . Runs forever ."""

implementation:
# Create the model for predicting CPU usage .
model = ModelFactory . create ( model_params . MODEL_PARAMS )
model . enableInference ( { 'predictedField' : 'cpu' } )
# The shifter will align prediction and actual values .
shifter = InferenceShifter ( )
# Keep the last WINDOW predicted and actual values for plotting .
actHistory = deque ( [ 0.0 ] * WINDOW , maxlen = 60 )
predHistory = deque ( [ 0.0 ] * WINDOW , maxlen = 60 )
# Initialize the plot lines that we will update with each new record .
actline , = plt . plot ( range ( WINDOW ) , actHistory )
predline , = plt . plot ( range ( WINDOW ) , predHistory )
# Set the y - axis range .
actline . axes . set_ylim ( 0 , 100 )
predline . axes . set_ylim ( 0 , 100 )
while True :
    s = time . time ( )
    # Get the CPU usage .
    cpu = psutil . cpu_percent ( )
    # Run the input through the model and shift the resulting prediction .
    modelInput = { 'cpu' : cpu }
    result = shifter . shift ( model . run ( modelInput ) )
    # Update the trailing predicted and actual value deques .
    inference = result . inferences [ 'multiStepBestPredictions' ] [ 5 ]
    if inference is not None :
        actHistory . append ( result . rawInput [ 'cpu' ] )
        predHistory . append ( inference )
    # Redraw the chart with the new data .
    actline . set_ydata ( actHistory )
    # update the data
    predline . set_ydata ( predHistory )
    # update the data
    plt . draw ( )
    plt . legend ( ( 'actual' , 'predicted' ) )
    # Make sure we wait a total of 2 seconds per iteration .
    try :
        plt . pause ( SECONDS_PER_STEP )
    except :
        pass


signature:
def _extractCallingMethodArgs ( ) :
    """Returns args dictionary from the calling method"""

implementation:
import inspect
import copy
callingFrame = inspect . stack ( ) [ 1 ] [ 0 ]
argNames , _ , _ , frameLocalVarDict = inspect . getargvalues ( callingFrame )
argNames . remove ( "self" )
args = copy . copy ( frameLocalVarDict )
for varName in frameLocalVarDict :
    if varName not in argNames :
        args . pop ( varName )
return args


signature:
def write ( self , proto ) :
    """Populate serialization proto instance .
 : param proto : ( BacktrackingTMCppProto ) the proto instance to populate"""

implementation:
# Write base class to proto . baseTM ( BacktrackingTMProto )
super ( BacktrackingTMCPP , self ) . write ( proto . baseTM )
self . cells4 . write ( proto . cells4 )
proto . makeCells4Ephemeral = self . makeCells4Ephemeral
proto . seed = self . seed
proto . checkSynapseConsistency = self . checkSynapseConsistency
proto . initArgs = json . dumps ( self . _initArgsDict )


signature:
def read ( cls , proto ) :
    """Deserialize from proto instance .
 : param proto : ( BacktrackingTMCppProto ) the proto instance to read from"""

implementation:
# Use base class to create initial class from proto . baseTM
# ( BacktrackingTMProto )
obj = BacktrackingTM . read ( proto . baseTM )
obj . __class__ = cls
# Additional CPP - specific deserialization
newCells4 = Cells4 . read ( proto . cells4 )
print newCells4
obj . cells4 = newCells4
obj . makeCells4Ephemeral = proto . makeCells4Ephemeral
obj . seed = proto . seed
obj . checkSynapseConsistency = proto . checkSynapseConsistency
obj . _initArgsDict = json . loads ( proto . initArgs )
# Convert unicode to str
obj . _initArgsDict [ "outputType" ] = str ( obj . _initArgsDict [ "outputType" ] )
# Initialize ephemeral attributes
obj . allocateStatesInCPP = False
obj . retrieveLearningStates = False
obj . _setStatePointers ( )
return obj


signature:
def _getEphemeralMembers ( self ) :
    """List of our member variables that we don ' t need to be saved"""

implementation:
e = BacktrackingTM . _getEphemeralMembers ( self )
if self . makeCells4Ephemeral :
    e . extend ( [ 'cells4' ] )
return e


signature:
def _initEphemerals ( self ) :
    """Initialize all ephemeral members after being restored to a pickled state ."""

implementation:
BacktrackingTM . _initEphemerals ( self )
# cells4 specific initialization
# If True , let C + + allocate memory for activeState , predictedState , and
# learnState . In this case we can retrieve copies of these states but can ' t
# set them directly from Python . If False , Python can allocate them as
# numpy arrays and we can pass pointers to the C + + using setStatePointers
self . allocateStatesInCPP = False
# Set this to true for debugging or accessing learning states
self . retrieveLearningStates = False
if self . makeCells4Ephemeral :
    self . _initCells4 ( )


signature:
def compute ( self , bottomUpInput , enableLearn , enableInference = None ) :
    """Overrides : meth : ` nupic . algorithms . backtracking _ tm . BacktrackingTM . compute ` ."""

implementation:
# The C + + TM takes 32 bit floats as input . uint32 works as well since the
# code only checks whether elements are non - zero
assert ( bottomUpInput . dtype == numpy . dtype ( 'float32' ) ) or ( bottomUpInput . dtype == numpy . dtype ( 'uint32' ) ) or ( bottomUpInput . dtype == numpy . dtype ( 'int32' ) )
self . iterationIdx = self . iterationIdx + 1
# As a speed optimization for now ( until we need online learning ) , skip
# computing the inference output while learning
if enableInference is None :
    if enableLearn :
        enableInference = False
    else :
        enableInference = True
# Run compute and retrieve selected state and member variables
self . _setStatePointers ( )
y = self . cells4 . compute ( bottomUpInput , enableInference , enableLearn )
self . currentOutput = y . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
self . avgLearnedSeqLength = self . cells4 . getAvgLearnedSeqLength ( )
self . _copyAllocatedStates ( )
# Update the prediction score stats
# Learning always includes inference
if self . collectStats :
    activeColumns = bottomUpInput . nonzero ( ) [ 0 ]
    if enableInference :
        predictedState = self . infPredictedState [ 't-1' ]
    else :
        predictedState = self . lrnPredictedState [ 't-1' ]
    self . _updateStatsInferEnd ( self . _internalStats , activeColumns , predictedState , self . colConfidence [ 't-1' ] )
# Finally return the TM output
output = self . _computeOutput ( )
# Print diagnostic information based on the current verbosity level
self . printComputeEnd ( output , learn = enableLearn )
self . resetCalled = False
return output


signature:
def _copyAllocatedStates ( self ) :
    """If state is allocated in CPP , copy over the data into our numpy arrays ."""

implementation:
# Get learn states if we need to print them out
if self . verbosity > 1 or self . retrieveLearningStates :
    ( activeT , activeT1 , predT , predT1 ) = self . cells4 . getLearnStates ( )
    self . lrnActiveState [ 't-1' ] = activeT1 . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . lrnActiveState [ 't' ] = activeT . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . lrnPredictedState [ 't-1' ] = predT1 . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . lrnPredictedState [ 't' ] = predT . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
if self . allocateStatesInCPP :
    assert False
    ( activeT , activeT1 , predT , predT1 , colConfidenceT , colConfidenceT1 , confidenceT , confidenceT1 ) = self . cells4 . getStates ( )
    self . cellConfidence [ 't' ] = confidenceT . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . cellConfidence [ 't-1' ] = confidenceT1 . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . colConfidence [ 't' ] = colConfidenceT . reshape ( self . numberOfCols )
    self . colConfidence [ 't-1' ] = colConfidenceT1 . reshape ( self . numberOfCols )
    self . infActiveState [ 't-1' ] = activeT1 . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . infActiveState [ 't' ] = activeT . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . infPredictedState [ 't-1' ] = predT1 . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )
    self . infPredictedState [ 't' ] = predT . reshape ( ( self . numberOfCols , self . cellsPerColumn ) )


signature:
def _setStatePointers ( self ) :
    """If we are having CPP use numpy - allocated buffers , set these buffer
 pointers . This is a relatively fast operation and , for safety , should be
 done before every call to the cells4 compute methods . This protects us
 in situations where code can cause Python or numpy to create copies ."""

implementation:
if not self . allocateStatesInCPP :
    self . cells4 . setStatePointers ( self . infActiveState [ "t" ] , self . infActiveState [ "t-1" ] , self . infPredictedState [ "t" ] , self . infPredictedState [ "t-1" ] , self . colConfidence [ "t" ] , self . colConfidence [ "t-1" ] , self . cellConfidence [ "t" ] , self . cellConfidence [ "t-1" ] )


signature:
def reset ( self ) :
    """Overrides : meth : ` nupic . algorithms . backtracking _ tm . BacktrackingTM . reset ` ."""

implementation:
if self . verbosity >= 3 :
    print "TM Reset"
self . _setStatePointers ( )
self . cells4 . reset ( )
BacktrackingTM . reset ( self )


signature:
def trimSegments ( self , minPermanence = None , minNumSyns = None ) :
    """Overrides : meth : ` nupic . algorithms . backtracking _ tm . BacktrackingTM . trimSegments ` ."""

implementation:
# Fill in defaults
if minPermanence is None :
    minPermanence = 0.0
if minNumSyns is None :
    minNumSyns = 0
# Print all cells if verbosity says to
if self . verbosity >= 5 :
    print "Cells, all segments:"
    self . printCells ( predictedOnly = False )
return self . cells4 . trimSegments ( minPermanence = minPermanence , minNumSyns = minNumSyns )


signature:
def printSegmentUpdates ( self ) :
    """Overrides : meth : ` nupic . algorithms . backtracking _ tm . BacktrackingTM . printSegmentUpdates ` ."""

implementation:
# TODO : need to add C + + accessors to implement this method
assert False
print "=== SEGMENT UPDATES ===, Num = " , len ( self . segmentUpdates )
for key , updateList in self . segmentUpdates . iteritems ( ) :
    c , i = key [ 0 ] , key [ 1 ]
    print c , i , updateList


signature:
def _slowIsSegmentActive ( self , seg , timeStep ) :
    """A segment is active if it has > = activationThreshold connected
 synapses that are active due to infActiveState ."""

implementation:
numSyn = seg . size ( )
numActiveSyns = 0
for synIdx in xrange ( numSyn ) :
    if seg . getPermanence ( synIdx ) < self . connectedPerm :
        continue
    sc , si = self . getColCellIdx ( seg . getSrcCellIdx ( synIdx ) )
    if self . infActiveState [ timeStep ] [ sc , si ] :
        numActiveSyns += 1
        if numActiveSyns >= self . activationThreshold :
            return True
return numActiveSyns >= self . activationThreshold


signature:
def printCell ( self , c , i , onlyActiveSegments = False ) :
    """Overrides : meth : ` nupic . algorithms . backtracking _ tm . BacktrackingTM . printCell ` ."""

implementation:
nSegs = self . cells4 . nSegmentsOnCell ( c , i )
if nSegs > 0 :
    segList = self . cells4 . getNonEmptySegList ( c , i )
    gidx = c * self . cellsPerColumn + i
    print "Column" , c , "Cell" , i , "(%d)" % ( gidx ) , ":" , nSegs , "segment(s)"
    for k , segIdx in enumerate ( segList ) :
        seg = self . cells4 . getSegment ( c , i , segIdx )
        isActive = self . _slowIsSegmentActive ( seg , 't' )
        if onlyActiveSegments and not isActive :
            continue
        isActiveStr = "*" if isActive else " "
        print "  %sSeg #%-3d" % ( isActiveStr , segIdx ) ,
        print seg . size ( ) ,
        print seg . isSequenceSegment ( ) , "%9.7f" % ( seg . dutyCycle ( self . cells4 . getNLrnIterations ( ) , False , True ) ) ,
        # numPositive / totalActivations
        print "(%4d/%-4d)" % ( seg . getPositiveActivations ( ) , seg . getTotalActivations ( ) ) ,
        # Age
        print "%4d" % ( self . cells4 . getNLrnIterations ( ) - seg . getLastActiveIteration ( ) ) ,
        numSyn = seg . size ( )
        for s in xrange ( numSyn ) :
            sc , si = self . getColCellIdx ( seg . getSrcCellIdx ( s ) )
            print "[%d,%d]%4.2f" % ( sc , si , seg . getPermanence ( s ) ) ,
        print


signature:
def getColCellIdx ( self , idx ) :
    """Get column and cell within column from a global cell index .
 The global index is ` ` idx = colIdx * nCellsPerCol ( ) + cellIdxInCol ` `
 : param idx : ( int ) global cell index
 : returns : ( tuple ) ( colIdx , cellIdxInCol )"""

implementation:
c = idx // self . cellsPerColumn
i = idx - c * self . cellsPerColumn
return c , i


signature:
def getSegmentOnCell ( self , c , i , segIdx ) :
    """Overrides : meth : ` nupic . algorithms . backtracking _ tm . BacktrackingTM . getSegmentOnCell ` ."""

implementation:
segList = self . cells4 . getNonEmptySegList ( c , i )
seg = self . cells4 . getSegment ( c , i , segList [ segIdx ] )
numSyn = seg . size ( )
assert numSyn != 0
# Accumulate segment information
result = [ ]
result . append ( [ int ( segIdx ) , bool ( seg . isSequenceSegment ( ) ) , seg . getPositiveActivations ( ) , seg . getTotalActivations ( ) , seg . getLastActiveIteration ( ) , seg . getLastPosDutyCycle ( ) , seg . getLastPosDutyCycleIteration ( ) ] )
for s in xrange ( numSyn ) :
    sc , si = self . getColCellIdx ( seg . getSrcCellIdx ( s ) )
    result . append ( [ int ( sc ) , int ( si ) , seg . getPermanence ( s ) ] )
return result


signature:
def getSegmentInfo ( self , collectActiveData = False ) :
    """Overrides : meth : ` nupic . algorithms . backtracking _ tm . BacktrackingTM . getSegmentInfo ` ."""

implementation:
# Requires appropriate accessors in C + + cells4 ( currently unimplemented )
assert collectActiveData == False
nSegments , nSynapses = self . getNumSegments ( ) , self . cells4 . nSynapses ( )
distSegSizes , distNSegsPerCell = { } , { }
nActiveSegs , nActiveSynapses = 0 , 0
distPermValues = { }
# Num synapses with given permanence values
numAgeBuckets = 20
distAges = [ ]
ageBucketSize = int ( ( self . iterationIdx + 20 ) / 20 )
for i in range ( numAgeBuckets ) :
    distAges . append ( [ '%d-%d' % ( i * ageBucketSize , ( i + 1 ) * ageBucketSize - 1 ) , 0 ] )
for c in xrange ( self . numberOfCols ) :
    for i in xrange ( self . cellsPerColumn ) : # Update histogram counting cell sizes
        nSegmentsThisCell = self . getNumSegmentsInCell ( c , i )
        if nSegmentsThisCell > 0 :
            if distNSegsPerCell . has_key ( nSegmentsThisCell ) :
                distNSegsPerCell [ nSegmentsThisCell ] += 1
            else :
                distNSegsPerCell [ nSegmentsThisCell ] = 1
            # Update histogram counting segment sizes .
            segList = self . cells4 . getNonEmptySegList ( c , i )
            for segIdx in xrange ( nSegmentsThisCell ) :
                seg = self . getSegmentOnCell ( c , i , segIdx )
                nSynapsesThisSeg = len ( seg ) - 1
                if nSynapsesThisSeg > 0 :
                    if distSegSizes . has_key ( nSynapsesThisSeg ) :
                        distSegSizes [ nSynapsesThisSeg ] += 1
                    else :
                        distSegSizes [ nSynapsesThisSeg ] = 1
                    # Accumulate permanence value histogram
                    for syn in seg [ 1 : ] :
                        p = int ( syn [ 2 ] * 10 )
                        if distPermValues . has_key ( p ) :
                            distPermValues [ p ] += 1
                        else :
                            distPermValues [ p ] = 1
                segObj = self . cells4 . getSegment ( c , i , segList [ segIdx ] )
                age = self . iterationIdx - segObj . getLastActiveIteration ( )
                ageBucket = int ( age / ageBucketSize )
                distAges [ ageBucket ] [ 1 ] += 1
return ( nSegments , nSynapses , nActiveSegs , nActiveSynapses , distSegSizes , distNSegsPerCell , distPermValues , distAges )


signature:
def main ( argv ) :
    """The main function of the HypersearchWorker script . This parses the command
 line arguments , instantiates a HypersearchWorker instance , and then
 runs it .
 Parameters :
 retval : jobID of the job we ran . This is used by unit test code
 when calling this working using the - - params command
 line option ( which tells this worker to insert the job
 itself ) ."""

implementation:
parser = OptionParser ( helpString )
parser . add_option ( "--jobID" , action = "store" , type = "int" , default = None , help = "jobID of the job within the dbTable [default: %default]." )
parser . add_option ( "--modelID" , action = "store" , type = "str" , default = None , help = ( "Tell worker to re-run this model ID. When specified, jobID " "must also be specified [default: %default]." ) )
parser . add_option ( "--workerID" , action = "store" , type = "str" , default = None , help = ( "workerID of the scheduler's SlotAgent (GenericWorker) that " "hosts this SpecializedWorker [default: %default]." ) )
parser . add_option ( "--params" , action = "store" , default = None , help = "Create and execute a new hypersearch request using this JSON " "format params string. This is helpful for unit tests and debugging. " "When specified jobID must NOT be specified. [default: %default]." )
parser . add_option ( "--clearModels" , action = "store_true" , default = False , help = "clear out the models table before starting [default: %default]." )
parser . add_option ( "--resetJobStatus" , action = "store_true" , default = False , help = "Reset the job status before starting  [default: %default]." )
parser . add_option ( "--logLevel" , action = "store" , type = "int" , default = None , help = "override default log level. Pass in an integer value that " "represents the desired logging level (10=logging.DEBUG, " "20=logging.INFO, etc.) [default: %default]." )
# Evaluate command line arguments
( options , args ) = parser . parse_args ( argv [ 1 : ] )
if len ( args ) != 0 :
    raise RuntimeError ( "Expected no command line arguments, but got: %s" % ( args ) )
if ( options . jobID and options . params ) :
    raise RuntimeError ( "--jobID and --params can not be used at the same time" )
if ( options . jobID is None and options . params is None ) :
    raise RuntimeError ( "Either --jobID or --params must be specified." )
initLogging ( verbose = True )
# Instantiate the HypersearchWorker and run it
hst = HypersearchWorker ( options , argv [ 1 : ] )
# Normal use . This is one of among a number of workers . If we encounter
# an exception at the outer loop here , we fail the entire job .
if options . params is None :
    try :
        jobID = hst . run ( )
    except Exception , e :
        jobID = options . jobID
        msg = StringIO . StringIO ( )
        print >> msg , "%s: Exception occurred in Hypersearch Worker: %r" % ( ErrorCodes . hypersearchLogicErr , e )
        traceback . print_exc ( None , msg )
        completionReason = ClientJobsDAO . CMPL_REASON_ERROR
        completionMsg = msg . getvalue ( )
        hst . logger . error ( completionMsg )
        # If no other worker already marked the job as failed , do so now .
        jobsDAO = ClientJobsDAO . get ( )
        workerCmpReason = jobsDAO . jobGetFields ( options . jobID , [ 'workerCompletionReason' ] ) [ 0 ]
        if workerCmpReason == ClientJobsDAO . CMPL_REASON_SUCCESS :
            jobsDAO . jobSetFields ( options . jobID , fields = dict ( cancel = True , workerCompletionReason = ClientJobsDAO . CMPL_REASON_ERROR , workerCompletionMsg = completionMsg ) , useConnectionID = False , ignoreUnchanged = True )
# Run just 1 worker for the entire job . Used for unit tests that run in
# 1 process
else :
    jobID = None
    completionReason = ClientJobsDAO . CMPL_REASON_SUCCESS
    completionMsg = "Success"
    try :
        jobID = hst . run ( )
    except Exception , e :
        jobID = hst . _options . jobID
        completionReason = ClientJobsDAO . CMPL_REASON_ERROR
        completionMsg = "ERROR: %s" % ( e , )
        raise
    finally :
        if jobID is not None :
            cjDAO = ClientJobsDAO . get ( )
            cjDAO . jobSetCompleted ( jobID = jobID , completionReason = completionReason , completionMsg = completionMsg )
return jobID


signature:
def _processUpdatedModels ( self , cjDAO ) :
    """For all models that modified their results since last time this method
 was called , send their latest results to the Hypersearch implementation ."""

implementation:
# Get the latest update counters . This returns a list of tuples :
# ( modelID , updateCounter )
curModelIDCtrList = cjDAO . modelsGetUpdateCounters ( self . _options . jobID )
if len ( curModelIDCtrList ) == 0 :
    return
self . logger . debug ( "current modelID/updateCounters: %s" % ( str ( curModelIDCtrList ) ) )
self . logger . debug ( "last modelID/updateCounters: %s" % ( str ( self . _modelIDCtrList ) ) )
# Find out which ones have changed update counters . Since these are models
# that the Hypersearch implementation already knows about , we don ' t need to
# send params or paramsHash
curModelIDCtrList = sorted ( curModelIDCtrList )
numItems = len ( curModelIDCtrList )
# Each item in the list we are filtering contains :
# ( idxIntoModelIDCtrList , ( modelID , curCtr ) , ( modelID , oldCtr ) )
# We only want to keep the ones where the oldCtr ! = curCtr
changedEntries = filter ( lambda x : x [ 1 ] [ 1 ] != x [ 2 ] [ 1 ] , itertools . izip ( xrange ( numItems ) , curModelIDCtrList , self . _modelIDCtrList ) )
if len ( changedEntries ) > 0 : # Update values in our cache
    self . logger . debug ( "changedEntries: %s" , str ( changedEntries ) )
    for entry in changedEntries :
        ( idx , ( modelID , curCtr ) , ( _ , oldCtr ) ) = entry
        self . _modelIDCtrDict [ modelID ] = curCtr
        assert ( self . _modelIDCtrList [ idx ] [ 0 ] == modelID )
        assert ( curCtr != oldCtr )
        self . _modelIDCtrList [ idx ] [ 1 ] = curCtr
    # Tell Hypersearch implementation of the updated results for each model
    changedModelIDs = [ x [ 1 ] [ 0 ] for x in changedEntries ]
    modelResults = cjDAO . modelsGetResultAndStatus ( changedModelIDs )
    for mResult in modelResults :
        results = mResult . results
        if results is not None :
            results = json . loads ( results )
        self . _hs . recordModelProgress ( modelID = mResult . modelId , modelParams = None , modelParamsHash = mResult . engParamsHash , results = results , completed = ( mResult . status == cjDAO . STATUS_COMPLETED ) , completionReason = mResult . completionReason , matured = mResult . engMatured , numRecords = mResult . numRecords )
# Figure out which ones are newly arrived and add them to our
# cache
curModelIDSet = set ( [ x [ 0 ] for x in curModelIDCtrList ] )
newModelIDs = curModelIDSet . difference ( self . _modelIDSet )
if len ( newModelIDs ) > 0 : # Add new modelID and counters to our cache
    self . _modelIDSet . update ( newModelIDs )
    curModelIDCtrDict = dict ( curModelIDCtrList )
    # Get the results for each of these models and send them to the
    # Hypersearch implementation .
    modelInfos = cjDAO . modelsGetResultAndStatus ( newModelIDs )
    modelInfos . sort ( )
    modelParamsAndHashs = cjDAO . modelsGetParams ( newModelIDs )
    modelParamsAndHashs . sort ( )
    for ( mResult , mParamsAndHash ) in itertools . izip ( modelInfos , modelParamsAndHashs ) :
        modelID = mResult . modelId
        assert ( modelID == mParamsAndHash . modelId )
        # Update our cache of IDs and update counters
        self . _modelIDCtrDict [ modelID ] = curModelIDCtrDict [ modelID ]
        self . _modelIDCtrList . append ( [ modelID , curModelIDCtrDict [ modelID ] ] )
        # Tell the Hypersearch implementation of the new model
        results = mResult . results
        if results is not None :
            results = json . loads ( mResult . results )
        self . _hs . recordModelProgress ( modelID = modelID , modelParams = json . loads ( mParamsAndHash . params ) , modelParamsHash = mParamsAndHash . engParamsHash , results = results , completed = ( mResult . status == cjDAO . STATUS_COMPLETED ) , completionReason = ( mResult . completionReason ) , matured = mResult . engMatured , numRecords = mResult . numRecords )
    # Keep our list sorted
    self . _modelIDCtrList . sort ( )


signature:
def run ( self ) :
    """Run this worker .
 Parameters :
 retval : jobID of the job we ran . This is used by unit test code
 when calling this working using the - - params command
 line option ( which tells this worker to insert the job
 itself ) ."""

implementation:
# Easier access to options
options = self . _options
# Connect to the jobs database
self . logger . info ( "Connecting to the jobs database" )
cjDAO = ClientJobsDAO . get ( )
# Get our worker ID
self . _workerID = cjDAO . getConnectionID ( )
if options . clearModels :
    cjDAO . modelsClearAll ( )
# if params were specified on the command line , insert a new job using
# them .
if options . params is not None :
    options . jobID = cjDAO . jobInsert ( client = 'hwTest' , cmdLine = "echo 'test mode'" , params = options . params , alreadyRunning = True , minimumWorkers = 1 , maximumWorkers = 1 , jobType = cjDAO . JOB_TYPE_HS )
if options . workerID is not None :
    wID = options . workerID
else :
    wID = self . _workerID
buildID = Configuration . get ( 'nupic.software.buildNumber' , 'N/A' )
logPrefix = '<BUILDID=%s, WORKER=HW, WRKID=%s, JOBID=%s> ' % ( buildID , wID , options . jobID )
ExtendedLogger . setLogPrefix ( logPrefix )
# Get the search parameters
# If asked to reset the job status , do that now
if options . resetJobStatus :
    cjDAO . jobSetFields ( options . jobID , fields = { 'workerCompletionReason' : ClientJobsDAO . CMPL_REASON_SUCCESS , 'cancel' : False , # ' engWorkerState ' : None
    } , useConnectionID = False , ignoreUnchanged = True )
jobInfo = cjDAO . jobInfo ( options . jobID )
self . logger . info ( "Job info retrieved: %s" % ( str ( clippedObj ( jobInfo ) ) ) )
# Instantiate the Hypersearch object , which will handle the logic of
# which models to create when we need more to evaluate .
jobParams = json . loads ( jobInfo . params )
# Validate job params
jsonSchemaPath = os . path . join ( os . path . dirname ( __file__ ) , "jsonschema" , "jobParamsSchema.json" )
validate ( jobParams , schemaPath = jsonSchemaPath )
hsVersion = jobParams . get ( 'hsVersion' , None )
if hsVersion == 'v2' :
    self . _hs = HypersearchV2 ( searchParams = jobParams , workerID = self . _workerID , cjDAO = cjDAO , jobID = options . jobID , logLevel = options . logLevel )
else :
    raise RuntimeError ( "Invalid Hypersearch implementation (%s) specified" % ( hsVersion ) )
# The main loop .
try :
    exit = False
    numModelsTotal = 0
    print >> sys . stderr , "reporter:status:Evaluating first model..."
    while not exit : # Choose a model to evaluate
        batchSize = 10
        # How many to try at a time .
        modelIDToRun = None
        while modelIDToRun is None :
            if options . modelID is None : # Get the latest results on all running models and send them to
            # the Hypersearch implementation
            # This calls cjDAO . modelsGetUpdateCounters ( ) , compares the
            # updateCounters with what we have cached , fetches the results for the
            # changed and new models , and sends those to the Hypersearch
            # implementation ' s self . _ hs . recordModelProgress ( ) method .
                self . _processUpdatedModels ( cjDAO )
                # Create a new batch of models
                ( exit , newModels ) = self . _hs . createModels ( numModels = batchSize )
                if exit :
                    break
                # No more models left to create , just loop . The _ hs is waiting for
                # all remaining running models to complete , and may pick up on an
                # orphan if it detects one .
                if len ( newModels ) == 0 :
                    continue
                # Try and insert one that we will run
                for ( modelParams , modelParamsHash , particleHash ) in newModels :
                    jsonModelParams = json . dumps ( modelParams )
                    ( modelID , ours ) = cjDAO . modelInsertAndStart ( options . jobID , jsonModelParams , modelParamsHash , particleHash )
                    # Some other worker is already running it , tell the Hypersearch object
                    # so that it doesn ' t try and insert it again
                    if not ours :
                        mParamsAndHash = cjDAO . modelsGetParams ( [ modelID ] ) [ 0 ]
                        mResult = cjDAO . modelsGetResultAndStatus ( [ modelID ] ) [ 0 ]
                        results = mResult . results
                        if results is not None :
                            results = json . loads ( results )
                        modelParams = json . loads ( mParamsAndHash . params )
                        particleHash = cjDAO . modelsGetFields ( modelID , [ 'engParticleHash' ] ) [ 0 ]
                        particleInst = "%s.%s" % ( modelParams [ 'particleState' ] [ 'id' ] , modelParams [ 'particleState' ] [ 'genIdx' ] )
                        self . logger . info ( "Adding model %d to our internal DB " "because modelInsertAndStart() failed to insert it: " "paramsHash=%s, particleHash=%s, particleId='%s'" , modelID , mParamsAndHash . engParamsHash . encode ( 'hex' ) , particleHash . encode ( 'hex' ) , particleInst )
                        self . _hs . recordModelProgress ( modelID = modelID , modelParams = modelParams , modelParamsHash = mParamsAndHash . engParamsHash , results = results , completed = ( mResult . status == cjDAO . STATUS_COMPLETED ) , completionReason = mResult . completionReason , matured = mResult . engMatured , numRecords = mResult . numRecords )
                    else :
                        modelIDToRun = modelID
                        break
            else : # A specific modelID was passed on the command line
                modelIDToRun = int ( options . modelID )
                mParamsAndHash = cjDAO . modelsGetParams ( [ modelIDToRun ] ) [ 0 ]
                modelParams = json . loads ( mParamsAndHash . params )
                modelParamsHash = mParamsAndHash . engParamsHash
                # Make us the worker
                cjDAO . modelSetFields ( modelIDToRun , dict ( engWorkerConnId = self . _workerID ) )
                if False : # Change the hash and params of the old entry so that we can
                # create a new model with the same params
                    for attempt in range ( 1000 ) :
                        paramsHash = hashlib . md5 ( "OrphanParams.%d.%d" % ( modelIDToRun , attempt ) ) . digest ( )
                        particleHash = hashlib . md5 ( "OrphanParticle.%d.%d" % ( modelIDToRun , attempt ) ) . digest ( )
                        try :
                            cjDAO . modelSetFields ( modelIDToRun , dict ( engParamsHash = paramsHash , engParticleHash = particleHash ) )
                            success = True
                        except :
                            success = False
                        if success :
                            break
                    if not success :
                        raise RuntimeError ( "Unexpected failure to change paramsHash and " "particleHash of orphaned model" )
                    ( modelIDToRun , ours ) = cjDAO . modelInsertAndStart ( options . jobID , mParamsAndHash . params , modelParamsHash )
                # ^ ^ ^ end while modelIDToRun ^ ^ ^ ^ ^
        # We have a model , evaluate it now
        # All done ?
        if exit :
            break
        # Run the model now
        self . logger . info ( "RUNNING MODEL GID=%d, paramsHash=%s, params=%s" , modelIDToRun , modelParamsHash . encode ( 'hex' ) , modelParams )
        # Construct model checkpoint GUID for this model :
        # jobParams [ ' persistentJobGUID ' ] contains the client ' s ( e . g . , API Server )
        # persistent , globally - unique model identifier , which is what we need ;
        persistentJobGUID = jobParams [ 'persistentJobGUID' ]
        assert persistentJobGUID , "persistentJobGUID: %r" % ( persistentJobGUID , )
        modelCheckpointGUID = jobInfo . client + "_" + persistentJobGUID + ( '_' + str ( modelIDToRun ) )
        self . _hs . runModel ( modelID = modelIDToRun , jobID = options . jobID , modelParams = modelParams , modelParamsHash = modelParamsHash , jobsDAO = cjDAO , modelCheckpointGUID = modelCheckpointGUID )
        # TODO : don ' t increment for orphaned models
        numModelsTotal += 1
        self . logger . info ( "COMPLETED MODEL GID=%d; EVALUATED %d MODELs" , modelIDToRun , numModelsTotal )
        print >> sys . stderr , "reporter:status:Evaluated %d models..." % ( numModelsTotal )
        print >> sys . stderr , "reporter:counter:HypersearchWorker,numModels,1"
        if options . modelID is not None :
            exit = True
        # ^ ^ ^ end while not exit
finally : # Provide Hypersearch instance an opportunity to clean up temporary files
    self . _hs . close ( )
self . logger . info ( "FINISHED. Evaluated %d models." % ( numModelsTotal ) )
print >> sys . stderr , "reporter:status:Finished, evaluated %d models" % ( numModelsTotal )
return options . jobID


signature:
def getBucketIndices ( self , x ) :
    """See method description in base . py"""

implementation:
if ( ( isinstance ( x , float ) and math . isnan ( x ) ) or x == SENTINEL_VALUE_FOR_MISSING_DATA ) :
    return [ None ]
if self . _offset is None :
    self . _offset = x
bucketIdx = ( ( self . _maxBuckets / 2 ) + int ( round ( ( x - self . _offset ) / self . resolution ) ) )
if bucketIdx < 0 :
    bucketIdx = 0
elif bucketIdx >= self . _maxBuckets :
    bucketIdx = self . _maxBuckets - 1
return [ bucketIdx ]


signature:
def mapBucketIndexToNonZeroBits ( self , index ) :
    """Given a bucket index , return the list of non - zero bits . If the bucket
 index does not exist , it is created . If the index falls outside our range
 we clip it .
 : param index The bucket index to get non - zero bits for .
 @ returns numpy array of indices of non - zero bits for specified index ."""

implementation:
if index < 0 :
    index = 0
if index >= self . _maxBuckets :
    index = self . _maxBuckets - 1
if not self . bucketMap . has_key ( index ) :
    if self . verbosity >= 2 :
        print "Adding additional buckets to handle index=" , index
    self . _createBucket ( index )
return self . bucketMap [ index ]


signature:
def encodeIntoArray ( self , x , output ) :
    """See method description in base . py"""

implementation:
if x is not None and not isinstance ( x , numbers . Number ) :
    raise TypeError ( "Expected a scalar input but got input of type %s" % type ( x ) )
# Get the bucket index to use
bucketIdx = self . getBucketIndices ( x ) [ 0 ]
# None is returned for missing value in which case we return all 0 ' s .
output [ 0 : self . n ] = 0
if bucketIdx is not None :
    output [ self . mapBucketIndexToNonZeroBits ( bucketIdx ) ] = 1


signature:
def _createBucket ( self , index ) :
    """Create the given bucket index . Recursively create as many in - between
 bucket indices as necessary ."""

implementation:
if index < self . minIndex :
    if index == self . minIndex - 1 : # Create a new representation that has exactly w - 1 overlapping bits
    # as the min representation
        self . bucketMap [ index ] = self . _newRepresentation ( self . minIndex , index )
        self . minIndex = index
    else : # Recursively create all the indices above and then this index
        self . _createBucket ( index + 1 )
        self . _createBucket ( index )
else :
    if index == self . maxIndex + 1 : # Create a new representation that has exactly w - 1 overlapping bits
    # as the max representation
        self . bucketMap [ index ] = self . _newRepresentation ( self . maxIndex , index )
        self . maxIndex = index
    else : # Recursively create all the indices below and then this index
        self . _createBucket ( index - 1 )
        self . _createBucket ( index )


signature:
def _newRepresentation ( self , index , newIndex ) :
    """Return a new representation for newIndex that overlaps with the
 representation at index by exactly w - 1 bits"""

implementation:
newRepresentation = self . bucketMap [ index ] . copy ( )
# Choose the bit we will replace in this representation . We need to shift
# this bit deterministically . If this is always chosen randomly then there
# is a 1 in w chance of the same bit being replaced in neighboring
# representations , which is fairly high
ri = newIndex % self . w
# Now we choose a bit such that the overlap rules are satisfied .
newBit = self . random . getUInt32 ( self . n )
newRepresentation [ ri ] = newBit
while newBit in self . bucketMap [ index ] or not self . _newRepresentationOK ( newRepresentation , newIndex ) :
    self . numTries += 1
    newBit = self . random . getUInt32 ( self . n )
    newRepresentation [ ri ] = newBit
return newRepresentation


signature:
def _newRepresentationOK ( self , newRep , newIndex ) :
    """Return True if this new candidate representation satisfies all our overlap
 rules . Since we know that neighboring representations differ by at most
 one bit , we compute running overlaps ."""

implementation:
if newRep . size != self . w :
    return False
if ( newIndex < self . minIndex - 1 ) or ( newIndex > self . maxIndex + 1 ) :
    raise ValueError ( "newIndex must be within one of existing indices" )
# A binary representation of newRep . We will use this to test containment
newRepBinary = numpy . array ( [ False ] * self . n )
newRepBinary [ newRep ] = True
# Midpoint
midIdx = self . _maxBuckets / 2
# Start by checking the overlap at minIndex
runningOverlap = self . _countOverlap ( self . bucketMap [ self . minIndex ] , newRep )
if not self . _overlapOK ( self . minIndex , newIndex , overlap = runningOverlap ) :
    return False
# Compute running overlaps all the way to the midpoint
for i in range ( self . minIndex + 1 , midIdx + 1 ) : # This is the bit that is going to change
    newBit = ( i - 1 ) % self . w
    # Update our running overlap
    if newRepBinary [ self . bucketMap [ i - 1 ] [ newBit ] ] :
        runningOverlap -= 1
    if newRepBinary [ self . bucketMap [ i ] [ newBit ] ] :
        runningOverlap += 1
    # Verify our rules
    if not self . _overlapOK ( i , newIndex , overlap = runningOverlap ) :
        return False
# At this point , runningOverlap contains the overlap for midIdx
# Compute running overlaps all the way to maxIndex
for i in range ( midIdx + 1 , self . maxIndex + 1 ) : # This is the bit that is going to change
    newBit = i % self . w
    # Update our running overlap
    if newRepBinary [ self . bucketMap [ i - 1 ] [ newBit ] ] :
        runningOverlap -= 1
    if newRepBinary [ self . bucketMap [ i ] [ newBit ] ] :
        runningOverlap += 1
    # Verify our rules
    if not self . _overlapOK ( i , newIndex , overlap = runningOverlap ) :
        return False
return True


signature:
def _countOverlapIndices ( self , i , j ) :
    """Return the overlap between bucket indices i and j"""

implementation:
if self . bucketMap . has_key ( i ) and self . bucketMap . has_key ( j ) :
    iRep = self . bucketMap [ i ]
    jRep = self . bucketMap [ j ]
    return self . _countOverlap ( iRep , jRep )
else :
    raise ValueError ( "Either i or j don't exist" )


signature:
def _countOverlap ( rep1 , rep2 ) :
    """Return the overlap between two representations . rep1 and rep2 are lists of
 non - zero indices ."""

implementation:
overlap = 0
for e in rep1 :
    if e in rep2 :
        overlap += 1
return overlap


signature:
def _overlapOK ( self , i , j , overlap = None ) :
    """Return True if the given overlap between bucket indices i and j are
 acceptable . If overlap is not specified , calculate it from the bucketMap"""

implementation:
if overlap is None :
    overlap = self . _countOverlapIndices ( i , j )
if abs ( i - j ) < self . w :
    if overlap == ( self . w - abs ( i - j ) ) :
        return True
    else :
        return False
else :
    if overlap <= self . _maxOverlap :
        return True
    else :
        return False


signature:
def _initializeBucketMap ( self , maxBuckets , offset ) :
    """Initialize the bucket map assuming the given number of maxBuckets ."""

implementation:
# The first bucket index will be _ maxBuckets / 2 and bucket indices will be
# allowed to grow lower or higher as long as they don ' t become negative .
# _ maxBuckets is required because the current SDR Classifier assumes bucket
# indices must be non - negative . This normally does not need to be changed
# but if altered , should be set to an even number .
self . _maxBuckets = maxBuckets
self . minIndex = self . _maxBuckets / 2
self . maxIndex = self . _maxBuckets / 2
# The scalar offset used to map scalar values to bucket indices . The middle
# bucket will correspond to numbers in the range
# [ offset - resolution / 2 , offset + resolution / 2 ) .
# The bucket index for a number x will be :
# maxBuckets / 2 + int ( round ( ( x - offset ) / resolution ) )
self . _offset = offset
# This dictionary maps a bucket index into its bit representation
# We initialize the class with a single bucket with index 0
self . bucketMap = { }
def _permutation ( n ) :
    r = numpy . arange ( n , dtype = numpy . uint32 )
    self . random . shuffle ( r )
    return r
self . bucketMap [ self . minIndex ] = _permutation ( self . n ) [ 0 : self . w ]
# How often we need to retry when generating valid encodings
self . numTries = 0


signature:
def retrySQL ( timeoutSec = 60 * 5 , logger = None ) :
    """Return a closure suitable for use as a decorator for
 retrying a pymysql DAO function on certain failures that warrant retries (
 e . g . , RDS / MySQL server down temporarily , transaction deadlock , etc . ) .
 We share this function across multiple scripts ( e . g . , ClientJobsDAO ,
 StreamMgr ) for consitent behavior .
 . . note : : Please ensure that the operation being retried is idempotent .
 . . note : : logging must be initialized * before * any loggers are created , else
 there will be no output ; see nupic . support . initLogging ( )
 Usage Example :
 . . code - block : : python
 @ retrySQL ( )
 def jobInfo ( self , jobID ) :
 : param timeoutSec : How many seconds from time of initial call to stop retrying
 ( floating point )
 : param logger : User - supplied logger instance ."""

implementation:
if logger is None :
    logger = logging . getLogger ( __name__ )
def retryFilter ( e , args , kwargs ) :
    if isinstance ( e , ( pymysql . InternalError , pymysql . OperationalError ) ) :
        if e . args and e . args [ 0 ] in _ALL_RETRIABLE_ERROR_CODES :
            return True
    elif isinstance ( e , pymysql . Error ) :
        if ( e . args and inspect . isclass ( e . args [ 0 ] ) and issubclass ( e . args [ 0 ] , socket_error ) ) :
            return True
    return False
retryExceptions = tuple ( [ pymysql . InternalError , pymysql . OperationalError , pymysql . Error , ] )
return make_retry_decorator ( timeoutSec = timeoutSec , initialRetryDelaySec = 0.1 , maxRetryDelaySec = 10 , retryExceptions = retryExceptions , retryFilter = retryFilter , logger = logger )


signature:
def create ( * args , ** kwargs ) :
    """Create a SDR classifier factory .
 The implementation of the SDR Classifier can be specified with
 the " implementation " keyword argument .
 The SDRClassifierFactory uses the implementation as specified in
 ` Default NuPIC Configuration < default - config . html > ` _ ."""

implementation:
impl = kwargs . pop ( 'implementation' , None )
if impl is None :
    impl = Configuration . get ( 'nupic.opf.sdrClassifier.implementation' )
if impl == 'py' :
    return SDRClassifier ( * args , ** kwargs )
elif impl == 'cpp' :
    return FastSDRClassifier ( * args , ** kwargs )
elif impl == 'diff' :
    return SDRClassifierDiff ( * args , ** kwargs )
else :
    raise ValueError ( 'Invalid classifier implementation (%r). Value must be ' '"py", "cpp" or "diff".' % impl )


signature:
def read ( proto ) :
    """: param proto : SDRClassifierRegionProto capnproto object"""

implementation:
impl = proto . implementation
if impl == 'py' :
    return SDRClassifier . read ( proto . sdrClassifier )
elif impl == 'cpp' :
    return FastSDRClassifier . read ( proto . sdrClassifier )
elif impl == 'diff' :
    return SDRClassifierDiff . read ( proto . sdrClassifier )
else :
    raise ValueError ( 'Invalid classifier implementation (%r). Value must be ' '"py", "cpp" or "diff".' % impl )


signature:
def cross_list ( * sequences ) :
    """From : http : / / book . opensourceproject . org . cn / lamp / python / pythoncook2 / opensource / 0596007973 / pythoncook2 - chp - 19 - sect - 9 . html"""

implementation:
result = [ [ ] ]
for seq in sequences :
    result = [ sublist + [ item ] for sublist in result for item in seq ]
return result


signature:
def cross ( * sequences ) :
    """From : http : / / book . opensourceproject . org . cn / lamp / python / pythoncook2 / opensource / 0596007973 / pythoncook2 - chp - 19 - sect - 9 . html"""

implementation:
# visualize an odometer , with " wheels " displaying " digits " . . . :
wheels = map ( iter , sequences )
digits = [ it . next ( ) for it in wheels ]
while True :
    yield tuple ( digits )
    for i in range ( len ( digits ) - 1 , - 1 , - 1 ) :
        try :
            digits [ i ] = wheels [ i ] . next ( )
            break
        except StopIteration :
            wheels [ i ] = iter ( sequences [ i ] )
            digits [ i ] = wheels [ i ] . next ( )
    else :
        break


signature:
def dcross ( ** keywords ) :
    """Similar to cross ( ) , but generates output dictionaries instead of tuples ."""

implementation:
keys = keywords . keys ( )
# Could use keywords . values ( ) , but unsure whether the order
# the values come out in is guaranteed to be the same as that of keys
# ( appears to be anecdotally true ) .
sequences = [ keywords [ key ] for key in keys ]
wheels = map ( iter , sequences )
digits = [ it . next ( ) for it in wheels ]
while True :
    yield dict ( zip ( keys , digits ) )
    for i in range ( len ( digits ) - 1 , - 1 , - 1 ) :
        try :
            digits [ i ] = wheels [ i ] . next ( )
            break
        except StopIteration :
            wheels [ i ] = iter ( sequences [ i ] )
            digits [ i ] = wheels [ i ] . next ( )
    else :
        break


signature:
def mmGetMetricFromTrace ( self , trace ) :
    """Convenience method to compute a metric over an indices trace , excluding
 resets .
 @ param ( IndicesTrace ) Trace of indices
 @ return ( Metric ) Metric over trace excluding resets"""

implementation: return Metric . createFromTrace ( trace . makeCountsTrace ( ) , excludeResets = self . mmGetTraceResets ( ) )


signature:
def mmGetMetricSequencesPredictedActiveCellsPerColumn ( self ) :
    """Metric for number of predicted = > active cells per column for each sequence
 @ return ( Metric ) metric"""

implementation:
self . _mmComputeTransitionTraces ( )
numCellsPerColumn = [ ]
for predictedActiveCells in ( self . _mmData [ "predictedActiveCellsForSequence" ] . values ( ) ) :
    cellsForColumn = self . mapCellsToColumns ( predictedActiveCells )
    numCellsPerColumn += [ len ( x ) for x in cellsForColumn . values ( ) ]
return Metric ( self , "# predicted => active cells per column for each sequence" , numCellsPerColumn )


signature:
def mmGetMetricSequencesPredictedActiveCellsShared ( self ) :
    """Metric for number of sequences each predicted = > active cell appears in
 Note : This metric is flawed when it comes to high - order sequences .
 @ return ( Metric ) metric"""

implementation:
self . _mmComputeTransitionTraces ( )
numSequencesForCell = defaultdict ( lambda : 0 )
for predictedActiveCells in ( self . _mmData [ "predictedActiveCellsForSequence" ] . values ( ) ) :
    for cell in predictedActiveCells :
        numSequencesForCell [ cell ] += 1
return Metric ( self , "# sequences each predicted => active cells appears in" , numSequencesForCell . values ( ) )


signature:
def mmPrettyPrintConnections ( self ) :
    """Pretty print the connections in the temporal memory .
 TODO : Use PrettyTable .
 @ return ( string ) Pretty - printed text"""

implementation:
text = ""
text += ( "Segments: (format => " "(#) [(source cell=permanence ...),       ...]\n" )
text += "------------------------------------\n"
columns = range ( self . numberOfColumns ( ) )
for column in columns :
    cells = self . cellsForColumn ( column )
    for cell in cells :
        segmentDict = dict ( )
        for seg in self . connections . segmentsForCell ( cell ) :
            synapseList = [ ]
            for synapse in self . connections . synapsesForSegment ( seg ) :
                synapseData = self . connections . dataForSynapse ( synapse )
                synapseList . append ( ( synapseData . presynapticCell , synapseData . permanence ) )
            synapseList . sort ( )
            synapseStringList = [ "{0:3}={1:.2f}" . format ( sourceCell , permanence ) for sourceCell , permanence in synapseList ]
            segmentDict [ seg ] = "({0})" . format ( " " . join ( synapseStringList ) )
        text += ( "Column {0:3} / Cell {1:3}:\t({2}) {3}\n" . format ( column , cell , len ( segmentDict . values ( ) ) , "[{0}]" . format ( ",       " . join ( segmentDict . values ( ) ) ) ) )
    if column < len ( columns ) - 1 : # not last
        text += "\n"
text += "------------------------------------\n"
return text


signature:
def mmPrettyPrintSequenceCellRepresentations ( self , sortby = "Column" ) :
    """Pretty print the cell representations for sequences in the history .
 @ param sortby ( string ) Column of table to sort by
 @ return ( string ) Pretty - printed text"""

implementation:
self . _mmComputeTransitionTraces ( )
table = PrettyTable ( [ "Pattern" , "Column" , "predicted=>active cells" ] )
for sequenceLabel , predictedActiveCells in ( self . _mmData [ "predictedActiveCellsForSequence" ] . iteritems ( ) ) :
    cellsForColumn = self . mapCellsToColumns ( predictedActiveCells )
    for column , cells in cellsForColumn . iteritems ( ) :
        table . add_row ( [ sequenceLabel , column , list ( cells ) ] )
return table . get_string ( sortby = sortby ) . encode ( "utf-8" )


signature:
def createTemporalAnomaly ( recordParams , spatialParams = _SP_PARAMS , temporalParams = _TM_PARAMS , verbosity = _VERBOSITY ) :
    """Generates a Network with connected RecordSensor , SP , TM .
 This function takes care of generating regions and the canonical links .
 The network has a sensor region reading data from a specified input and
 passing the encoded representation to an SPRegion .
 The SPRegion output is passed to a TMRegion .
 Note : this function returns a network that needs to be initialized . This
 allows the user to extend the network by adding further regions and
 connections .
 : param recordParams : a dict with parameters for creating RecordSensor region .
 : param spatialParams : a dict with parameters for creating SPRegion .
 : param temporalParams : a dict with parameters for creating TMRegion .
 : param verbosity : an integer representing how chatty the network will be ."""

implementation:
inputFilePath = recordParams [ "inputFilePath" ]
scalarEncoderArgs = recordParams [ "scalarEncoderArgs" ]
dateEncoderArgs = recordParams [ "dateEncoderArgs" ]
scalarEncoder = ScalarEncoder ( ** scalarEncoderArgs )
dateEncoder = DateEncoder ( ** dateEncoderArgs )
encoder = MultiEncoder ( )
encoder . addEncoder ( scalarEncoderArgs [ "name" ] , scalarEncoder )
encoder . addEncoder ( dateEncoderArgs [ "name" ] , dateEncoder )
network = Network ( )
network . addRegion ( "sensor" , "py.RecordSensor" , json . dumps ( { "verbosity" : verbosity } ) )
sensor = network . regions [ "sensor" ] . getSelf ( )
sensor . encoder = encoder
sensor . dataSource = FileRecordStream ( streamID = inputFilePath )
# Create the spatial pooler region
spatialParams [ "inputWidth" ] = sensor . encoder . getWidth ( )
network . addRegion ( "spatialPoolerRegion" , "py.SPRegion" , json . dumps ( spatialParams ) )
# Link the SP region to the sensor input
network . link ( "sensor" , "spatialPoolerRegion" , "UniformLink" , "" )
network . link ( "sensor" , "spatialPoolerRegion" , "UniformLink" , "" , srcOutput = "resetOut" , destInput = "resetIn" )
network . link ( "spatialPoolerRegion" , "sensor" , "UniformLink" , "" , srcOutput = "spatialTopDownOut" , destInput = "spatialTopDownIn" )
network . link ( "spatialPoolerRegion" , "sensor" , "UniformLink" , "" , srcOutput = "temporalTopDownOut" , destInput = "temporalTopDownIn" )
# Add the TPRegion on top of the SPRegion
network . addRegion ( "temporalPoolerRegion" , "py.TMRegion" , json . dumps ( temporalParams ) )
network . link ( "spatialPoolerRegion" , "temporalPoolerRegion" , "UniformLink" , "" )
network . link ( "temporalPoolerRegion" , "spatialPoolerRegion" , "UniformLink" , "" , srcOutput = "topDownOut" , destInput = "topDownIn" )
spatialPoolerRegion = network . regions [ "spatialPoolerRegion" ]
# Make sure learning is enabled
spatialPoolerRegion . setParameter ( "learningMode" , True )
# We want temporal anomalies so disable anomalyMode in the SP . This mode is
# used for computing anomalies in a non - temporal model .
spatialPoolerRegion . setParameter ( "anomalyMode" , False )
temporalPoolerRegion = network . regions [ "temporalPoolerRegion" ]
# Enable topDownMode to get the predicted columns output
temporalPoolerRegion . setParameter ( "topDownMode" , True )
# Make sure learning is enabled ( this is the default )
temporalPoolerRegion . setParameter ( "learningMode" , True )
# Enable inference mode so we get predictions
temporalPoolerRegion . setParameter ( "inferenceMode" , True )
# Enable anomalyMode to compute the anomaly score .
temporalPoolerRegion . setParameter ( "anomalyMode" , True )
return network


signature:
def runNetwork ( network , writer ) :
    """Run the network and write output to writer .
 : param network : a Network instance to run
 : param writer : a csv . writer instance to write output to"""

implementation:
sensorRegion = network . regions [ "sensor" ]
temporalPoolerRegion = network . regions [ "temporalPoolerRegion" ]
for i in xrange ( _NUM_RECORDS ) : # Run the network for a single iteration
    network . run ( 1 )
    # Write out the anomaly score along with the record number and consumption
    # value .
    anomalyScore = temporalPoolerRegion . getOutputData ( "anomalyScore" ) [ 0 ]
    consumption = sensorRegion . getOutputData ( "sourceOut" ) [ 0 ]
    writer . writerow ( ( i , consumption , anomalyScore ) )


signature:
def __appendActivities ( self , periodicActivities ) :
    """periodicActivities : A sequence of PeriodicActivityRequest elements"""

implementation:
for req in periodicActivities :
    act = self . Activity ( repeating = req . repeating , period = req . period , cb = req . cb , iteratorHolder = [ iter ( xrange ( req . period - 1 ) ) ] )
    self . __activities . append ( act )
return


signature:
def add ( reader , writer , column , start , stop , value ) :
    """Adds a value over a range of rows .
 Args :
 reader : A FileRecordStream object with input data .
 writer : A FileRecordStream object to write output data to .
 column : The column of data to modify .
 start : The first row in the range to modify .
 end : The last row in the range to modify .
 value : The value to add ."""

implementation:
for i , row in enumerate ( reader ) :
    if i >= start and i <= stop :
        row [ column ] = type ( value ) ( row [ column ] ) + value
    writer . appendRecord ( row )


signature:
def scale ( reader , writer , column , start , stop , multiple ) :
    """Multiplies a value over a range of rows .
 Args :
 reader : A FileRecordStream object with input data .
 writer : A FileRecordStream object to write output data to .
 column : The column of data to modify .
 start : The first row in the range to modify .
 end : The last row in the range to modify .
 multiple : The value to scale / multiply by ."""

implementation:
for i , row in enumerate ( reader ) :
    if i >= start and i <= stop :
        row [ column ] = type ( multiple ) ( row [ column ] ) * multiple
    writer . appendRecord ( row )


signature:
def copy ( reader , writer , start , stop , insertLocation = None , tsCol = None ) :
    """Copies a range of values to a new location in the data set .
 Args :
 reader : A FileRecordStream object with input data .
 writer : A FileRecordStream object to write output data to .
 start : The first row in the range to copy .
 stop : The last row in the range to copy .
 insertLocation : The location to insert the copied range . If not specified ,
 the range is inserted immediately following itself ."""

implementation:
assert stop >= start
startRows = [ ]
copyRows = [ ]
ts = None
inc = None
if tsCol is None :
    tsCol = reader . getTimestampFieldIdx ( )
for i , row in enumerate ( reader ) : # Get the first timestamp and the increment .
    if ts is None :
        ts = row [ tsCol ]
    elif inc is None :
        inc = row [ tsCol ] - ts
    # Keep a list of all rows and a list of rows to copy .
    if i >= start and i <= stop :
        copyRows . append ( row )
    startRows . append ( row )
# Insert the copied rows .
if insertLocation is None :
    insertLocation = stop + 1
startRows [ insertLocation : insertLocation ] = copyRows
# Update the timestamps .
for row in startRows :
    row [ tsCol ] = ts
    writer . appendRecord ( row )
    ts += inc


signature:
def sample ( reader , writer , n , start = None , stop = None , tsCol = None , writeSampleOnly = True ) :
    """Samples n rows .
 Args :
 reader : A FileRecordStream object with input data .
 writer : A FileRecordStream object to write output data to .
 n : The number of elements to sample .
 start : The first row in the range to sample from .
 stop : The last row in the range to sample from .
 tsCol : If specified , the timestamp column to update .
 writeSampleOnly : If False , the rows before start are written before the
 sample and the rows after stop are written after the sample ."""

implementation:
rows = list ( reader )
if tsCol is not None :
    ts = rows [ 0 ] [ tsCol ]
    inc = rows [ 1 ] [ tsCol ] - ts
if start is None :
    start = 0
if stop is None :
    stop = len ( rows ) - 1
initialN = stop - start + 1
# Select random rows in the sample range to delete until the desired number
# of rows are left .
numDeletes = initialN - n
for i in xrange ( numDeletes ) :
    delIndex = random . randint ( start , stop - i )
    del rows [ delIndex ]
# Remove outside rows if specified .
if writeSampleOnly :
    rows = rows [ start : start + n ]
# Rewrite columns if tsCol is given .
if tsCol is not None :
    ts = rows [ 0 ] [ tsCol ]
# Write resulting rows .
for row in rows :
    if tsCol is not None :
        row [ tsCol ] = ts
        ts += inc
    writer . appendRecord ( row )


signature:
def _initEncoder ( self , w , minval , maxval , n , radius , resolution ) :
    """( helper function ) There are three different ways of thinking about the representation .
 Handle each case here ."""

implementation:
if n != 0 :
    if ( radius != 0 or resolution != 0 ) :
        raise ValueError ( "Only one of n/radius/resolution can be specified for a ScalarEncoder" )
    assert n > w
    self . n = n
    if ( minval is not None and maxval is not None ) :
        if not self . periodic :
            self . resolution = float ( self . rangeInternal ) / ( self . n - self . w )
        else :
            self . resolution = float ( self . rangeInternal ) / ( self . n )
        self . radius = self . w * self . resolution
        if self . periodic :
            self . range = self . rangeInternal
        else :
            self . range = self . rangeInternal + self . resolution
else :
    if radius != 0 :
        if ( resolution != 0 ) :
            raise ValueError ( "Only one of radius/resolution can be specified for a ScalarEncoder" )
        self . radius = radius
        self . resolution = float ( self . radius ) / w
    elif resolution != 0 :
        self . resolution = float ( resolution )
        self . radius = self . resolution * self . w
    else :
        raise Exception ( "One of n, radius, resolution must be specified for a ScalarEncoder" )
    if ( minval is not None and maxval is not None ) :
        if self . periodic :
            self . range = self . rangeInternal
        else :
            self . range = self . rangeInternal + self . resolution
        nfloat = self . w * ( self . range / self . radius ) + 2 * self . padding
        self . n = int ( math . ceil ( nfloat ) )


signature:
def _getFirstOnBit ( self , input ) :
    """Return the bit offset of the first bit to be set in the encoder output .
 For periodic encoders , this can be a negative number when the encoded output
 wraps around ."""

implementation:
if input == SENTINEL_VALUE_FOR_MISSING_DATA :
    return [ None ]
else :
    if input < self . minval : # Don ' t clip periodic inputs . Out - of - range input is always an error
        if self . clipInput and not self . periodic :
            if self . verbosity > 0 :
                print "Clipped input %s=%.2f to minval %.2f" % ( self . name , input , self . minval )
            input = self . minval
        else :
            raise Exception ( 'input (%s) less than range (%s - %s)' % ( str ( input ) , str ( self . minval ) , str ( self . maxval ) ) )
    if self . periodic : # Don ' t clip periodic inputs . Out - of - range input is always an error
        if input >= self . maxval :
            raise Exception ( 'input (%s) greater than periodic range (%s - %s)' % ( str ( input ) , str ( self . minval ) , str ( self . maxval ) ) )
    else :
        if input > self . maxval :
            if self . clipInput :
                if self . verbosity > 0 :
                    print "Clipped input %s=%.2f to maxval %.2f" % ( self . name , input , self . maxval )
                input = self . maxval
            else :
                raise Exception ( 'input (%s) greater than range (%s - %s)' % ( str ( input ) , str ( self . minval ) , str ( self . maxval ) ) )
    if self . periodic :
        centerbin = int ( ( input - self . minval ) * self . nInternal / self . range ) + self . padding
    else :
        centerbin = int ( ( ( input - self . minval ) + self . resolution / 2 ) / self . resolution ) + self . padding
    # We use the first bit to be set in the encoded output as the bucket index
    minbin = centerbin - self . halfwidth
    return [ minbin ]


signature:
def getBucketIndices ( self , input ) :
    """See method description in base . py"""

implementation:
if type ( input ) is float and math . isnan ( input ) :
    input = SENTINEL_VALUE_FOR_MISSING_DATA
if input == SENTINEL_VALUE_FOR_MISSING_DATA :
    return [ None ]
minbin = self . _getFirstOnBit ( input ) [ 0 ]
# For periodic encoders , the bucket index is the index of the center bit
if self . periodic :
    bucketIdx = minbin + self . halfwidth
    if bucketIdx < 0 :
        bucketIdx += self . n
# for non - periodic encoders , the bucket index is the index of the left bit
else :
    bucketIdx = minbin
return [ bucketIdx ]


signature:
def encodeIntoArray ( self , input , output , learn = True ) :
    """See method description in base . py"""

implementation:
if input is not None and not isinstance ( input , numbers . Number ) :
    raise TypeError ( "Expected a scalar input but got input of type %s" % type ( input ) )
if type ( input ) is float and math . isnan ( input ) :
    input = SENTINEL_VALUE_FOR_MISSING_DATA
# Get the bucket index to use
bucketIdx = self . _getFirstOnBit ( input ) [ 0 ]
if bucketIdx is None : # None is returned for missing value
    output [ 0 : self . n ] = 0
# TODO : should all 1s , or random SDR be returned instead ?
else : # The bucket index is the index of the first bit to set in the output
    output [ : self . n ] = 0
    minbin = bucketIdx
    maxbin = minbin + 2 * self . halfwidth
    if self . periodic : # Handle the edges by computing wrap - around
        if maxbin >= self . n :
            bottombins = maxbin - self . n + 1
            output [ : bottombins ] = 1
            maxbin = self . n - 1
        if minbin < 0 :
            topbins = - minbin
            output [ self . n - topbins : self . n ] = 1
            minbin = 0
    assert minbin >= 0
    assert maxbin < self . n
    # set the output ( except for periodic wraparound )
    output [ minbin : maxbin + 1 ] = 1
# Debug the decode ( ) method
if self . verbosity >= 2 :
    print
    print "input:" , input
    print "range:" , self . minval , "-" , self . maxval
    print "n:" , self . n , "w:" , self . w , "resolution:" , self . resolution , "radius" , self . radius , "periodic:" , self . periodic
    print "output:" ,
    self . pprint ( output )
    print "input desc:" , self . decodedToStr ( self . decode ( output ) )


signature:
def decode ( self , encoded , parentFieldName = '' ) :
    """See the function description in base . py"""

implementation:
# For now , we simply assume any top - down output greater than 0
# is ON . Eventually , we will probably want to incorporate the strength
# of each top - down output .
tmpOutput = numpy . array ( encoded [ : self . n ] > 0 ) . astype ( encoded . dtype )
if not tmpOutput . any ( ) :
    return ( dict ( ) , [ ] )
# First , assume the input pool is not sampled 100 % , and fill in the
# " holes " in the encoded representation ( which are likely to be present
# if this is a coincidence that was learned by the SP ) .
# Search for portions of the output that have " holes "
maxZerosInARow = self . halfwidth
for i in xrange ( maxZerosInARow ) :
    searchStr = numpy . ones ( i + 3 , dtype = encoded . dtype )
    searchStr [ 1 : - 1 ] = 0
    subLen = len ( searchStr )
    # Does this search string appear in the output ?
    if self . periodic :
        for j in xrange ( self . n ) :
            outputIndices = numpy . arange ( j , j + subLen )
            outputIndices %= self . n
            if numpy . array_equal ( searchStr , tmpOutput [ outputIndices ] ) :
                tmpOutput [ outputIndices ] = 1
    else :
        for j in xrange ( self . n - subLen + 1 ) :
            if numpy . array_equal ( searchStr , tmpOutput [ j : j + subLen ] ) :
                tmpOutput [ j : j + subLen ] = 1
if self . verbosity >= 2 :
    print "raw output:" , encoded [ : self . n ]
    print "filtered output:" , tmpOutput
# Find each run of 1 ' s .
nz = tmpOutput . nonzero ( ) [ 0 ]
runs = [ ]
# will be tuples of ( startIdx , runLength )
run = [ nz [ 0 ] , 1 ]
i = 1
while ( i < len ( nz ) ) :
    if nz [ i ] == run [ 0 ] + run [ 1 ] :
        run [ 1 ] += 1
    else :
        runs . append ( run )
        run = [ nz [ i ] , 1 ]
    i += 1
runs . append ( run )
# If we have a periodic encoder , merge the first and last run if they
# both go all the way to the edges
if self . periodic and len ( runs ) > 1 :
    if runs [ 0 ] [ 0 ] == 0 and runs [ - 1 ] [ 0 ] + runs [ - 1 ] [ 1 ] == self . n :
        runs [ - 1 ] [ 1 ] += runs [ 0 ] [ 1 ]
        runs = runs [ 1 : ]
# Now , for each group of 1 ' s , determine the " left " and " right " edges , where
# the " left " edge is inset by halfwidth and the " right " edge is inset by
# halfwidth .
# For a group of width w or less , the " left " and " right " edge are both at
# the center position of the group .
ranges = [ ]
for run in runs :
    ( start , runLen ) = run
    if runLen <= self . w :
        left = right = start + runLen / 2
    else :
        left = start + self . halfwidth
        right = start + runLen - 1 - self . halfwidth
    # Convert to input space .
    if not self . periodic :
        inMin = ( left - self . padding ) * self . resolution + self . minval
        inMax = ( right - self . padding ) * self . resolution + self . minval
    else :
        inMin = ( left - self . padding ) * self . range / self . nInternal + self . minval
        inMax = ( right - self . padding ) * self . range / self . nInternal + self . minval
    # Handle wrap - around if periodic
    if self . periodic :
        if inMin >= self . maxval :
            inMin -= self . range
            inMax -= self . range
    # Clip low end
    if inMin < self . minval :
        inMin = self . minval
    if inMax < self . minval :
        inMax = self . minval
    # If we have a periodic encoder , and the max is past the edge , break into
    # 2 separate ranges
    if self . periodic and inMax >= self . maxval :
        ranges . append ( [ inMin , self . maxval ] )
        ranges . append ( [ self . minval , inMax - self . range ] )
    else :
        if inMax > self . maxval :
            inMax = self . maxval
        if inMin > self . maxval :
            inMin = self . maxval
        ranges . append ( [ inMin , inMax ] )
desc = self . _generateRangeDescription ( ranges )
# Return result
if parentFieldName != '' :
    fieldName = "%s.%s" % ( parentFieldName , self . name )
else :
    fieldName = self . name
return ( { fieldName : ( ranges , desc ) } , [ fieldName ] )


signature:
def _generateRangeDescription ( self , ranges ) :
    """generate description from a text description of the ranges"""

implementation:
desc = ""
numRanges = len ( ranges )
for i in xrange ( numRanges ) :
    if ranges [ i ] [ 0 ] != ranges [ i ] [ 1 ] :
        desc += "%.2f-%.2f" % ( ranges [ i ] [ 0 ] , ranges [ i ] [ 1 ] )
    else :
        desc += "%.2f" % ( ranges [ i ] [ 0 ] )
    if i < numRanges - 1 :
        desc += ", "
return desc


signature:
def _getTopDownMapping ( self ) :
    """Return the interal _ topDownMappingM matrix used for handling the
 bucketInfo ( ) and topDownCompute ( ) methods . This is a matrix , one row per
 category ( bucket ) where each row contains the encoded output for that
 category ."""

implementation:
# Do we need to build up our reverse mapping table ?
if self . _topDownMappingM is None : # The input scalar value corresponding to each possible output encoding
    if self . periodic :
        self . _topDownValues = numpy . arange ( self . minval + self . resolution / 2.0 , self . maxval , self . resolution )
    else : # Number of values is ( max - min ) / resolutions
        self . _topDownValues = numpy . arange ( self . minval , self . maxval + self . resolution / 2.0 , self . resolution )
    # Each row represents an encoded output pattern
    numCategories = len ( self . _topDownValues )
    self . _topDownMappingM = SM32 ( numCategories , self . n )
    outputSpace = numpy . zeros ( self . n , dtype = GetNTAReal ( ) )
    for i in xrange ( numCategories ) :
        value = self . _topDownValues [ i ]
        value = max ( value , self . minval )
        value = min ( value , self . maxval )
        self . encodeIntoArray ( value , outputSpace , learn = False )
        self . _topDownMappingM . setRowFromDense ( i , outputSpace )
return self . _topDownMappingM


signature:
def getBucketValues ( self ) :
    """See the function description in base . py"""

implementation:
# Need to re - create ?
if self . _bucketValues is None :
    topDownMappingM = self . _getTopDownMapping ( )
    numBuckets = topDownMappingM . nRows ( )
    self . _bucketValues = [ ]
    for bucketIdx in range ( numBuckets ) :
        self . _bucketValues . append ( self . getBucketInfo ( [ bucketIdx ] ) [ 0 ] . value )
return self . _bucketValues


signature:
def getBucketInfo ( self , buckets ) :
    """See the function description in base . py"""

implementation:
# Get / generate the topDown mapping table
# NOTE : although variable topDownMappingM is unused , some ( bad - style ) actions
# are executed during _ getTopDownMapping ( ) so this line must stay here
topDownMappingM = self . _getTopDownMapping ( )
# The " category " is simply the bucket index
category = buckets [ 0 ]
encoding = self . _topDownMappingM . getRow ( category )
# Which input value does this correspond to ?
if self . periodic :
    inputVal = ( self . minval + ( self . resolution / 2.0 ) + ( category * self . resolution ) )
else :
    inputVal = self . minval + ( category * self . resolution )
return [ EncoderResult ( value = inputVal , scalar = inputVal , encoding = encoding ) ]


signature:
def topDownCompute ( self , encoded ) :
    """See the function description in base . py"""

implementation:
# Get / generate the topDown mapping table
topDownMappingM = self . _getTopDownMapping ( )
# See which " category " we match the closest .
category = topDownMappingM . rightVecProd ( encoded ) . argmax ( )
# Return that bucket info
return self . getBucketInfo ( [ category ] )


signature:
def closenessScores ( self , expValues , actValues , fractional = True ) :
    """See the function description in base . py"""

implementation:
expValue = expValues [ 0 ]
actValue = actValues [ 0 ]
if self . periodic :
    expValue = expValue % self . maxval
    actValue = actValue % self . maxval
err = abs ( expValue - actValue )
if self . periodic :
    err = min ( err , self . maxval - err )
if fractional :
    pctErr = float ( err ) / ( self . maxval - self . minval )
    pctErr = min ( 1.0 , pctErr )
    closeness = 1.0 - pctErr
else :
    closeness = err
return numpy . array ( [ closeness ] )


signature:
def _initEphemerals ( self ) :
    """Initialize all ephemeral members after being restored to a pickled state ."""

implementation:
# # We store the lists of segments updates , per cell , so that they can be
# applied later during learning , when the cell gets bottom - up activation .
# We store one list per cell . The lists are identified with a hash key which
# is a tuple ( column index , cell index ) .
self . segmentUpdates = { }
# Allocate and reset all stats
self . resetStats ( )
# NOTE : We don ' t use the same backtrack buffer for inference and learning
# because learning has a different metric for determining if an input from
# the past is potentially useful again for backtracking .
# Our inference backtrack buffer . This keeps track of up to
# maxInfBacktrack of previous input . Each entry is a list of active column
# inputs .
self . _prevInfPatterns = [ ]
# Our learning backtrack buffer . This keeps track of up to maxLrnBacktrack
# of previous input . Each entry is a list of active column inputs
self . _prevLrnPatterns = [ ]
# Keep integers rather than bools . Float ?
stateShape = ( self . numberOfCols , self . cellsPerColumn )
self . lrnActiveState = { }
self . lrnActiveState [ "t" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . lrnActiveState [ "t-1" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . lrnPredictedState = { }
self . lrnPredictedState [ "t" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . lrnPredictedState [ "t-1" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infActiveState = { }
self . infActiveState [ "t" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infActiveState [ "t-1" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infActiveState [ "backup" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infActiveState [ "candidate" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infPredictedState = { }
self . infPredictedState [ "t" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infPredictedState [ "t-1" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infPredictedState [ "backup" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . infPredictedState [ "candidate" ] = numpy . zeros ( stateShape , dtype = "int8" )
self . cellConfidence = { }
self . cellConfidence [ "t" ] = numpy . zeros ( stateShape , dtype = "float32" )
self . cellConfidence [ "t-1" ] = numpy . zeros ( stateShape , dtype = "float32" )
self . cellConfidence [ "candidate" ] = numpy . zeros ( stateShape , dtype = "float32" )
self . colConfidence = { }
self . colConfidence [ "t" ] = numpy . zeros ( self . numberOfCols , dtype = "float32" )
self . colConfidence [ "t-1" ] = numpy . zeros ( self . numberOfCols , dtype = "float32" )
self . colConfidence [ "candidate" ] = numpy . zeros ( self . numberOfCols , dtype = "float32" )