signature:
def _generateEncoderChoicesV1 ( fieldInfo ) :
    """Return a list of possible encoder parameter combinations for the given
 field and the default aggregation function to use . Each parameter combination
 is a dict defining the parameters for the encoder . Here is an example
 return value for the encoderChoicesList :
 None ,
 { ' fieldname ' : ' timestamp ' ,
 ' name ' : ' timestamp _ timeOfDay ' ,
 ' type ' : ' DateEncoder '
 ' dayOfWeek ' : ( 7,1)
 { ' fieldname ' : ' timestamp ' ,
 ' name ' : ' timestamp _ timeOfDay ' ,
 ' type ' : ' DateEncoder '
 ' dayOfWeek ' : ( 7,3)
 Parameters :
 fieldInfo : item from the ' includedFields ' section of the
 description JSON object
 retval : ( encoderChoicesList , aggFunction )
 encoderChoicesList : a list of encoder choice lists for this field .
 Most fields will generate just 1 encoder choice list .
 DateTime fields can generate 2 or more encoder choice lists ,
 one for dayOfWeek , one for timeOfDay , etc .
 aggFunction : name of aggregation function to use for this
 field type"""

implementation:
width = 7
fieldName = fieldInfo [ 'fieldName' ]
fieldType = fieldInfo [ 'fieldType' ]
encoderChoicesList = [ ]
# Scalar ?
if fieldType in [ 'float' , 'int' ] :
    aggFunction = 'mean'
    encoders = [ None ]
    for n in ( 13 , 50 , 150 , 500 ) :
        encoder = dict ( type = 'ScalarSpaceEncoder' , name = fieldName , fieldname = fieldName , n = n , w = width , clipInput = True , space = "absolute" )
        if 'minValue' in fieldInfo :
            encoder [ 'minval' ] = fieldInfo [ 'minValue' ]
        if 'maxValue' in fieldInfo :
            encoder [ 'maxval' ] = fieldInfo [ 'maxValue' ]
        encoders . append ( encoder )
    encoderChoicesList . append ( encoders )
# String ?
elif fieldType == 'string' :
    aggFunction = 'first'
    encoders = [ None ]
    encoder = dict ( type = 'SDRCategoryEncoder' , name = fieldName , fieldname = fieldName , n = 100 , w = width )
    encoders . append ( encoder )
    encoderChoicesList . append ( encoders )
# Datetime ?
elif fieldType == 'datetime' :
    aggFunction = 'first'
    # First , the time of day representation
    encoders = [ None ]
    for radius in ( 1 , 8 ) :
        encoder = dict ( type = 'DateEncoder' , name = '%s_timeOfDay' % ( fieldName ) , fieldname = fieldName , timeOfDay = ( width , radius ) )
        encoders . append ( encoder )
    encoderChoicesList . append ( encoders )
    # Now , the day of week representation
    encoders = [ None ]
    for radius in ( 1 , 3 ) :
        encoder = dict ( type = 'DateEncoder' , name = '%s_dayOfWeek' % ( fieldName ) , fieldname = fieldName , dayOfWeek = ( width , radius ) )
        encoders . append ( encoder )
    encoderChoicesList . append ( encoders )
else :
    raise RuntimeError ( "Unsupported field type '%s'" % ( fieldType ) )
# Return results
return ( encoderChoicesList , aggFunction )


signature:
def _generateEncoderStringsV1 ( includedFields ) :
    """Generate and return the following encoder related substitution variables :
 encoderSpecsStr :
 For the base description file , this string defines the default
 encoding dicts for each encoder . For example :
 ' _ _ gym _ encoder ' : { ' fieldname ' : ' gym ' ,
 ' n ' : 13,
 ' name ' : ' gym ' ,
 ' type ' : ' SDRCategoryEncoder ' ,
 ' w ' : 7 } ,
 ' _ _ address _ encoder ' : { ' fieldname ' : ' address ' ,
 ' n ' : 13,
 ' name ' : ' address ' ,
 ' type ' : ' SDRCategoryEncoder ' ,
 ' w ' : 7}
 encoderSchemaStr :
 For the base description file , this is a list containing a
 DeferredDictLookup entry for each encoder . For example :
 [ DeferredDictLookup ( ' _ _ gym _ encoder ' ) ,
 DeferredDictLookup ( ' _ _ address _ encoder ' ) ,
 DeferredDictLookup ( ' _ _ timestamp _ timeOfDay _ encoder ' ) ,
 DeferredDictLookup ( ' _ _ timestamp _ dayOfWeek _ encoder ' ) ,
 DeferredDictLookup ( ' _ _ consumption _ encoder ' ) ] ,
 permEncoderChoicesStr :
 For the permutations file , this defines the possible
 encoder dicts for each encoder . For example :
 ' _ _ timestamp _ dayOfWeek _ encoder ' : [
 None ,
 { ' fieldname ' : ' timestamp ' ,
 ' name ' : ' timestamp _ timeOfDay ' ,
 ' type ' : ' DateEncoder '
 ' dayOfWeek ' : ( 7,1)
 { ' fieldname ' : ' timestamp ' ,
 ' name ' : ' timestamp _ timeOfDay ' ,
 ' type ' : ' DateEncoder '
 ' dayOfWeek ' : ( 7,3)
 ' _ _ field _ consumption _ encoder ' : [
 None ,
 { ' fieldname ' : ' consumption ' ,
 ' name ' : ' consumption ' ,
 ' type ' : ' AdaptiveScalarEncoder ' ,
 ' n ' : 13,
 ' w ' : 7,
 Parameters :
 includedFields : item from the ' includedFields ' section of the
 description JSON object . This is a list of dicts , each
 dict defining the field name , type , and optional min
 and max values .
 retval : ( encoderSpecsStr , encoderSchemaStr permEncoderChoicesStr )"""

implementation:
# First accumulate the possible choices for each encoder
encoderChoicesList = [ ]
for fieldInfo in includedFields :
    fieldName = fieldInfo [ 'fieldName' ]
    # Get the list of encoder choices for this field
    ( choicesList , aggFunction ) = _generateEncoderChoicesV1 ( fieldInfo )
    encoderChoicesList . extend ( choicesList )
# Generate the string containing the encoder specs and encoder schema . See
# the function comments for an example of the encoderSpecsStr and
# encoderSchemaStr
encoderSpecsList = [ ]
for encoderChoices in encoderChoicesList : # Use the last choice as the default in the base file because the 1st is
# often None
    encoder = encoderChoices [ - 1 ]
    # Check for bad characters
    for c in _ILLEGAL_FIELDNAME_CHARACTERS :
        if encoder [ 'name' ] . find ( c ) >= 0 :
            raise _ExpGeneratorException ( "Illegal character in field: %r (%r)" % ( c , encoder [ 'name' ] ) )
    encoderSpecsList . append ( "%s: \n%s%s" % ( _quoteAndEscape ( encoder [ 'name' ] ) , 2 * _ONE_INDENT , pprint . pformat ( encoder , indent = 2 * _INDENT_STEP ) ) )
encoderSpecsStr = ',\n  ' . join ( encoderSpecsList )
# Generate the string containing the permutation encoder choices . See the
# function comments above for an example of the permEncoderChoicesStr
permEncoderChoicesList = [ ]
for encoderChoices in encoderChoicesList :
    permEncoderChoicesList . append ( "%s: %s," % ( _quoteAndEscape ( encoderChoices [ - 1 ] [ 'name' ] ) , pprint . pformat ( encoderChoices , indent = 2 * _INDENT_STEP ) ) )
permEncoderChoicesStr = '\n' . join ( permEncoderChoicesList )
permEncoderChoicesStr = _indentLines ( permEncoderChoicesStr , 1 , indentFirstLine = False )
# Return results
return ( encoderSpecsStr , permEncoderChoicesStr )


signature:
def _generatePermEncoderStr ( options , encoderDict ) :
    """Generate the string that defines the permutations to apply for a given
 encoder .
 Parameters :
 options : experiment params
 encoderDict : the encoder dict , which gets placed into the description . py
 For example , if the encoderDict contains :
 ' consumption ' : {
 ' clipInput ' : True ,
 ' fieldname ' : u ' consumption ' ,
 ' n ' : 100,
 ' name ' : u ' consumption ' ,
 ' type ' : ' AdaptiveScalarEncoder ' ,
 ' w ' : 21 } ,
 The return string will contain :
 " PermuteEncoder ( fieldName = ' consumption ' ,
 encoderClass = ' AdaptiveScalarEncoder ' ,
 w = 21,
 n = PermuteInt ( 28 , 521 ) ,
 clipInput = True ) " """

implementation:
permStr = ""
# If it ' s the encoder for the classifier input , then it ' s always present so
# put it in as a dict in the permutations . py file instead of a
# PermuteEncoder ( ) .
if encoderDict . get ( 'classifierOnly' , False ) :
    permStr = "dict("
    for key , value in encoderDict . items ( ) :
        if key == "name" :
            continue
        if key == 'n' and encoderDict [ 'type' ] != 'SDRCategoryEncoder' :
            permStr += "n=PermuteInt(%d, %d), " % ( encoderDict [ "w" ] + 7 , encoderDict [ "w" ] + 500 )
        else :
            if issubclass ( type ( value ) , basestring ) :
                permStr += "%s='%s', " % ( key , value )
            else :
                permStr += "%s=%s, " % ( key , value )
    permStr += ")"
else : # Scalar encoders
    if encoderDict [ "type" ] in [ "ScalarSpaceEncoder" , "AdaptiveScalarEncoder" , "ScalarEncoder" , "LogEncoder" ] :
        permStr = "PermuteEncoder("
        for key , value in encoderDict . items ( ) :
            if key == "fieldname" :
                key = "fieldName"
            elif key == "type" :
                key = "encoderClass"
            elif key == "name" :
                continue
            if key == "n" :
                permStr += "n=PermuteInt(%d, %d), " % ( encoderDict [ "w" ] + 1 , encoderDict [ "w" ] + 500 )
            elif key == "runDelta" :
                if value and not "space" in encoderDict :
                    permStr += "space=PermuteChoices([%s,%s]), " % ( _quoteAndEscape ( "delta" ) , _quoteAndEscape ( "absolute" ) )
                encoderDict . pop ( "runDelta" )
            else :
                if issubclass ( type ( value ) , basestring ) :
                    permStr += "%s='%s', " % ( key , value )
                else :
                    permStr += "%s=%s, " % ( key , value )
        permStr += ")"
    # Category encoder
    elif encoderDict [ "type" ] in [ "SDRCategoryEncoder" ] :
        permStr = "PermuteEncoder("
        for key , value in encoderDict . items ( ) :
            if key == "fieldname" :
                key = "fieldName"
            elif key == "type" :
                key = "encoderClass"
            elif key == "name" :
                continue
            if issubclass ( type ( value ) , basestring ) :
                permStr += "%s='%s', " % ( key , value )
            else :
                permStr += "%s=%s, " % ( key , value )
        permStr += ")"
    # Datetime encoder
    elif encoderDict [ "type" ] in [ "DateEncoder" ] :
        permStr = "PermuteEncoder("
        for key , value in encoderDict . items ( ) :
            if key == "fieldname" :
                key = "fieldName"
            elif key == "type" :
                continue
            elif key == "name" :
                continue
            if key == "timeOfDay" :
                permStr += "encoderClass='%s.timeOfDay', " % ( encoderDict [ "type" ] )
                permStr += "radius=PermuteFloat(0.5, 12), "
                permStr += "w=%d, " % ( value [ 0 ] )
            elif key == "dayOfWeek" :
                permStr += "encoderClass='%s.dayOfWeek', " % ( encoderDict [ "type" ] )
                permStr += "radius=PermuteFloat(1, 6), "
                permStr += "w=%d, " % ( value [ 0 ] )
            elif key == "weekend" :
                permStr += "encoderClass='%s.weekend', " % ( encoderDict [ "type" ] )
                permStr += "radius=PermuteChoices([1]),  "
                permStr += "w=%d, " % ( value )
            else :
                if issubclass ( type ( value ) , basestring ) :
                    permStr += "%s='%s', " % ( key , value )
                else :
                    permStr += "%s=%s, " % ( key , value )
        permStr += ")"
    else :
        raise RuntimeError ( "Unsupported encoder type '%s'" % ( encoderDict [ "type" ] ) )
return permStr


signature:
def _generateEncoderStringsV2 ( includedFields , options ) :
    """Generate and return the following encoder related substitution variables :
 encoderSpecsStr :
 For the base description file , this string defines the default
 encoding dicts for each encoder . For example :
 _ _ gym _ encoder = { ' fieldname ' : ' gym ' ,
 ' n ' : 13,
 ' name ' : ' gym ' ,
 ' type ' : ' SDRCategoryEncoder ' ,
 ' w ' : 7 } ,
 _ _ address _ encoder = { ' fieldname ' : ' address ' ,
 ' n ' : 13,
 ' name ' : ' address ' ,
 ' type ' : ' SDRCategoryEncoder ' ,
 ' w ' : 7}
 permEncoderChoicesStr :
 For the permutations file , this defines the possible
 encoder dicts for each encoder . For example :
 ' _ _ gym _ encoder ' : PermuteEncoder ( ' gym ' , ' SDRCategoryEncoder ' , w = 7,
 n = 100 ) ,
 ' _ _ address _ encoder ' : PermuteEncoder ( ' address ' , ' SDRCategoryEncoder ' ,
 w = 7 , n = 100 ) ,
 ' _ _ timestamp _ dayOfWeek _ encoder ' : PermuteEncoder ( ' timestamp ' ,
 ' DateEncoder . timeOfDay ' , w = 7 , radius = PermuteChoices ( [ 1 , 8 ] ) ) ,
 ' _ _ consumption _ encoder ' : PermuteEncoder ( ' consumption ' , ' AdaptiveScalarEncoder ' ,
 w = 7 , n = PermuteInt ( 13 , 500 , 20 ) , minval = 0,
 maxval = PermuteInt ( 100 , 300 , 25 ) ) ,
 Parameters :
 includedFields : item from the ' includedFields ' section of the
 description JSON object . This is a list of dicts , each
 dict defining the field name , type , and optional min
 and max values .
 retval : ( encoderSpecsStr permEncoderChoicesStr )"""

implementation:
width = 21
encoderDictsList = [ ]
# If this is a NontemporalClassification experiment , then the
# the " predicted " field ( the classification value ) should be marked to ONLY
# go to the classifier
if options [ 'inferenceType' ] in [ "NontemporalClassification" , "NontemporalMultiStep" , "TemporalMultiStep" , "MultiStep" ] :
    classifierOnlyField = options [ 'inferenceArgs' ] [ 'predictedField' ]
else :
    classifierOnlyField = None
# For each field , generate the default encoding dict and PermuteEncoder
# constructor arguments
for fieldInfo in includedFields :
    fieldName = fieldInfo [ 'fieldName' ]
    fieldType = fieldInfo [ 'fieldType' ]
    # Scalar ?
    if fieldType in [ 'float' , 'int' ] : # n = 100 is reasonably hardcoded value for n when used by description . py
    # The swarming will use PermuteEncoder below , where n is variable and
    # depends on w
        runDelta = fieldInfo . get ( "runDelta" , False )
        if runDelta or "space" in fieldInfo :
            encoderDict = dict ( type = 'ScalarSpaceEncoder' , name = fieldName , fieldname = fieldName , n = 100 , w = width , clipInput = True )
            if runDelta :
                encoderDict [ "runDelta" ] = True
        else :
            encoderDict = dict ( type = 'AdaptiveScalarEncoder' , name = fieldName , fieldname = fieldName , n = 100 , w = width , clipInput = True )
        if 'minValue' in fieldInfo :
            encoderDict [ 'minval' ] = fieldInfo [ 'minValue' ]
        if 'maxValue' in fieldInfo :
            encoderDict [ 'maxval' ] = fieldInfo [ 'maxValue' ]
        # If both min and max were specified , use a non - adaptive encoder
        if ( 'minValue' in fieldInfo and 'maxValue' in fieldInfo ) and ( encoderDict [ 'type' ] == 'AdaptiveScalarEncoder' ) :
            encoderDict [ 'type' ] = 'ScalarEncoder'
        # Defaults may have been over - ridden by specifying an encoder type
        if 'encoderType' in fieldInfo :
            encoderDict [ 'type' ] = fieldInfo [ 'encoderType' ]
        if 'space' in fieldInfo :
            encoderDict [ 'space' ] = fieldInfo [ 'space' ]
        encoderDictsList . append ( encoderDict )
    # String ?
    elif fieldType == 'string' :
        encoderDict = dict ( type = 'SDRCategoryEncoder' , name = fieldName , fieldname = fieldName , n = 100 + width , w = width )
        if 'encoderType' in fieldInfo :
            encoderDict [ 'type' ] = fieldInfo [ 'encoderType' ]
        encoderDictsList . append ( encoderDict )
    # Datetime ?
    elif fieldType == 'datetime' : # First , the time of day representation
        encoderDict = dict ( type = 'DateEncoder' , name = '%s_timeOfDay' % ( fieldName ) , fieldname = fieldName , timeOfDay = ( width , 1 ) )
        if 'encoderType' in fieldInfo :
            encoderDict [ 'type' ] = fieldInfo [ 'encoderType' ]
        encoderDictsList . append ( encoderDict )
        # Now , the day of week representation
        encoderDict = dict ( type = 'DateEncoder' , name = '%s_dayOfWeek' % ( fieldName ) , fieldname = fieldName , dayOfWeek = ( width , 1 ) )
        if 'encoderType' in fieldInfo :
            encoderDict [ 'type' ] = fieldInfo [ 'encoderType' ]
        encoderDictsList . append ( encoderDict )
        # Now , the day of week representation
        encoderDict = dict ( type = 'DateEncoder' , name = '%s_weekend' % ( fieldName ) , fieldname = fieldName , weekend = ( width ) )
        if 'encoderType' in fieldInfo :
            encoderDict [ 'type' ] = fieldInfo [ 'encoderType' ]
        encoderDictsList . append ( encoderDict )
    else :
        raise RuntimeError ( "Unsupported field type '%s'" % ( fieldType ) )
    # If this was the predicted field , insert another encoder that sends it
    # to the classifier only
    if fieldName == classifierOnlyField :
        clEncoderDict = dict ( encoderDict )
        clEncoderDict [ 'classifierOnly' ] = True
        clEncoderDict [ 'name' ] = '_classifierInput'
        encoderDictsList . append ( clEncoderDict )
        # If the predicted field needs to be excluded , take it out of the encoder
        # lists
        if options [ "inferenceArgs" ] [ "inputPredictedField" ] == "no" :
            encoderDictsList . remove ( encoderDict )
# Remove any encoders not in fixedFields
if options . get ( 'fixedFields' ) is not None :
    tempList = [ ]
    for encoderDict in encoderDictsList :
        if encoderDict [ 'name' ] in options [ 'fixedFields' ] :
            tempList . append ( encoderDict )
    encoderDictsList = tempList
# Now generate the encoderSpecsStr and permEncoderChoicesStr strings from
# encoderDictsList and constructorStringList
encoderSpecsList = [ ]
permEncoderChoicesList = [ ]
for encoderDict in encoderDictsList :
    if encoderDict [ 'name' ] . find ( '\\' ) >= 0 :
        raise _ExpGeneratorException ( "Illegal character in field: '\\'" )
    # Check for bad characters
    for c in _ILLEGAL_FIELDNAME_CHARACTERS :
        if encoderDict [ 'name' ] . find ( c ) >= 0 :
            raise _ExpGeneratorException ( "Illegal character %s in field %r" % ( c , encoderDict [ 'name' ] ) )
    constructorStr = _generatePermEncoderStr ( options , encoderDict )
    encoderKey = _quoteAndEscape ( encoderDict [ 'name' ] )
    encoderSpecsList . append ( "%s: %s%s" % ( encoderKey , 2 * _ONE_INDENT , pprint . pformat ( encoderDict , indent = 2 * _INDENT_STEP ) ) )
    # Each permEncoderChoicesStr is of the form :
    # PermuteEncoder ( ' gym ' , ' SDRCategoryEncoder ' ,
    # w = 7 , n = 100 ) ,
    permEncoderChoicesList . append ( "%s: %s," % ( encoderKey , constructorStr ) )
# Join into strings
encoderSpecsStr = ',\n  ' . join ( encoderSpecsList )
permEncoderChoicesStr = '\n' . join ( permEncoderChoicesList )
permEncoderChoicesStr = _indentLines ( permEncoderChoicesStr , 1 , indentFirstLine = True )
# Return results
return ( encoderSpecsStr , permEncoderChoicesStr )


signature:
def _handleJAVAParameters ( options ) :
    """Handle legacy options ( TEMPORARY )"""

implementation:
# Find the correct InferenceType for the Model
if 'inferenceType' not in options :
    prediction = options . get ( 'prediction' , { InferenceType . TemporalNextStep : { 'optimize' : True } } )
    inferenceType = None
    for infType , value in prediction . iteritems ( ) :
        if value [ 'optimize' ] :
            inferenceType = infType
            break
    if inferenceType == 'temporal' :
        inferenceType = InferenceType . TemporalNextStep
    if inferenceType != InferenceType . TemporalNextStep :
        raise _ExpGeneratorException ( "Unsupported inference type %s" % ( inferenceType ) )
    options [ 'inferenceType' ] = inferenceType
# Find the best value for the predicted field
if 'predictionField' in options :
    if 'inferenceArgs' not in options :
        options [ 'inferenceArgs' ] = { 'predictedField' : options [ 'predictionField' ] }
    elif 'predictedField' not in options [ 'inferenceArgs' ] :
        options [ 'inferenceArgs' ] [ 'predictedField' ] = options [ 'predictionField' ]


signature:
def _getPropertyValue ( schema , propertyName , options ) :
    """Checks to see if property is specified in ' options ' . If not , reads the
 default value from the schema"""

implementation:
if propertyName not in options :
    paramsSchema = schema [ 'properties' ] [ propertyName ]
    if 'default' in paramsSchema :
        options [ propertyName ] = paramsSchema [ 'default' ]
    else :
        options [ propertyName ] = None


signature:
def _getExperimentDescriptionSchema ( ) :
    """Returns the experiment description schema . This implementation loads it in
 from file experimentDescriptionSchema . json .
 Parameters :
 Returns : returns a dict representing the experiment description schema ."""

implementation:
installPath = os . path . dirname ( os . path . abspath ( __file__ ) )
schemaFilePath = os . path . join ( installPath , "experimentDescriptionSchema.json" )
return json . loads ( open ( schemaFilePath , 'r' ) . read ( ) )


signature:
def _generateExperiment ( options , outputDirPath , hsVersion , claDescriptionTemplateFile ) :
    """Executes the - - description option , which includes :
 1 . Perform provider compatibility checks
 2 . Preprocess the training and testing datasets ( filter , join providers )
 3 . If test dataset omitted , split the training dataset into training
 and testing datasets .
 4 . Gather statistics about the training and testing datasets .
 5 . Generate experiment scripts ( description . py , permutaions . py )
 Parameters :
 options : dictionary that matches the schema defined by the return value of
 _ getExperimentDescriptionSchema ( ) ; NOTE : this arg may be modified
 by this function .
 outputDirPath : where to place generated files
 hsVersion : which version of hypersearch permutations file to generate , can
 be ' v1 ' or ' v2'
 claDescriptionTemplateFile : Filename containing the template description
 Returns : on success , returns a dictionary per _ experimentResultsJSONSchema ;
 raises exception on error
 Assumption1 : input train and test files have identical field metadata"""

implementation:
_gExperimentDescriptionSchema = _getExperimentDescriptionSchema ( )
# Validate JSON arg using JSON schema validator
try :
    validictory . validate ( options , _gExperimentDescriptionSchema )
except Exception , e :
    raise _InvalidCommandArgException ( ( "JSON arg validation failed for option --description: " + "%s\nOPTION ARG=%s" ) % ( str ( e ) , pprint . pformat ( options ) ) )
# Validate the streamDef
streamSchema = json . load ( resource_stream ( jsonschema . __name__ , 'stream_def.json' ) )
try :
    validictory . validate ( options [ 'streamDef' ] , streamSchema )
except Exception , e :
    raise _InvalidCommandArgException ( ( "JSON arg validation failed for streamDef " + "%s\nOPTION ARG=%s" ) % ( str ( e ) , json . dumps ( options ) ) )
# Handle legacy parameters from JAVA API server
# TODO : remove this !
_handleJAVAParameters ( options )
# Get default values
for propertyName in _gExperimentDescriptionSchema [ 'properties' ] :
    _getPropertyValue ( _gExperimentDescriptionSchema , propertyName , options )
if options [ 'inferenceArgs' ] is not None :
    infArgs = _gExperimentDescriptionSchema [ 'properties' ] [ 'inferenceArgs' ]
    for schema in infArgs [ 'type' ] :
        if isinstance ( schema , dict ) :
            for propertyName in schema [ 'properties' ] :
                _getPropertyValue ( schema , propertyName , options [ 'inferenceArgs' ] )
if options [ 'anomalyParams' ] is not None :
    anomalyArgs = _gExperimentDescriptionSchema [ 'properties' ] [ 'anomalyParams' ]
    for schema in anomalyArgs [ 'type' ] :
        if isinstance ( schema , dict ) :
            for propertyName in schema [ 'properties' ] :
                _getPropertyValue ( schema , propertyName , options [ 'anomalyParams' ] )
# If the user specified nonTemporalClassification , make sure prediction
# steps is 0
predictionSteps = options [ 'inferenceArgs' ] . get ( 'predictionSteps' , None )
if options [ 'inferenceType' ] == InferenceType . NontemporalClassification :
    if predictionSteps is not None and predictionSteps != [ 0 ] :
        raise RuntimeError ( "When NontemporalClassification is used, prediction" " steps must be [0]" )
# If the user asked for 0 steps of prediction , then make this a spatial
# classification experiment
if predictionSteps == [ 0 ] and options [ 'inferenceType' ] in [ 'NontemporalMultiStep' , 'TemporalMultiStep' , 'MultiStep' ] :
    options [ 'inferenceType' ] = InferenceType . NontemporalClassification
# If NontemporalClassification was chosen as the inferenceType , then the
# predicted field can NOT be used as an input
if options [ "inferenceType" ] == InferenceType . NontemporalClassification :
    if options [ "inferenceArgs" ] [ "inputPredictedField" ] == "yes" or options [ "inferenceArgs" ] [ "inputPredictedField" ] == "auto" :
        raise RuntimeError ( "When the inference type is NontemporalClassification" " inputPredictedField must be set to 'no'" )
    options [ "inferenceArgs" ] [ "inputPredictedField" ] = "no"
# Process the swarmSize setting , if provided
swarmSize = options [ 'swarmSize' ]
if swarmSize is None :
    if options [ "inferenceArgs" ] [ "inputPredictedField" ] is None :
        options [ "inferenceArgs" ] [ "inputPredictedField" ] = "auto"
elif swarmSize == 'small' :
    if options [ 'minParticlesPerSwarm' ] is None :
        options [ 'minParticlesPerSwarm' ] = 3
    if options [ 'iterationCount' ] is None :
        options [ 'iterationCount' ] = 100
    if options [ 'maxModels' ] is None :
        options [ 'maxModels' ] = 1
    if options [ "inferenceArgs" ] [ "inputPredictedField" ] is None :
        options [ "inferenceArgs" ] [ "inputPredictedField" ] = "yes"
elif swarmSize == 'medium' :
    if options [ 'minParticlesPerSwarm' ] is None :
        options [ 'minParticlesPerSwarm' ] = 5
    if options [ 'iterationCount' ] is None :
        options [ 'iterationCount' ] = 4000
    if options [ 'maxModels' ] is None :
        options [ 'maxModels' ] = 200
    if options [ "inferenceArgs" ] [ "inputPredictedField" ] is None :
        options [ "inferenceArgs" ] [ "inputPredictedField" ] = "auto"
elif swarmSize == 'large' :
    if options [ 'minParticlesPerSwarm' ] is None :
        options [ 'minParticlesPerSwarm' ] = 15
    # options [ ' killUselessSwarms ' ] = False
    # options [ ' minFieldContribution ' ] = - 1000
    # options [ ' maxFieldBranching ' ] = 10
    # options [ ' tryAll3FieldCombinations ' ] = True
    options [ 'tryAll3FieldCombinationsWTimestamps' ] = True
    if options [ "inferenceArgs" ] [ "inputPredictedField" ] is None :
        options [ "inferenceArgs" ] [ "inputPredictedField" ] = "auto"
else :
    raise RuntimeError ( "Unsupported swarm size: %s" % ( swarmSize ) )
# Get token replacements
tokenReplacements = dict ( )
# Generate the encoder related substitution strings
includedFields = options [ 'includedFields' ]
if hsVersion == 'v1' :
    ( encoderSpecsStr , permEncoderChoicesStr ) = _generateEncoderStringsV1 ( includedFields )
elif hsVersion in [ 'v2' , 'ensemble' ] :
    ( encoderSpecsStr , permEncoderChoicesStr ) = _generateEncoderStringsV2 ( includedFields , options )
else :
    raise RuntimeError ( "Unsupported hsVersion of %s" % ( hsVersion ) )
# Generate the string containing the sensor auto - reset dict .
if options [ 'resetPeriod' ] is not None :
    sensorAutoResetStr = pprint . pformat ( options [ 'resetPeriod' ] , indent = 2 * _INDENT_STEP )
else :
    sensorAutoResetStr = 'None'
# Generate the string containing the aggregation settings .
aggregationPeriod = { 'days' : 0 , 'hours' : 0 , 'microseconds' : 0 , 'milliseconds' : 0 , 'minutes' : 0 , 'months' : 0 , 'seconds' : 0 , 'weeks' : 0 , 'years' : 0 , }
# Honor any overrides provided in the stream definition
aggFunctionsDict = { }
if 'aggregation' in options [ 'streamDef' ] :
    for key in aggregationPeriod . keys ( ) :
        if key in options [ 'streamDef' ] [ 'aggregation' ] :
            aggregationPeriod [ key ] = options [ 'streamDef' ] [ 'aggregation' ] [ key ]
    if 'fields' in options [ 'streamDef' ] [ 'aggregation' ] :
        for ( fieldName , func ) in options [ 'streamDef' ] [ 'aggregation' ] [ 'fields' ] :
            aggFunctionsDict [ fieldName ] = str ( func )
# Do we have any aggregation at all ?
hasAggregation = False
for v in aggregationPeriod . values ( ) :
    if v != 0 :
        hasAggregation = True
        break
# Convert the aggFunctionsDict to a list
aggFunctionList = aggFunctionsDict . items ( )
aggregationInfo = dict ( aggregationPeriod )
aggregationInfo [ 'fields' ] = aggFunctionList
# Form the aggregation strings
aggregationInfoStr = "%s" % ( pprint . pformat ( aggregationInfo , indent = 2 * _INDENT_STEP ) )
# Generate the string defining the dataset . This is basically the
# streamDef , but referencing the aggregation we already pulled out into the
# config dict ( which enables permuting over it )
datasetSpec = options [ 'streamDef' ]
if 'aggregation' in datasetSpec :
    datasetSpec . pop ( 'aggregation' )
if hasAggregation :
    datasetSpec [ 'aggregation' ] = '$SUBSTITUTE'
datasetSpecStr = pprint . pformat ( datasetSpec , indent = 2 * _INDENT_STEP )
datasetSpecStr = datasetSpecStr . replace ( "'$SUBSTITUTE'" , "config['aggregationInfo']" )
datasetSpecStr = _indentLines ( datasetSpecStr , 2 , indentFirstLine = False )
# Was computeInterval specified with Multistep prediction ? If so , this swarm
# should permute over different aggregations
computeInterval = options [ 'computeInterval' ]
if computeInterval is not None and options [ 'inferenceType' ] in [ 'NontemporalMultiStep' , 'TemporalMultiStep' , 'MultiStep' ] : # Compute the predictAheadTime based on the minAggregation ( specified in
# the stream definition ) and the number of prediction steps
    predictionSteps = options [ 'inferenceArgs' ] . get ( 'predictionSteps' , [ 1 ] )
    if len ( predictionSteps ) > 1 :
        raise _InvalidCommandArgException ( "Invalid predictionSteps: %s. " "When computeInterval is specified, there can only be one " "stepSize in predictionSteps." % predictionSteps )
    if max ( aggregationInfo . values ( ) ) == 0 :
        raise _InvalidCommandArgException ( "Missing or nil stream aggregation: " "When computeInterval is specified, then the stream aggregation " "interval must be non-zero." )
    # Compute the predictAheadTime
    numSteps = predictionSteps [ 0 ]
    predictAheadTime = dict ( aggregationPeriod )
    for key in predictAheadTime . iterkeys ( ) :
        predictAheadTime [ key ] *= numSteps
    predictAheadTimeStr = pprint . pformat ( predictAheadTime , indent = 2 * _INDENT_STEP )
    # This tells us to plug in a wildcard string for the prediction steps that
    # we use in other parts of the description file ( metrics , inferenceArgs ,
    # etc . )
    options [ 'dynamicPredictionSteps' ] = True
else :
    options [ 'dynamicPredictionSteps' ] = False
    predictAheadTimeStr = "None"
# Save environment - common token substitutions
tokenReplacements [ '\$EXP_GENERATOR_PROGRAM_PATH' ] = _quoteAndEscape ( os . path . abspath ( __file__ ) )
# If the " uber " metric ' MultiStep ' was specified , then plug in TemporalMultiStep
# by default
inferenceType = options [ 'inferenceType' ]
if inferenceType == 'MultiStep' :
    inferenceType = InferenceType . TemporalMultiStep
tokenReplacements [ '\$INFERENCE_TYPE' ] = "'%s'" % inferenceType
# Nontemporal classificaion uses only encoder and classifier
if inferenceType == InferenceType . NontemporalClassification :
    tokenReplacements [ '\$SP_ENABLE' ] = "False"
    tokenReplacements [ '\$TP_ENABLE' ] = "False"
else :
    tokenReplacements [ '\$SP_ENABLE' ] = "True"
    tokenReplacements [ '\$TP_ENABLE' ] = "True"
    tokenReplacements [ '\$CLA_CLASSIFIER_IMPL' ] = ""
tokenReplacements [ '\$ANOMALY_PARAMS' ] = pprint . pformat ( options [ 'anomalyParams' ] , indent = 2 * _INDENT_STEP )
tokenReplacements [ '\$ENCODER_SPECS' ] = encoderSpecsStr
tokenReplacements [ '\$SENSOR_AUTO_RESET' ] = sensorAutoResetStr
tokenReplacements [ '\$AGGREGATION_INFO' ] = aggregationInfoStr
tokenReplacements [ '\$DATASET_SPEC' ] = datasetSpecStr
if options [ 'iterationCount' ] is None :
    options [ 'iterationCount' ] = - 1
tokenReplacements [ '\$ITERATION_COUNT' ] = str ( options [ 'iterationCount' ] )
tokenReplacements [ '\$SP_POOL_PCT' ] = str ( options [ 'spCoincInputPoolPct' ] )
tokenReplacements [ '\$HS_MIN_PARTICLES' ] = str ( options [ 'minParticlesPerSwarm' ] )
tokenReplacements [ '\$SP_PERM_CONNECTED' ] = str ( options [ 'spSynPermConnected' ] )
tokenReplacements [ '\$FIELD_PERMUTATION_LIMIT' ] = str ( options [ 'fieldPermutationLimit' ] )
tokenReplacements [ '\$PERM_ENCODER_CHOICES' ] = permEncoderChoicesStr
predictionSteps = options [ 'inferenceArgs' ] . get ( 'predictionSteps' , [ 1 ] )
predictionStepsStr = ',' . join ( [ str ( x ) for x in predictionSteps ] )
tokenReplacements [ '\$PREDICTION_STEPS' ] = "'%s'" % ( predictionStepsStr )
tokenReplacements [ '\$PREDICT_AHEAD_TIME' ] = predictAheadTimeStr
# Option permuting over SP synapse decrement value
tokenReplacements [ '\$PERM_SP_CHOICES' ] = ""
if options [ 'spPermuteDecrement' ] and options [ 'inferenceType' ] != 'NontemporalClassification' :
    tokenReplacements [ '\$PERM_SP_CHOICES' ] = _ONE_INDENT + "'synPermInactiveDec': PermuteFloat(0.0003, 0.1),\n"
# The TM permutation parameters are not required for non - temporal networks
if options [ 'inferenceType' ] in [ 'NontemporalMultiStep' , 'NontemporalClassification' ] :
    tokenReplacements [ '\$PERM_TP_CHOICES' ] = ""
else :
    tokenReplacements [ '\$PERM_TP_CHOICES' ] = "  'activationThreshold': PermuteInt(12, 16),\n" + "  'minThreshold': PermuteInt(9, 12),\n" + "  'pamLength': PermuteInt(1, 5),\n"
# If the inference type is just the generic ' MultiStep ' , then permute over
# temporal / nonTemporal multistep
if options [ 'inferenceType' ] == 'MultiStep' :
    tokenReplacements [ '\$PERM_INFERENCE_TYPE_CHOICES' ] = "  'inferenceType': PermuteChoices(['NontemporalMultiStep', " + "'TemporalMultiStep']),"
else :
    tokenReplacements [ '\$PERM_INFERENCE_TYPE_CHOICES' ] = ""
# The Classifier permutation parameters are only required for
# Multi - step inference types
if options [ 'inferenceType' ] in [ 'NontemporalMultiStep' , 'TemporalMultiStep' , 'MultiStep' , 'TemporalAnomaly' , 'NontemporalClassification' ] :
    tokenReplacements [ '\$PERM_CL_CHOICES' ] = "  'alpha': PermuteFloat(0.0001, 0.1),\n"
else :
    tokenReplacements [ '\$PERM_CL_CHOICES' ] = ""
# The Permutations alwaysIncludePredictedField setting .
# * When the experiment description has ' inputPredictedField ' set to ' no ' , we
# simply do not put in an encoder for the predicted field .
# * When ' inputPredictedField ' is set to ' auto ' , we include an encoder for the
# predicted field and swarming tries it out just like all the other fields .
# * When ' inputPredictedField ' is set to ' yes ' , we include this setting in
# the permutations file which informs swarming to always use the
# predicted field ( the first swarm will be the predicted field only )
tokenReplacements [ '\$PERM_ALWAYS_INCLUDE_PREDICTED_FIELD' ] = "inputPredictedField = '%s'" % ( options [ "inferenceArgs" ] [ "inputPredictedField" ] )
# The Permutations minFieldContribution setting
if options . get ( 'minFieldContribution' , None ) is not None :
    tokenReplacements [ '\$PERM_MIN_FIELD_CONTRIBUTION' ] = "minFieldContribution = %d" % ( options [ 'minFieldContribution' ] )
else :
    tokenReplacements [ '\$PERM_MIN_FIELD_CONTRIBUTION' ] = ""
# The Permutations killUselessSwarms setting
if options . get ( 'killUselessSwarms' , None ) is not None :
    tokenReplacements [ '\$PERM_KILL_USELESS_SWARMS' ] = "killUselessSwarms = %r" % ( options [ 'killUselessSwarms' ] )
else :
    tokenReplacements [ '\$PERM_KILL_USELESS_SWARMS' ] = ""
# The Permutations maxFieldBranching setting
if options . get ( 'maxFieldBranching' , None ) is not None :
    tokenReplacements [ '\$PERM_MAX_FIELD_BRANCHING' ] = "maxFieldBranching = %r" % ( options [ 'maxFieldBranching' ] )
else :
    tokenReplacements [ '\$PERM_MAX_FIELD_BRANCHING' ] = ""
# The Permutations tryAll3FieldCombinations setting
if options . get ( 'tryAll3FieldCombinations' , None ) is not None :
    tokenReplacements [ '\$PERM_TRY_ALL_3_FIELD_COMBINATIONS' ] = "tryAll3FieldCombinations = %r" % ( options [ 'tryAll3FieldCombinations' ] )
else :
    tokenReplacements [ '\$PERM_TRY_ALL_3_FIELD_COMBINATIONS' ] = ""
# The Permutations tryAll3FieldCombinationsWTimestamps setting
if options . get ( 'tryAll3FieldCombinationsWTimestamps' , None ) is not None :
    tokenReplacements [ '\$PERM_TRY_ALL_3_FIELD_COMBINATIONS_W_TIMESTAMPS' ] = "tryAll3FieldCombinationsWTimestamps = %r" % ( options [ 'tryAll3FieldCombinationsWTimestamps' ] )
else :
    tokenReplacements [ '\$PERM_TRY_ALL_3_FIELD_COMBINATIONS_W_TIMESTAMPS' ] = ""
# The Permutations fieldFields setting
if options . get ( 'fixedFields' , None ) is not None :
    tokenReplacements [ '\$PERM_FIXED_FIELDS' ] = "fixedFields = %r" % ( options [ 'fixedFields' ] )
else :
    tokenReplacements [ '\$PERM_FIXED_FIELDS' ] = ""
# The Permutations fastSwarmModelParams setting
if options . get ( 'fastSwarmModelParams' , None ) is not None :
    tokenReplacements [ '\$PERM_FAST_SWARM_MODEL_PARAMS' ] = "fastSwarmModelParams = %r" % ( options [ 'fastSwarmModelParams' ] )
else :
    tokenReplacements [ '\$PERM_FAST_SWARM_MODEL_PARAMS' ] = ""
# The Permutations maxModels setting
if options . get ( 'maxModels' , None ) is not None :
    tokenReplacements [ '\$PERM_MAX_MODELS' ] = "maxModels = %r" % ( options [ 'maxModels' ] )
else :
    tokenReplacements [ '\$PERM_MAX_MODELS' ] = ""
# The Aggregation choices have to be determined when we are permuting over
# aggregations .
if options [ 'dynamicPredictionSteps' ] :
    debugAgg = True
    # First , we need to error check to insure that computeInterval is an integer
    # multiple of minAggregation ( aggregationPeriod )
    quotient = aggregationDivide ( computeInterval , aggregationPeriod )
    ( isInt , multiple ) = _isInt ( quotient )
    if not isInt or multiple < 1 :
        raise _InvalidCommandArgException ( "Invalid computeInterval: %s. " "computeInterval must be an integer multiple of the stream " "aggregation (%s)." % ( computeInterval , aggregationPeriod ) )
    # The valid aggregation choices are governed by the following constraint ,
    # 1 . ) ( minAggregation * N ) * M = predictAheadTime
    # ( minAggregation * N ) * M = maxPredictionSteps * minAggregation
    # N * M = maxPredictionSteps
    # 2 . ) computeInterval = K * aggregation
    # computeInterval = K * ( minAggregation * N )
    # where : aggregation = minAggregation * N
    # K , M and N are integers > = 1
    # N = aggregation / minAggregation
    # M = predictionSteps , for a particular aggregation
    # K = number of predictions within each compute interval
    # Let ' s build up a a list of the possible N ' s that satisfy the
    # N * M = maxPredictionSteps constraint
    mTimesN = float ( predictionSteps [ 0 ] )
    possibleNs = [ ]
    for n in xrange ( 1 , int ( mTimesN ) + 1 ) :
        m = mTimesN / n
        mInt = int ( round ( m ) )
        if mInt < 1 :
            break
        if abs ( m - mInt ) > 0.0001 * m :
            continue
        possibleNs . append ( n )
    if debugAgg :
        print "All integer factors of %d are: %s" % ( mTimesN , possibleNs )
    # Now go through and throw out any N ' s that don ' t satisfy the constraint :
    # computeInterval = K * ( minAggregation * N )
    aggChoices = [ ]
    for n in possibleNs : # Compute minAggregation * N
        agg = dict ( aggregationPeriod )
        for key in agg . iterkeys ( ) :
            agg [ key ] *= n
        # Make sure computeInterval is an integer multiple of the aggregation
        # period
        quotient = aggregationDivide ( computeInterval , agg )
        # print computeInterval , agg
        # print quotient
        # import sys ; sys . exit ( )
        ( isInt , multiple ) = _isInt ( quotient )
        if not isInt or multiple < 1 :
            continue
        aggChoices . append ( agg )
    # Only eveluate up to 5 different aggregations
    aggChoices = aggChoices [ - 5 : ]
    if debugAgg :
        print "Aggregation choices that will be evaluted during swarming:"
        for agg in aggChoices :
            print "  ==>" , agg
        print
    tokenReplacements [ '\$PERM_AGGREGATION_CHOICES' ] = ( "PermuteChoices(%s)" % ( pprint . pformat ( aggChoices , indent = 2 * _INDENT_STEP ) ) )
else :
    tokenReplacements [ '\$PERM_AGGREGATION_CHOICES' ] = aggregationInfoStr
# Generate the inferenceArgs replacement tokens
_generateInferenceArgs ( options , tokenReplacements )
# Generate the metric replacement tokens
_generateMetricsSubstitutions ( options , tokenReplacements )
# Generate Control dictionary
environment = options [ 'environment' ]
if environment == OpfEnvironment . Nupic :
    tokenReplacements [ '\$ENVIRONMENT' ] = "'%s'" % OpfEnvironment . Nupic
    controlTemplate = "nupicEnvironmentTemplate.tpl"
elif environment == OpfEnvironment . Experiment :
    tokenReplacements [ '\$ENVIRONMENT' ] = "'%s'" % OpfEnvironment . Experiment
    controlTemplate = "opfExperimentTemplate.tpl"
else :
    raise _InvalidCommandArgException ( "Invalid environment type %s" % environment )
if outputDirPath is None :
    outputDirPath = tempfile . mkdtemp ( )
if not os . path . exists ( outputDirPath ) :
    os . makedirs ( outputDirPath )
print "Generating experiment files in directory: %s..." % ( outputDirPath )
descriptionPyPath = os . path . join ( outputDirPath , "description.py" )
_generateFileFromTemplates ( [ claDescriptionTemplateFile , controlTemplate ] , descriptionPyPath , tokenReplacements )
permutationsPyPath = os . path . join ( outputDirPath , "permutations.py" )
if hsVersion == 'v1' :
    _generateFileFromTemplates ( [ 'permutationsTemplateV1.tpl' ] , permutationsPyPath , tokenReplacements )
elif hsVersion == 'ensemble' :
    _generateFileFromTemplates ( [ 'permutationsTemplateEnsemble.tpl' ] , permutationsPyPath , tokenReplacements )
elif hsVersion == 'v2' :
    _generateFileFromTemplates ( [ 'permutationsTemplateV2.tpl' ] , permutationsPyPath , tokenReplacements )
else :
    raise ( ValueError ( "This permutation version is not supported yet: %s" % hsVersion ) )
print "done."


signature:
def _generateMetricsSubstitutions ( options , tokenReplacements ) :
    """Generate the token substitution for metrics related fields .
 This includes :
 \$METRICS
 \$LOGGED_METRICS
 \$PERM_OPTIMIZE_SETTING"""

implementation:
options [ 'loggedMetrics' ] = [ ".*" ]
# Generate the required metrics
metricList , optimizeMetricLabel = _generateMetricSpecs ( options )
metricListString = ",\n" . join ( metricList )
metricListString = _indentLines ( metricListString , 2 , indentFirstLine = False )
permOptimizeSettingStr = 'minimize = "%s"' % optimizeMetricLabel
# Specify which metrics should be logged
loggedMetricsListAsStr = "[%s]" % ( ", " . join ( [ "'%s'" % ptrn for ptrn in options [ 'loggedMetrics' ] ] ) )
tokenReplacements [ '\$LOGGED_METRICS' ] = loggedMetricsListAsStr
tokenReplacements [ '\$METRICS' ] = metricListString
tokenReplacements [ '\$PERM_OPTIMIZE_SETTING' ] = permOptimizeSettingStr


signature:
def _generateMetricSpecs ( options ) :
    """Generates the Metrics for a given InferenceType
 Parameters :
 options : ExpGenerator options
 retval : ( metricsList , optimizeMetricLabel )
 metricsList : list of metric string names
 optimizeMetricLabel : Name of the metric which to optimize over"""

implementation:
inferenceType = options [ 'inferenceType' ]
inferenceArgs = options [ 'inferenceArgs' ]
predictionSteps = inferenceArgs [ 'predictionSteps' ]
metricWindow = options [ 'metricWindow' ]
if metricWindow is None :
    metricWindow = int ( Configuration . get ( "nupic.opf.metricWindow" ) )
metricSpecStrings = [ ]
optimizeMetricLabel = ""
# Generate the metrics specified by the expGenerator paramters
metricSpecStrings . extend ( _generateExtraMetricSpecs ( options ) )
optimizeMetricSpec = None
# If using a dynamically computed prediction steps ( i . e . when swarming
# over aggregation is requested ) , then we will plug in the variable
# predictionSteps in place of the statically provided predictionSteps
# from the JSON description .
if options [ 'dynamicPredictionSteps' ] :
    assert len ( predictionSteps ) == 1
    predictionSteps = [ '$REPLACE_ME' ]
# Metrics for temporal prediction
if inferenceType in ( InferenceType . TemporalNextStep , InferenceType . TemporalAnomaly , InferenceType . TemporalMultiStep , InferenceType . NontemporalMultiStep , InferenceType . NontemporalClassification , 'MultiStep' ) :
    predictedFieldName , predictedFieldType = _getPredictedField ( options )
    isCategory = _isCategory ( predictedFieldType )
    metricNames = ( 'avg_err' , ) if isCategory else ( 'aae' , 'altMAPE' )
    trivialErrorMetric = 'avg_err' if isCategory else 'altMAPE'
    oneGramErrorMetric = 'avg_err' if isCategory else 'altMAPE'
    movingAverageBaselineName = 'moving_mode' if isCategory else 'moving_mean'
    # Multi - step metrics
    for metricName in metricNames :
        metricSpec , metricLabel = _generateMetricSpecString ( field = predictedFieldName , inferenceElement = InferenceElement . multiStepBestPredictions , metric = 'multiStep' , params = { 'errorMetric' : metricName , 'window' : metricWindow , 'steps' : predictionSteps } , returnLabel = True )
        metricSpecStrings . append ( metricSpec )
    # If the custom error metric was specified , add that
    if options [ "customErrorMetric" ] is not None :
        metricParams = dict ( options [ "customErrorMetric" ] )
        metricParams [ 'errorMetric' ] = 'custom_error_metric'
        metricParams [ 'steps' ] = predictionSteps
        # If errorWindow is not specified , make it equal to the default window
        if not "errorWindow" in metricParams :
            metricParams [ "errorWindow" ] = metricWindow
        metricSpec , metricLabel = _generateMetricSpecString ( field = predictedFieldName , inferenceElement = InferenceElement . multiStepPredictions , metric = "multiStep" , params = metricParams , returnLabel = True )
        metricSpecStrings . append ( metricSpec )
    # If this is the first specified step size , optimize for it . Be sure to
    # escape special characters since this is a regular expression
    optimizeMetricSpec = metricSpec
    metricLabel = metricLabel . replace ( '[' , '\\[' )
    metricLabel = metricLabel . replace ( ']' , '\\]' )
    optimizeMetricLabel = metricLabel
    if options [ "customErrorMetric" ] is not None :
        optimizeMetricLabel = ".*custom_error_metric.*"
    # Add in the trivial metrics
    if options [ "runBaselines" ] and inferenceType != InferenceType . NontemporalClassification :
        for steps in predictionSteps :
            metricSpecStrings . append ( _generateMetricSpecString ( field = predictedFieldName , inferenceElement = InferenceElement . prediction , metric = "trivial" , params = { 'window' : metricWindow , "errorMetric" : trivialErrorMetric , 'steps' : steps } ) )
            # # Add in the One - Gram baseline error metric
            # metricSpecStrings . append (
            # _ generateMetricSpecString ( field = predictedFieldName ,
            # inferenceElement = InferenceElement . encodings ,
            # metric = " two _ gram " ,
            # params = { ' window ' : metricWindow ,
            # " errorMetric " : oneGramErrorMetric ,
            # ' predictionField ' : predictedFieldName ,
            # ' steps ' : steps } )
            # Include the baseline moving mean / mode metric
            if isCategory :
                metricSpecStrings . append ( _generateMetricSpecString ( field = predictedFieldName , inferenceElement = InferenceElement . prediction , metric = movingAverageBaselineName , params = { 'window' : metricWindow , "errorMetric" : "avg_err" , "mode_window" : 200 , "steps" : steps } ) )
            else :
                metricSpecStrings . append ( _generateMetricSpecString ( field = predictedFieldName , inferenceElement = InferenceElement . prediction , metric = movingAverageBaselineName , params = { 'window' : metricWindow , "errorMetric" : "altMAPE" , "mean_window" : 200 , "steps" : steps } ) )
# Metrics for classification
elif inferenceType in ( InferenceType . TemporalClassification ) :
    metricName = 'avg_err'
    trivialErrorMetric = 'avg_err'
    oneGramErrorMetric = 'avg_err'
    movingAverageBaselineName = 'moving_mode'
    optimizeMetricSpec , optimizeMetricLabel = _generateMetricSpecString ( inferenceElement = InferenceElement . classification , metric = metricName , params = { 'window' : metricWindow } , returnLabel = True )
    metricSpecStrings . append ( optimizeMetricSpec )
    if options [ "runBaselines" ] : # If temporal , generate the trivial predictor metric
        if inferenceType == InferenceType . TemporalClassification :
            metricSpecStrings . append ( _generateMetricSpecString ( inferenceElement = InferenceElement . classification , metric = "trivial" , params = { 'window' : metricWindow , "errorMetric" : trivialErrorMetric } ) )
            metricSpecStrings . append ( _generateMetricSpecString ( inferenceElement = InferenceElement . classification , metric = "two_gram" , params = { 'window' : metricWindow , "errorMetric" : oneGramErrorMetric } ) )
            metricSpecStrings . append ( _generateMetricSpecString ( inferenceElement = InferenceElement . classification , metric = movingAverageBaselineName , params = { 'window' : metricWindow , "errorMetric" : "avg_err" , "mode_window" : 200 } ) )
    # Custom Error Metric
    if not options [ "customErrorMetric" ] == None : # If errorWindow is not specified , make it equal to the default window
        if not "errorWindow" in options [ "customErrorMetric" ] :
            options [ "customErrorMetric" ] [ "errorWindow" ] = metricWindow
        optimizeMetricSpec = _generateMetricSpecString ( inferenceElement = InferenceElement . classification , metric = "custom" , params = options [ "customErrorMetric" ] )
        optimizeMetricLabel = ".*custom_error_metric.*"
        metricSpecStrings . append ( optimizeMetricSpec )
# If plug in the predictionSteps variable for any dynamically generated
# prediction steps
if options [ 'dynamicPredictionSteps' ] :
    for i in range ( len ( metricSpecStrings ) ) :
        metricSpecStrings [ i ] = metricSpecStrings [ i ] . replace ( "'$REPLACE_ME'" , "predictionSteps" )
    optimizeMetricLabel = optimizeMetricLabel . replace ( "'$REPLACE_ME'" , ".*" )
return metricSpecStrings , optimizeMetricLabel


signature:
def _generateExtraMetricSpecs ( options ) :
    """Generates the non - default metrics specified by the expGenerator params"""

implementation:
_metricSpecSchema = { 'properties' : { } }
results = [ ]
for metric in options [ 'metrics' ] :
    for propertyName in _metricSpecSchema [ 'properties' ] . keys ( ) :
        _getPropertyValue ( _metricSpecSchema , propertyName , metric )
    specString , label = _generateMetricSpecString ( field = metric [ 'field' ] , metric = metric [ 'metric' ] , params = metric [ 'params' ] , inferenceElement = metric [ 'inferenceElement' ] , returnLabel = True )
    if metric [ 'logged' ] :
        options [ 'loggedMetrics' ] . append ( label )
    results . append ( specString )
return results


signature:
def _getPredictedField ( options ) :
    """Gets the predicted field and it ' s datatype from the options dictionary
 Returns : ( predictedFieldName , predictedFieldType )"""

implementation:
if not options [ 'inferenceArgs' ] or not options [ 'inferenceArgs' ] [ 'predictedField' ] :
    return None , None
predictedField = options [ 'inferenceArgs' ] [ 'predictedField' ]
predictedFieldInfo = None
includedFields = options [ 'includedFields' ]
for info in includedFields :
    if info [ 'fieldName' ] == predictedField :
        predictedFieldInfo = info
        break
if predictedFieldInfo is None :
    raise ValueError ( "Predicted field '%s' does not exist in included fields." % predictedField )
predictedFieldType = predictedFieldInfo [ 'fieldType' ]
return predictedField , predictedFieldType


signature:
def _generateInferenceArgs ( options , tokenReplacements ) :
    """Generates the token substitutions related to the predicted field
 and the supplemental arguments for prediction"""

implementation:
inferenceType = options [ 'inferenceType' ]
optionInferenceArgs = options . get ( 'inferenceArgs' , None )
resultInferenceArgs = { }
predictedField = _getPredictedField ( options ) [ 0 ]
if inferenceType in ( InferenceType . TemporalNextStep , InferenceType . TemporalAnomaly ) :
    assert predictedField , "Inference Type '%s' needs a predictedField " "specified in the inferenceArgs dictionary" % inferenceType
if optionInferenceArgs : # If we will be using a dynamically created predictionSteps , plug in that
# variable name in place of the constant scalar value
    if options [ 'dynamicPredictionSteps' ] :
        altOptionInferenceArgs = copy . deepcopy ( optionInferenceArgs )
        altOptionInferenceArgs [ 'predictionSteps' ] = '$REPLACE_ME'
        resultInferenceArgs = pprint . pformat ( altOptionInferenceArgs )
        resultInferenceArgs = resultInferenceArgs . replace ( "'$REPLACE_ME'" , '[predictionSteps]' )
    else :
        resultInferenceArgs = pprint . pformat ( optionInferenceArgs )
tokenReplacements [ '\$INFERENCE_ARGS' ] = resultInferenceArgs
tokenReplacements [ '\$PREDICTION_FIELD' ] = predictedField


signature:
def expGenerator ( args ) :
    """Parses , validates , and executes command - line options ;
 On success : Performs requested operation and exits program normally
 On Error : Dumps exception / error info in JSON format to stdout and exits the
 program with non - zero status ."""

implementation:
# Parse command line options
parser = OptionParser ( )
parser . set_usage ( "%prog [options] --description='{json object with args}'\n" + "%prog [options] --descriptionFromFile='{filename}'\n" + "%prog [options] --showSchema" )
parser . add_option ( "--description" , dest = "description" , help = "Tells ExpGenerator to generate an experiment description.py and " "permutations.py file using the given JSON formatted experiment " "description string." )
parser . add_option ( "--descriptionFromFile" , dest = 'descriptionFromFile' , help = "Tells ExpGenerator to open the given filename and use it's " "contents as the JSON formatted experiment description." )
parser . add_option ( "--claDescriptionTemplateFile" , dest = 'claDescriptionTemplateFile' , default = 'claDescriptionTemplate.tpl' , help = "The file containing the template description file for " " ExpGenerator [default: %default]" )
parser . add_option ( "--showSchema" , action = "store_true" , dest = "showSchema" , help = "Prints the JSON schemas for the --description arg." )
parser . add_option ( "--version" , dest = 'version' , default = 'v2' , help = "Generate the permutations file for this version of hypersearch." " Possible choices are 'v1' and 'v2' [default: %default]." )
parser . add_option ( "--outDir" , dest = "outDir" , default = None , help = "Where to generate experiment. If not specified, " "then a temp directory will be created" )
( options , remainingArgs ) = parser . parse_args ( args )
# print ( " OPTIONS = % s " % ( str ( options ) ) )
# Check for unprocessed args
if len ( remainingArgs ) > 0 :
    raise _InvalidCommandArgException ( _makeUsageErrorStr ( "Unexpected command-line args: <%s>" % ( ' ' . join ( remainingArgs ) , ) , parser . get_usage ( ) ) )
# Check for use of mutually - exclusive options
activeOptions = filter ( lambda x : getattr ( options , x ) != None , ( 'description' , 'showSchema' ) )
if len ( activeOptions ) > 1 :
    raise _InvalidCommandArgException ( _makeUsageErrorStr ( ( "The specified command options are " + "mutually-exclusive: %s" ) % ( activeOptions , ) , parser . get_usage ( ) ) )
# Process requests
if options . showSchema :
    _handleShowSchemaOption ( )
elif options . description :
    _handleDescriptionOption ( options . description , options . outDir , parser . get_usage ( ) , hsVersion = options . version , claDescriptionTemplateFile = options . claDescriptionTemplateFile )
elif options . descriptionFromFile :
    _handleDescriptionFromFileOption ( options . descriptionFromFile , options . outDir , parser . get_usage ( ) , hsVersion = options . version , claDescriptionTemplateFile = options . claDescriptionTemplateFile )
else :
    raise _InvalidCommandArgException ( _makeUsageErrorStr ( "Error in validating command options. No option " "provided:\n" , parser . get_usage ( ) ) )


signature:
def parseTimestamp ( s ) :
    """Parses a textual datetime format and return a Python datetime object .
 The supported format is : ` ` yyyy - mm - dd h : m : s . ms ` `
 The time component is optional .
 - hours are 00 . . 23 ( no AM / PM )
 - minutes are 00 . . 59
 - seconds are 00 . . 59
 - micro - seconds are 00000 . . 99999
 : param s : ( string ) input time text
 : return : ( datetime . datetime )"""

implementation:
s = s . strip ( )
for pattern in DATETIME_FORMATS :
    try :
        return datetime . datetime . strptime ( s , pattern )
    except ValueError :
        pass
raise ValueError ( 'The provided timestamp %s is malformed. The supported ' 'formats are: [%s]' % ( s , ', ' . join ( DATETIME_FORMATS ) ) )


signature:
def parseBool ( s ) :
    """String to boolean
 : param s : ( string )
 : return : ( bool )"""

implementation:
l = s . lower ( )
if l in ( "true" , "t" , "1" ) :
    return True
if l in ( "false" , "f" , "0" ) :
    return False
raise Exception ( "Unable to convert string '%s' to a boolean value" % s )


signature:
def escape ( s ) :
    """Escape commas , tabs , newlines and dashes in a string
 Commas are encoded as tabs .
 : param s : ( string ) to escape
 : returns : ( string ) escaped string"""

implementation:
if s is None :
    return ''
assert isinstance ( s , basestring ) , "expected %s but got %s; value=%s" % ( basestring , type ( s ) , s )
s = s . replace ( '\\' , '\\\\' )
s = s . replace ( '\n' , '\\n' )
s = s . replace ( '\t' , '\\t' )
s = s . replace ( ',' , '\t' )
return s


signature:
def unescape ( s ) :
    """Unescapes a string that may contain commas , tabs , newlines and dashes
 Commas are decoded from tabs .
 : param s : ( string ) to unescape
 : returns : ( string ) unescaped string"""

implementation:
assert isinstance ( s , basestring )
s = s . replace ( '\t' , ',' )
s = s . replace ( '\\,' , ',' )
s = s . replace ( '\\n' , '\n' )
s = s . replace ( '\\\\' , '\\' )
return s


signature:
def parseSdr ( s ) :
    """Parses a string containing only 0 ' s and 1 ' s and return a Python list object .
 : param s : ( string ) string to parse
 : returns : ( list ) SDR out"""

implementation:
assert isinstance ( s , basestring )
sdr = [ int ( c ) for c in s if c in ( "0" , "1" ) ]
if len ( sdr ) != len ( s ) :
    raise ValueError ( "The provided string %s is malformed. The string should " "have only 0's and 1's." )
return sdr


signature:
def parseStringList ( s ) :
    """Parse a string of space - separated numbers , returning a Python list .
 : param s : ( string ) to parse
 : returns : ( list ) binary SDR"""

implementation:
assert isinstance ( s , basestring )
return [ int ( i ) for i in s . split ( ) ]


signature:
def coordinatesFromIndex ( index , dimensions ) :
    """Translate an index into coordinates , using the given coordinate system .
 Similar to ` ` numpy . unravel _ index ` ` .
 : param index : ( int ) The index of the point . The coordinates are expressed as a
 single index by using the dimensions as a mixed radix definition . For
 example , in dimensions 42x10 , the point [ 1 , 4 ] is index
 1*420 + 4*10 = 460.
 : param dimensions ( list of ints ) The coordinate system .
 : returns : ( list ) of coordinates of length ` ` len ( dimensions ) ` ` ."""

implementation:
coordinates = [ 0 ] * len ( dimensions )
shifted = index
for i in xrange ( len ( dimensions ) - 1 , 0 , - 1 ) :
    coordinates [ i ] = shifted % dimensions [ i ]
    shifted = shifted / dimensions [ i ]
coordinates [ 0 ] = shifted
return coordinates


signature:
def indexFromCoordinates ( coordinates , dimensions ) :
    """Translate coordinates into an index , using the given coordinate system .
 Similar to ` ` numpy . ravel _ multi _ index ` ` .
 : param coordinates : ( list of ints ) A list of coordinates of length
 ` ` dimensions . size ( ) ` ` .
 : param dimensions : ( list of ints ) The coordinate system .
 : returns : ( int ) The index of the point . The coordinates are expressed as a
 single index by using the dimensions as a mixed radix definition .
 For example , in dimensions 42x10 , the point [ 1 , 4 ] is index
 1*420 + 4*10 = 460."""

implementation:
index = 0
for i , dimension in enumerate ( dimensions ) :
    index *= dimension
    index += coordinates [ i ]
return index


signature:
def neighborhood ( centerIndex , radius , dimensions ) :
    """Get the points in the neighborhood of a point .
 A point ' s neighborhood is the n - dimensional hypercube with sides ranging
 [ center - radius , center + radius ] , inclusive . For example , if there are two
 dimensions and the radius is 3 , the neighborhood is 6x6 . Neighborhoods are
 truncated when they are near an edge .
 This is designed to be fast . In C + + it ' s fastest to iterate through neighbors
 one by one , calculating them on - demand rather than creating a list of them .
 But in Python it ' s faster to build up the whole list in batch via a few calls
 to C code rather than calculating them on - demand with lots of calls to Python
 code .
 : param centerIndex : ( int ) The index of the point . The coordinates are
 expressed as a single index by using the dimensions as a mixed radix
 definition . For example , in dimensions 42x10 , the point [ 1 , 4 ] is index
 1*420 + 4*10 = 460.
 : param radius : ( int ) The radius of this neighborhood about the
 ` ` centerIndex ` ` .
 : param dimensions : ( indexable sequence ) The dimensions of the world outside
 this neighborhood .
 : returns : ( numpy array ) The points in the neighborhood , including
 ` ` centerIndex ` ` ."""

implementation:
centerPosition = coordinatesFromIndex ( centerIndex , dimensions )
intervals = [ ]
for i , dimension in enumerate ( dimensions ) :
    left = max ( 0 , centerPosition [ i ] - radius )
    right = min ( dimension - 1 , centerPosition [ i ] + radius )
    intervals . append ( xrange ( left , right + 1 ) )
coords = numpy . array ( list ( itertools . product ( * intervals ) ) )
return numpy . ravel_multi_index ( coords . T , dimensions )


signature:
def encodeIntoArray ( self , inputData , output ) :
    """See ` nupic . encoders . base . Encoder ` for more information .
 @ param inputData ( tuple ) Contains coordinate ( numpy . array , N - dimensional
 integer coordinate ) and radius ( int )
 @ param output ( numpy . array ) Stores encoded SDR in this numpy array"""

implementation:
( coordinate , radius ) = inputData
assert isinstance ( radius , int ) , ( "Expected integer radius, got: {} ({})" . format ( radius , type ( radius ) ) )
neighbors = self . _neighbors ( coordinate , radius )
winners = self . _topWCoordinates ( neighbors , self . w )
bitFn = lambda coordinate : self . _bitForCoordinate ( coordinate , self . n )
indices = numpy . array ( [ bitFn ( w ) for w in winners ] )
output [ : ] = 0
output [ indices ] = 1


signature:
def _neighbors ( coordinate , radius ) :
    """Returns coordinates around given coordinate , within given radius .
 Includes given coordinate .
 @ param coordinate ( numpy . array ) N - dimensional integer coordinate
 @ param radius ( int ) Radius around ` coordinate `
 @ return ( numpy . array ) List of coordinates"""

implementation:
ranges = ( xrange ( n - radius , n + radius + 1 ) for n in coordinate . tolist ( ) )
return numpy . array ( list ( itertools . product ( * ranges ) ) )


signature:
def _topWCoordinates ( cls , coordinates , w ) :
    """Returns the top W coordinates by order .
 @ param coordinates ( numpy . array ) A 2D numpy array , where each element
 is a coordinate
 @ param w ( int ) Number of top coordinates to return
 @ return ( numpy . array ) A subset of ` coordinates ` , containing only the
 top ones by order"""

implementation:
orders = numpy . array ( [ cls . _orderForCoordinate ( c ) for c in coordinates . tolist ( ) ] )
indices = numpy . argsort ( orders ) [ - w : ]
return coordinates [ indices ]


signature:
def _hashCoordinate ( coordinate ) :
    """Hash a coordinate to a 64 bit integer ."""

implementation:
coordinateStr = "," . join ( str ( v ) for v in coordinate )
# Compute the hash and convert to 64 bit int .
hash = int ( int ( hashlib . md5 ( coordinateStr ) . hexdigest ( ) , 16 ) % ( 2 ** 64 ) )
return hash


signature:
def _orderForCoordinate ( cls , coordinate ) :
    """Returns the order for a coordinate .
 @ param coordinate ( numpy . array ) Coordinate
 @ return ( float ) A value in the interval [ 0 , 1 ) , representing the
 order of the coordinate"""

implementation:
seed = cls . _hashCoordinate ( coordinate )
rng = Random ( seed )
return rng . getReal64 ( )


signature:
def _bitForCoordinate ( cls , coordinate , n ) :
    """Maps the coordinate to a bit in the SDR .
 @ param coordinate ( numpy . array ) Coordinate
 @ param n ( int ) The number of available bits in the SDR
 @ return ( int ) The index to a bit in the SDR"""

implementation:
seed = cls . _hashCoordinate ( coordinate )
rng = Random ( seed )
return rng . getUInt32 ( n )


signature:
def binSearch ( arr , val ) :
    """Function for running binary search on a sorted list .
 : param arr : ( list ) a sorted list of integers to search
 : param val : ( int ) a integer to search for in the sorted array
 : returns : ( int ) the index of the element if it is found and - 1 otherwise ."""

implementation:
i = bisect_left ( arr , val )
if i != len ( arr ) and arr [ i ] == val :
    return i
return - 1


signature:
def createSegment ( self , cell ) :
    """Adds a new segment on a cell .
 : param cell : ( int ) Cell index
 : returns : ( int ) New segment index"""

implementation:
cellData = self . _cells [ cell ]
if len ( self . _freeFlatIdxs ) > 0 :
    flatIdx = self . _freeFlatIdxs . pop ( )
else :
    flatIdx = self . _nextFlatIdx
    self . _segmentForFlatIdx . append ( None )
    self . _nextFlatIdx += 1
ordinal = self . _nextSegmentOrdinal
self . _nextSegmentOrdinal += 1
segment = Segment ( cell , flatIdx , ordinal )
cellData . _segments . append ( segment )
self . _segmentForFlatIdx [ flatIdx ] = segment
return segment


signature:
def destroySegment ( self , segment ) :
    """Destroys a segment .
 : param segment : ( : class : ` Segment ` ) representing the segment to be destroyed ."""

implementation:
# Remove the synapses from all data structures outside this Segment .
for synapse in segment . _synapses :
    self . _removeSynapseFromPresynapticMap ( synapse )
self . _numSynapses -= len ( segment . _synapses )
# Remove the segment from the cell ' s list .
segments = self . _cells [ segment . cell ] . _segments
i = segments . index ( segment )
del segments [ i ]
# Free the flatIdx and remove the final reference so the Segment can be
# garbage - collected .
self . _freeFlatIdxs . append ( segment . flatIdx )
self . _segmentForFlatIdx [ segment . flatIdx ] = None


signature:
def createSynapse ( self , segment , presynapticCell , permanence ) :
    """Creates a new synapse on a segment .
 : param segment : ( : class : ` Segment ` ) Segment object for synapse to be synapsed
 to .
 : param presynapticCell : ( int ) Source cell index .
 : param permanence : ( float ) Initial permanence of synapse .
 : returns : ( : class : ` Synapse ` ) created synapse"""

implementation:
idx = len ( segment . _synapses )
synapse = Synapse ( segment , presynapticCell , permanence , self . _nextSynapseOrdinal )
self . _nextSynapseOrdinal += 1
segment . _synapses . add ( synapse )
self . _synapsesForPresynapticCell [ presynapticCell ] . add ( synapse )
self . _numSynapses += 1
return synapse


signature:
def destroySynapse ( self , synapse ) :
    """Destroys a synapse .
 : param synapse : ( : class : ` Synapse ` ) synapse to destroy"""

implementation:
self . _numSynapses -= 1
self . _removeSynapseFromPresynapticMap ( synapse )
synapse . segment . _synapses . remove ( synapse )


signature:
def computeActivity ( self , activePresynapticCells , connectedPermanence ) :
    """Compute each segment ' s number of active synapses for a given input .
 In the returned lists , a segment ' s active synapse count is stored at index
 ` ` segment . flatIdx ` ` .
 : param activePresynapticCells : ( iter ) Active cells .
 : param connectedPermanence : ( float ) Permanence threshold for a synapse to be
 considered connected
 : returns : ( tuple ) ( ` ` numActiveConnectedSynapsesForSegment ` ` [ list ] ,
 ` ` numActivePotentialSynapsesForSegment ` ` [ list ] )"""

implementation:
numActiveConnectedSynapsesForSegment = [ 0 ] * self . _nextFlatIdx
numActivePotentialSynapsesForSegment = [ 0 ] * self . _nextFlatIdx
threshold = connectedPermanence - EPSILON
for cell in activePresynapticCells :
    for synapse in self . _synapsesForPresynapticCell [ cell ] :
        flatIdx = synapse . segment . flatIdx
        numActivePotentialSynapsesForSegment [ flatIdx ] += 1
        if synapse . permanence > threshold :
            numActiveConnectedSynapsesForSegment [ flatIdx ] += 1
return ( numActiveConnectedSynapsesForSegment , numActivePotentialSynapsesForSegment )


signature:
def numSegments ( self , cell = None ) :
    """Returns the number of segments .
 : param cell : ( int ) Optional parameter to get the number of segments on a
 cell .
 : returns : ( int ) Number of segments on all cells if cell is not specified , or
 on a specific specified cell"""

implementation:
if cell is not None :
    return len ( self . _cells [ cell ] . _segments )
return self . _nextFlatIdx - len ( self . _freeFlatIdxs )


signature:
def segmentPositionSortKey ( self , segment ) :
    """Return a numeric key for sorting this segment . This can be used with the
 python built - in ` ` sorted ( ) ` ` function .
 : param segment : ( : class : ` Segment ` ) within this : class : ` Connections `
 instance .
 : returns : ( float ) A numeric key for sorting ."""

implementation: return segment . cell + ( segment . _ordinal / float ( self . _nextSegmentOrdinal ) )


signature:
def write ( self , proto ) :
    """Writes serialized data to proto object .
 : param proto : ( DynamicStructBuilder ) Proto object"""

implementation:
protoCells = proto . init ( 'cells' , self . numCells )
for i in xrange ( self . numCells ) :
    segments = self . _cells [ i ] . _segments
    protoSegments = protoCells [ i ] . init ( 'segments' , len ( segments ) )
    for j , segment in enumerate ( segments ) :
        synapses = segment . _synapses
        protoSynapses = protoSegments [ j ] . init ( 'synapses' , len ( synapses ) )
        for k , synapse in enumerate ( sorted ( synapses , key = lambda s : s . _ordinal ) ) :
            protoSynapses [ k ] . presynapticCell = synapse . presynapticCell
            protoSynapses [ k ] . permanence = synapse . permanence


signature:
def read ( cls , proto ) :
    """Reads deserialized data from proto object
 : param proto : ( DynamicStructBuilder ) Proto object
 : returns : ( : class : ` Connections ` ) instance"""

implementation:
# pylint : disable = W0212
protoCells = proto . cells
connections = cls ( len ( protoCells ) )
for cellIdx , protoCell in enumerate ( protoCells ) :
    protoCell = protoCells [ cellIdx ]
    protoSegments = protoCell . segments
    connections . _cells [ cellIdx ] = CellData ( )
    segments = connections . _cells [ cellIdx ] . _segments
    for segmentIdx , protoSegment in enumerate ( protoSegments ) :
        segment = Segment ( cellIdx , connections . _nextFlatIdx , connections . _nextSegmentOrdinal )
        segments . append ( segment )
        connections . _segmentForFlatIdx . append ( segment )
        connections . _nextFlatIdx += 1
        connections . _nextSegmentOrdinal += 1
        synapses = segment . _synapses
        protoSynapses = protoSegment . synapses
        for synapseIdx , protoSynapse in enumerate ( protoSynapses ) :
            presynapticCell = protoSynapse . presynapticCell
            synapse = Synapse ( segment , presynapticCell , protoSynapse . permanence , ordinal = connections . _nextSynapseOrdinal )
            connections . _nextSynapseOrdinal += 1
            synapses . add ( synapse )
            connections . _synapsesForPresynapticCell [ presynapticCell ] . add ( synapse )
            connections . _numSynapses += 1
# pylint : enable = W0212
return connections


signature:
def getString ( cls , prop ) :
    """Retrieve the requested property as a string . If property does not exist ,
 then KeyError will be raised .
 : param prop : ( string ) name of the property
 : raises : KeyError
 : returns : ( string ) property value"""

implementation:
if cls . _properties is None :
    cls . _readStdConfigFiles ( )
# Allow configuration properties to be overridden via environment variables
envValue = os . environ . get ( "%s%s" % ( cls . envPropPrefix , prop . replace ( '.' , '_' ) ) , None )
if envValue is not None :
    return envValue
return cls . _properties [ prop ]


signature:
def getBool ( cls , prop ) :
    """Retrieve the requested property and return it as a bool . If property
 does not exist , then KeyError will be raised . If the property value is
 neither 0 nor 1 , then ValueError will be raised
 : param prop : ( string ) name of the property
 : raises : KeyError , ValueError
 : returns : ( bool ) property value"""

implementation:
value = cls . getInt ( prop )
if value not in ( 0 , 1 ) :
    raise ValueError ( "Expected 0 or 1, but got %r in config property %s" % ( value , prop ) )
return bool ( value )


signature:
def set ( cls , prop , value ) :
    """Set the value of the given configuration property .
 : param prop : ( string ) name of the property
 : param value : ( object ) value to set"""

implementation:
if cls . _properties is None :
    cls . _readStdConfigFiles ( )
cls . _properties [ prop ] = str ( value )


signature:
def dict ( cls ) :
    """Return a dict containing all of the configuration properties
 : returns : ( dict ) containing all configuration properties ."""

implementation:
if cls . _properties is None :
    cls . _readStdConfigFiles ( )
# Make a copy so we can update any current values obtained from environment
# variables
result = dict ( cls . _properties )
keys = os . environ . keys ( )
replaceKeys = filter ( lambda x : x . startswith ( cls . envPropPrefix ) , keys )
for envKey in replaceKeys :
    key = envKey [ len ( cls . envPropPrefix ) : ]
    key = key . replace ( '_' , '.' )
    result [ key ] = os . environ [ envKey ]
return result


signature:
def readConfigFile ( cls , filename , path = None ) :
    """Parse the given XML file and store all properties it describes .
 : param filename : ( string ) name of XML file to parse ( no path )
 : param path : ( string ) path of the XML file . If None , then use the standard
 configuration search path ."""

implementation:
properties = cls . _readConfigFile ( filename , path )
# Create properties dict if necessary
if cls . _properties is None :
    cls . _properties = dict ( )
for name in properties :
    if 'value' in properties [ name ] :
        cls . _properties [ name ] = properties [ name ] [ 'value' ]


signature:
def findConfigFile ( cls , filename ) :
    """Search the configuration path ( specified via the NTA _ CONF _ PATH
 environment variable ) for the given filename . If found , return the complete
 path to the file .
 : param filename : ( string ) name of file to locate"""

implementation:
paths = cls . getConfigPaths ( )
for p in paths :
    testPath = os . path . join ( p , filename )
    if os . path . isfile ( testPath ) :
        return os . path . join ( p , filename )


signature:
def getConfigPaths ( cls ) :
    """Return the list of paths to search for configuration files .
 : returns : ( list ) of paths"""

implementation:
configPaths = [ ]
if cls . _configPaths is not None :
    return cls . _configPaths
else :
    if 'NTA_CONF_PATH' in os . environ :
        configVar = os . environ [ 'NTA_CONF_PATH' ]
        # Return as a list of paths
        configPaths = configVar . split ( os . pathsep )
    return configPaths


signature:
def addNoise ( input , noise = 0.1 , doForeground = True , doBackground = True ) :
    """Add noise to the given input .
 Parameters :
 input : the input to add noise to
 noise : how much noise to add
 doForeground : If true , turn off some of the 1 bits in the input
 doBackground : If true , turn on some of the 0 bits in the input"""

implementation:
if doForeground and doBackground :
    return numpy . abs ( input - ( numpy . random . random ( input . shape ) < noise ) )
else :
    if doForeground :
        return numpy . logical_and ( input , numpy . random . random ( input . shape ) > noise )
    if doBackground :
        return numpy . logical_or ( input , numpy . random . random ( input . shape ) < noise )
return input


signature:
def generateCoincMatrix ( nCoinc = 10 , length = 500 , activity = 50 ) :
    """Generate a coincidence matrix . This is used to generate random inputs to the
 temporal learner and to compare the predicted output against .
 It generates a matrix of nCoinc rows , each row has length ' length ' and has
 a total of ' activity ' bits on .
 Parameters :
 nCoinc : the number of rows to generate
 length : the length of each row
 activity : the number of ones to put into each row ."""

implementation:
coincMatrix0 = SM32 ( int ( nCoinc ) , int ( length ) )
theOnes = numpy . array ( [ 1.0 ] * activity , dtype = numpy . float32 )
for rowIdx in xrange ( nCoinc ) :
    coinc = numpy . array ( random . sample ( xrange ( length ) , activity ) , dtype = numpy . uint32 )
    coinc . sort ( )
    coincMatrix0 . setRowFromSparse ( rowIdx , coinc , theOnes )
# This is the right code to use , it ' s faster , but it derails the unit
# testing of the pooling for now .
coincMatrix = SM32 ( int ( nCoinc ) , int ( length ) )
coincMatrix . initializeWithFixedNNZR ( activity )
return coincMatrix0


signature:
def generateVectors ( numVectors = 100 , length = 500 , activity = 50 ) :
    """Generate a list of random sparse distributed vectors . This is used to generate
 training vectors to the spatial or temporal learner and to compare the predicted
 output against .
 It generates a list of ' numVectors ' elements , each element has length ' length '
 and has a total of ' activity ' bits on .
 Parameters :
 numVectors : the number of vectors to generate
 length : the length of each row
 activity : the number of ones to put into each row ."""

implementation:
vectors = [ ]
coinc = numpy . zeros ( length , dtype = 'int32' )
indexList = range ( length )
for i in xrange ( numVectors ) :
    coinc [ : ] = 0
    coinc [ random . sample ( indexList , activity ) ] = 1
    vectors . append ( coinc . copy ( ) )
return vectors


signature:
def generateSimpleSequences ( nCoinc = 10 , seqLength = [ 5 , 6 , 7 ] , nSeq = 100 ) :
    """Generate a set of simple sequences . The elements of the sequences will be
 integers from 0 to ' nCoinc ' - 1 . The length of each sequence will be
 randomly chosen from the ' seqLength ' list .
 Parameters :
 nCoinc : the number of elements available to use in the sequences
 seqLength : a list of possible sequence lengths . The length of each
 sequence will be randomly chosen from here .
 nSeq : The number of sequences to generate
 retval : a list of sequences . Each sequence is itself a list
 containing the coincidence indices for that sequence ."""

implementation:
coincList = range ( nCoinc )
seqList = [ ]
for i in xrange ( nSeq ) :
    if max ( seqLength ) <= nCoinc :
        seqList . append ( random . sample ( coincList , random . choice ( seqLength ) ) )
    else :
        len = random . choice ( seqLength )
        seq = [ ]
        for x in xrange ( len ) :
            seq . append ( random . choice ( coincList ) )
        seqList . append ( seq )
return seqList


signature:
def generateHubSequences ( nCoinc = 10 , hubs = [ 2 , 6 ] , seqLength = [ 5 , 6 , 7 ] , nSeq = 100 ) :
    """Generate a set of hub sequences . These are sequences which contain a hub
 element in the middle . The elements of the sequences will be integers
 from 0 to ' nCoinc ' - 1 . The hub elements will only appear in the middle of
 each sequence . The length of each sequence will be randomly chosen from the
 ' seqLength ' list .
 Parameters :
 nCoinc : the number of elements available to use in the sequences
 hubs : which of the elements will be used as hubs .
 seqLength : a list of possible sequence lengths . The length of each
 sequence will be randomly chosen from here .
 nSeq : The number of sequences to generate
 retval : a list of sequences . Each sequence is itself a list
 containing the coincidence indices for that sequence ."""

implementation:
coincList = range ( nCoinc )
for hub in hubs :
    coincList . remove ( hub )
seqList = [ ]
for i in xrange ( nSeq ) :
    length = random . choice ( seqLength ) - 1
    seq = random . sample ( coincList , length )
    seq . insert ( length // 2 , random . choice ( hubs ) )
    seqList . append ( seq )
return seqList


signature:
def generateSimpleCoincMatrix ( nCoinc = 10 , length = 500 , activity = 50 ) :
    """Generate a non overlapping coincidence matrix . This is used to generate random
 inputs to the temporal learner and to compare the predicted output against .
 It generates a matrix of nCoinc rows , each row has length ' length ' and has
 a total of ' activity ' bits on .
 Parameters :
 nCoinc : the number of rows to generate
 length : the length of each row
 activity : the number of ones to put into each row ."""

implementation:
assert nCoinc * activity <= length , "can't generate non-overlapping coincidences"
coincMatrix = SM32 ( 0 , length )
coinc = numpy . zeros ( length , dtype = 'int32' )
for i in xrange ( nCoinc ) :
    coinc [ : ] = 0
    coinc [ i * activity : ( i + 1 ) * activity ] = 1
    coincMatrix . addRow ( coinc )
return coincMatrix


signature:
def generateSequences ( nPatterns = 10 , patternLen = 500 , patternActivity = 50 , hubs = [ 2 , 6 ] , seqLength = [ 5 , 6 , 7 ] , nSimpleSequences = 50 , nHubSequences = 50 ) :
    """Generate a set of simple and hub sequences . A simple sequence contains
 a randomly chosen set of elements from 0 to ' nCoinc - 1 ' . A hub sequence
 always contains a hub element in the middle of it .
 Parameters :
 nPatterns : the number of patterns to use in the sequences .
 patternLen : The number of elements in each pattern
 patternActivity : The number of elements that should be active in
 each pattern
 hubs : which of the elements will be used as hubs .
 seqLength : a list of possible sequence lengths . The length of each
 sequence will be randomly chosen from here .
 nSimpleSequences : The number of simple sequences to generate
 nHubSequences : The number of hub sequences to generate
 retval : ( seqList , patterns )
 seqList : a list of sequences . Each sequence is itself a list
 containing the input pattern indices for that sequence .
 patterns : the input patterns used in the seqList ."""

implementation:
# Create the input patterns
patterns = generateCoincMatrix ( nCoinc = nPatterns , length = patternLen , activity = patternActivity )
# Create the raw sequences
seqList = generateSimpleSequences ( nCoinc = nPatterns , seqLength = seqLength , nSeq = nSimpleSequences ) + generateHubSequences ( nCoinc = nPatterns , hubs = hubs , seqLength = seqLength , nSeq = nHubSequences )
# Return results
return ( seqList , patterns )


signature:
def generateL2Sequences ( nL1Patterns = 10 , l1Hubs = [ 2 , 6 ] , l1SeqLength = [ 5 , 6 , 7 ] , nL1SimpleSequences = 50 , nL1HubSequences = 50 , l1Pooling = 4 , perfectStability = False , spHysteresisFactor = 1.0 , patternLen = 500 , patternActivity = 50 ) :
    """Generate the simulated output from a spatial pooler that ' s sitting
 on top of another spatial pooler / temporal memory pair . The average on - time
 of the outputs from the simulated TM is given by the l1Pooling argument .
 In this routine , L1 refers to the first spatial and temporal memory and L2
 refers to the spatial pooler above that .
 Parameters :
 nL1Patterns : the number of patterns to use in the L1 sequences .
 l1Hubs : which of the elements will be used as hubs .
 l1SeqLength : a list of possible sequence lengths . The length of each
 sequence will be randomly chosen from here .
 nL1SimpleSequences : The number of simple sequences to generate for L1
 nL1HubSequences : The number of hub sequences to generate for L1
 l1Pooling : The number of time steps to pool over in the L1 temporal
 pooler
 perfectStability : If true , then the input patterns represented by the
 sequences generated will have perfect stability over
 l1Pooling time steps . This is the best case ideal input
 to a TM . In actual situations , with an actual SP
 providing input , the stability will always be less than
 this .
 spHystereisFactor : The hysteresisFactor to use in the L2 spatial pooler .
 Only used when perfectStability is False
 patternLen : The number of elements in each pattern output by L2
 patternActivity : The number of elements that should be active in
 each pattern
 @ retval : ( seqList , patterns )
 seqList : a list of sequences output from L2 . Each sequence is
 itself a list containing the input pattern indices for that
 sequence .
 patterns : the input patterns used in the L2 seqList ."""

implementation:
# First , generate the L1 sequences
l1SeqList = generateSimpleSequences ( nCoinc = nL1Patterns , seqLength = l1SeqLength , nSeq = nL1SimpleSequences ) + generateHubSequences ( nCoinc = nL1Patterns , hubs = l1Hubs , seqLength = l1SeqLength , nSeq = nL1HubSequences )
# Generate the L2 SP output from those
spOutput = generateSlowSPOutput ( seqListBelow = l1SeqList , poolingTimeBelow = l1Pooling , outputWidth = patternLen , activity = patternActivity , perfectStability = perfectStability , spHysteresisFactor = spHysteresisFactor )
# Map the spOutput patterns into indices into a pattern matrix which we
# generate now .
outSeq = None
outSeqList = [ ]
outPatterns = SM32 ( 0 , patternLen )
for pattern in spOutput : # If we have a reset vector start a new sequence
    if pattern . sum ( ) == 0 :
        if outSeq is not None :
            outSeqList . append ( outSeq )
        outSeq = [ ]
        continue
    # See if this vector matches a pattern we ' ve already seen before
    patternIdx = None
    if outPatterns . nRows ( ) > 0 : # Find most matching 1 ' s .
        matches = outPatterns . rightVecSumAtNZ ( pattern )
        outCoinc = matches . argmax ( ) . astype ( 'uint32' )
        # See if its number of 1 ' s is the same in the pattern and in the
        # coincidence row . If so , it is an exact match
        numOnes = pattern . sum ( )
        if matches [ outCoinc ] == numOnes and outPatterns . getRow ( int ( outCoinc ) ) . sum ( ) == numOnes :
            patternIdx = outCoinc
    # If no match , add this pattern to our matrix
    if patternIdx is None :
        outPatterns . addRow ( pattern )
        patternIdx = outPatterns . nRows ( ) - 1
    # Store the pattern index into the sequence
    outSeq . append ( patternIdx )
# Put in last finished sequence
if outSeq is not None :
    outSeqList . append ( outSeq )
# Return with the seqList and patterns matrix
return ( outSeqList , outPatterns )


signature:
def vectorsFromSeqList ( seqList , patternMatrix ) :
    """Convert a list of sequences of pattern indices , and a pattern lookup table
 into a an array of patterns
 Parameters :
 seq : the sequence , given as indices into the patternMatrix
 patternMatrix : a SparseMatrix contaning the possible patterns used in
 the sequence ."""

implementation:
totalLen = 0
for seq in seqList :
    totalLen += len ( seq )
vectors = numpy . zeros ( ( totalLen , patternMatrix . shape [ 1 ] ) , dtype = 'bool' )
vecOffset = 0
for seq in seqList :
    seq = numpy . array ( seq , dtype = 'uint32' )
    for idx , coinc in enumerate ( seq ) :
        vectors [ vecOffset ] = patternMatrix . getRow ( int ( coinc ) )
        vecOffset += 1
return vectors


signature:
def sameTMParams ( tp1 , tp2 ) :
    """Given two TM instances , see if any parameters are different ."""

implementation:
result = True
for param in [ "numberOfCols" , "cellsPerColumn" , "initialPerm" , "connectedPerm" , "minThreshold" , "newSynapseCount" , "permanenceInc" , "permanenceDec" , "permanenceMax" , "globalDecay" , "activationThreshold" , "doPooling" , "segUpdateValidDuration" , "burnIn" , "pamLength" , "maxAge" ] :
    if getattr ( tp1 , param ) != getattr ( tp2 , param ) :
        print param , "is different"
        print getattr ( tp1 , param ) , "vs" , getattr ( tp2 , param )
        result = False
return result


signature:
def sameSynapse ( syn , synapses ) :
    """Given a synapse and a list of synapses , check whether this synapse
 exist in the list . A synapse is represented as [ col , cell , permanence ] .
 A synapse matches if col and cell are identical and the permanence value is
 within 0.001."""

implementation:
for s in synapses :
    if ( s [ 0 ] == syn [ 0 ] ) and ( s [ 1 ] == syn [ 1 ] ) and ( abs ( s [ 2 ] - syn [ 2 ] ) <= 0.001 ) :
        return True
return False


signature:
def sameSegment ( seg1 , seg2 ) :
    """Return True if seg1 and seg2 are identical , ignoring order of synapses"""

implementation:
result = True
# check sequence segment , total activations etc . In case any are floats ,
# check that they are within 0.001.
for field in [ 1 , 2 , 3 , 4 , 5 , 6 ] :
    if abs ( seg1 [ 0 ] [ field ] - seg2 [ 0 ] [ field ] ) > 0.001 :
        result = False
# Compare number of synapses
if len ( seg1 [ 1 : ] ) != len ( seg2 [ 1 : ] ) :
    result = False
# Now compare synapses , ignoring order of synapses
for syn in seg2 [ 1 : ] :
    if syn [ 2 ] <= 0 :
        print "A synapse with zero permanence encountered"
        result = False
if result == True :
    for syn in seg1 [ 1 : ] :
        if syn [ 2 ] <= 0 :
            print "A synapse with zero permanence encountered"
            result = False
        res = sameSynapse ( syn , seg2 [ 1 : ] )
        if res == False :
            result = False
return result


signature:
def tmDiff ( tm1 , tm2 , verbosity = 0 , relaxSegmentTests = True ) :
    """Given two TM instances , list the difference between them and returns False
 if there is a difference . This function checks the major parameters . If this
 passes ( and checkLearn is true ) it checks the number of segments on
 each cell . If this passes , checks each synapse on each segment .
 When comparing C + + and Py , the segments are usually in different orders in the
 cells . tmDiff ignores segment order when comparing TM ' s ."""

implementation:
# First check basic parameters . If we fail here , don ' t continue
if sameTMParams ( tm1 , tm2 ) == False :
    print "Two TM's have different parameters"
    return False
result = True
# Compare states at t first , they usually diverge before the structure of the
# cells starts diverging
if ( tm1 . activeState [ 't' ] != tm2 . activeState [ 't' ] ) . any ( ) :
    print 'Active states diverge' , numpy . where ( tm1 . activeState [ 't' ] != tm2 . activeState [ 't' ] )
    result = False
if ( tm1 . predictedState [ 't' ] - tm2 . predictedState [ 't' ] ) . any ( ) :
    print 'Predicted states diverge' , numpy . where ( tm1 . predictedState [ 't' ] != tm2 . predictedState [ 't' ] )
    result = False
# TODO : check confidence at T ( confT )
# Now check some high level learned parameters .
if tm1 . getNumSegments ( ) != tm2 . getNumSegments ( ) :
    print "Number of segments are different" , tm1 . getNumSegments ( ) , tm2 . getNumSegments ( )
    result = False
if tm1 . getNumSynapses ( ) != tm2 . getNumSynapses ( ) :
    print "Number of synapses are different" , tm1 . getNumSynapses ( ) , tm2 . getNumSynapses ( )
    tm1 . printCells ( )
    tm2 . printCells ( )
    result = False
# Check that each cell has the same number of segments and synapses
for c in xrange ( tm1 . numberOfCols ) :
    for i in xrange ( tm2 . cellsPerColumn ) :
        if tm1 . getNumSegmentsInCell ( c , i ) != tm2 . getNumSegmentsInCell ( c , i ) :
            print "Num segments different in cell:" , c , i ,
            print tm1 . getNumSegmentsInCell ( c , i ) , tm2 . getNumSegmentsInCell ( c , i )
            result = False
# If the above tests pass , then check each segment and report differences
# Note that segments in tm1 can be in a different order than tm2 . Here we
# make sure that , for each segment in tm1 , there is an identical segment
# in tm2.
if result == True and not relaxSegmentTests :
    for c in xrange ( tm1 . numberOfCols ) :
        for i in xrange ( tm2 . cellsPerColumn ) :
            nSegs = tm1 . getNumSegmentsInCell ( c , i )
            for segIdx in xrange ( nSegs ) :
                tm1seg = tm1 . getSegmentOnCell ( c , i , segIdx )
                # Loop through all segments in tm2seg and see if any of them match tm1seg
                res = False
                for tm2segIdx in xrange ( nSegs ) :
                    tm2seg = tm2 . getSegmentOnCell ( c , i , tm2segIdx )
                    if sameSegment ( tm1seg , tm2seg ) == True :
                        res = True
                        break
                if res == False :
                    print "\nSegments are different for cell:" , c , i
                    if verbosity >= 1 :
                        print "C++"
                        tm1 . printCell ( c , i )
                        print "Py"
                        tm2 . printCell ( c , i )
                    result = False
if result == True and ( verbosity > 1 ) :
    print "TM's match"
return result


signature:
def tmDiff2 ( tm1 , tm2 , verbosity = 0 , relaxSegmentTests = True , checkLearn = True , checkStates = True ) :
    """Given two TM instances , list the difference between them and returns False
 if there is a difference . This function checks the major parameters . If this
 passes ( and checkLearn is true ) it checks the number of segments on each cell .
 If this passes , checks each synapse on each segment .
 When comparing C + + and Py , the segments are usually in different orders in the
 cells . tmDiff ignores segment order when comparing TM ' s .
 If checkLearn is True , will check learn states as well as all the segments
 If checkStates is True , will check the various state arrays"""

implementation:
# First check basic parameters . If we fail here , don ' t continue
if sameTMParams ( tm1 , tm2 ) == False :
    print "Two TM's have different parameters"
    return False
tm1Label = "<tm_1 (%s)>" % tm1 . __class__ . __name__
tm2Label = "<tm_2 (%s)>" % tm2 . __class__ . __name__
result = True
if checkStates : # Compare states at t first , they usually diverge before the structure of the
# cells starts diverging
    if ( tm1 . infActiveState [ 't' ] != tm2 . infActiveState [ 't' ] ) . any ( ) :
        print 'Active states diverged' , numpy . where ( tm1 . infActiveState [ 't' ] != tm2 . infActiveState [ 't' ] )
        result = False
    if ( tm1 . infPredictedState [ 't' ] - tm2 . infPredictedState [ 't' ] ) . any ( ) :
        print 'Predicted states diverged' , numpy . where ( tm1 . infPredictedState [ 't' ] != tm2 . infPredictedState [ 't' ] )
        result = False
    if checkLearn and ( tm1 . lrnActiveState [ 't' ] - tm2 . lrnActiveState [ 't' ] ) . any ( ) :
        print 'lrnActiveState[t] diverged' , numpy . where ( tm1 . lrnActiveState [ 't' ] != tm2 . lrnActiveState [ 't' ] )
        result = False
    if checkLearn and ( tm1 . lrnPredictedState [ 't' ] - tm2 . lrnPredictedState [ 't' ] ) . any ( ) :
        print 'lrnPredictedState[t] diverged' , numpy . where ( tm1 . lrnPredictedState [ 't' ] != tm2 . lrnPredictedState [ 't' ] )
        result = False
    if checkLearn and abs ( tm1 . getAvgLearnedSeqLength ( ) - tm2 . getAvgLearnedSeqLength ( ) ) > 0.01 :
        print "Average learned sequence lengths differ: " ,
        print tm1 . getAvgLearnedSeqLength ( ) , " vs " , tm2 . getAvgLearnedSeqLength ( )
        result = False
# TODO : check confidence at T ( confT )
# Now check some high level learned parameters .
if tm1 . getNumSegments ( ) != tm2 . getNumSegments ( ) :
    print "Number of segments are different" , tm1 . getNumSegments ( ) , tm2 . getNumSegments ( )
    result = False
if tm1 . getNumSynapses ( ) != tm2 . getNumSynapses ( ) :
    print "Number of synapses are different" , tm1 . getNumSynapses ( ) , tm2 . getNumSynapses ( )
    if verbosity >= 3 :
        print "%s: " % tm1Label ,
        tm1 . printCells ( )
        print "\n%s  : " % tm2Label ,
        tm2 . printCells ( )
    # result = False
# Check that each cell has the same number of segments and synapses
for c in xrange ( tm1 . numberOfCols ) :
    for i in xrange ( tm2 . cellsPerColumn ) :
        if tm1 . getNumSegmentsInCell ( c , i ) != tm2 . getNumSegmentsInCell ( c , i ) :
            print "Num segments different in cell:" , c , i ,
            print tm1 . getNumSegmentsInCell ( c , i ) , tm2 . getNumSegmentsInCell ( c , i )
            result = False
# If the above tests pass , then check each segment and report differences
# Note that segments in tm1 can be in a different order than tm2 . Here we
# make sure that , for each segment in tm1 , there is an identical segment
# in tm2.
if result == True and not relaxSegmentTests and checkLearn :
    for c in xrange ( tm1 . numberOfCols ) :
        for i in xrange ( tm2 . cellsPerColumn ) :
            nSegs = tm1 . getNumSegmentsInCell ( c , i )
            for segIdx in xrange ( nSegs ) :
                tm1seg = tm1 . getSegmentOnCell ( c , i , segIdx )
                # Loop through all segments in tm2seg and see if any of them match tm1seg
                res = False
                for tm2segIdx in xrange ( nSegs ) :
                    tm2seg = tm2 . getSegmentOnCell ( c , i , tm2segIdx )
                    if sameSegment ( tm1seg , tm2seg ) == True :
                        res = True
                        break
                if res == False :
                    print "\nSegments are different for cell:" , c , i
                    result = False
                    if verbosity >= 0 :
                        print "%s : " % tm1Label ,
                        tm1 . printCell ( c , i )
                        print "\n%s  : " % tm2Label ,
                        tm2 . printCell ( c , i )
if result == True and ( verbosity > 1 ) :
    print "TM's match"
return result