signature:
def learn ( self , inputPattern , inputCategory , partitionId = None , isSparse = 0 , rowID = None ) :
    """Train the classifier to associate specified input pattern with a
 particular category .
 : param inputPattern : ( list ) The pattern to be assigned a category . If
 isSparse is 0 , this should be a dense array ( both ON and OFF bits
 present ) . Otherwise , if isSparse > 0 , this should be a list of the
 indices of the non - zero bits in sorted order
 : param inputCategory : ( int ) The category to be associated to the training
 pattern
 : param partitionId : ( int ) partitionID allows you to associate an id with each
 input vector . It can be used to associate input patterns stored in the
 classifier with an external id . This can be useful for debugging or
 visualizing . Another use case is to ignore vectors with a specific id
 during inference ( see description of infer ( ) for details ) . There can be
 at most one partitionId per stored pattern ( i . e . if two patterns are
 within distThreshold , only the first partitionId will be stored ) . This
 is an optional parameter .
 : param isSparse : ( int ) 0 if the input pattern is a dense representation .
 When the input pattern is a list of non - zero indices , then isSparse
 is the number of total bits ( n ) . E . g . for the dense array
 [0 , 1 , 1 , 0 , 0 , 1 ] , isSparse should be ` 0 ` . For the equivalent sparse
 representation [ 1 , 2 , 5 ] ( which specifies the indices of active bits ) ,
 isSparse should be ` 6 ` , which is the total number of bits in the input
 space .
 : param rowID : ( int ) UNKNOWN
 : returns : The number of patterns currently stored in the classifier"""

implementation:
if self . verbosity >= 1 :
    print "%s learn:" % g_debugPrefix
    print "  category:" , int ( inputCategory )
    print "  active inputs:" , _labeledInput ( inputPattern , cellsPerCol = self . cellsPerCol )
if isSparse > 0 :
    assert all ( inputPattern [ i ] <= inputPattern [ i + 1 ] for i in xrange ( len ( inputPattern ) - 1 ) ) , "Sparse inputPattern must be sorted."
    assert all ( bit < isSparse for bit in inputPattern ) , ( "Sparse inputPattern must not index outside the dense " "representation's bounds." )
if rowID is None :
    rowID = self . _iterationIdx
# Dense vectors
if not self . useSparseMemory : # Not supported
    assert self . cellsPerCol == 0 , "not implemented for dense vectors"
    # If the input was given in sparse form , convert it to dense
    if isSparse > 0 :
        denseInput = numpy . zeros ( isSparse )
        denseInput [ inputPattern ] = 1.0
        inputPattern = denseInput
    if self . _specificIndexTraining and not self . _nextTrainingIndices : # Specific index mode without any index provided - skip training
        return self . _numPatterns
    if self . _Memory is None : # Initialize memory with 100 rows and numPatterns = 0
        inputWidth = len ( inputPattern )
        self . _Memory = numpy . zeros ( ( 100 , inputWidth ) )
        self . _numPatterns = 0
        self . _M = self . _Memory [ : self . _numPatterns ]
    addRow = True
    if self . _vt is not None : # Compute projection
        inputPattern = numpy . dot ( self . _vt , inputPattern - self . _mean )
    if self . distThreshold > 0 : # Check if input is too close to an existing input to be accepted
        dist = self . _calcDistance ( inputPattern )
        minDist = dist . min ( )
        addRow = ( minDist >= self . distThreshold )
    if addRow :
        self . _protoSizes = None
        # need to re - compute
        if self . _numPatterns == self . _Memory . shape [ 0 ] : # Double the size of the memory
            self . _doubleMemoryNumRows ( )
        if not self . _specificIndexTraining : # Normal learning - append the new input vector
            self . _Memory [ self . _numPatterns ] = inputPattern
            self . _numPatterns += 1
            self . _categoryList . append ( int ( inputCategory ) )
        else : # Specific index training mode - insert vector in specified slot
            vectorIndex = self . _nextTrainingIndices . pop ( 0 )
            while vectorIndex >= self . _Memory . shape [ 0 ] :
                self . _doubleMemoryNumRows ( )
            self . _Memory [ vectorIndex ] = inputPattern
            self . _numPatterns = max ( self . _numPatterns , vectorIndex + 1 )
            if vectorIndex >= len ( self . _categoryList ) :
                self . _categoryList += [ - 1 ] * ( vectorIndex - len ( self . _categoryList ) + 1 )
            self . _categoryList [ vectorIndex ] = int ( inputCategory )
        # Set _ M to the " active " part of _ Memory
        self . _M = self . _Memory [ 0 : self . _numPatterns ]
        self . _addPartitionId ( self . _numPatterns - 1 , partitionId )
# Sparse vectors
else : # If the input was given in sparse form , convert it to dense if necessary
    if isSparse > 0 and ( self . _vt is not None or self . distThreshold > 0 or self . numSVDDims is not None or self . numSVDSamples > 0 or self . numWinners > 0 ) :
        denseInput = numpy . zeros ( isSparse )
        denseInput [ inputPattern ] = 1.0
        inputPattern = denseInput
        isSparse = 0
    # Get the input width
    if isSparse > 0 :
        inputWidth = isSparse
    else :
        inputWidth = len ( inputPattern )
    # Allocate storage if this is the first training vector
    if self . _Memory is None :
        self . _Memory = NearestNeighbor ( 0 , inputWidth )
    # Support SVD if it is on
    if self . _vt is not None :
        inputPattern = numpy . dot ( self . _vt , inputPattern - self . _mean )
    # Threshold the input , zeroing out entries that are too close to 0.
    # This is only done if we are given a dense input .
    if isSparse == 0 :
        thresholdedInput = self . _sparsifyVector ( inputPattern , True )
    addRow = True
    # If given the layout of the cells , then turn on the logic that stores
    # only the start cell for bursting columns .
    if self . cellsPerCol >= 1 :
        burstingCols = thresholdedInput . reshape ( - 1 , self . cellsPerCol ) . min ( axis = 1 ) . nonzero ( ) [ 0 ]
        for col in burstingCols :
            thresholdedInput [ ( col * self . cellsPerCol ) + 1 : ( col * self . cellsPerCol ) + self . cellsPerCol ] = 0
    # Don ' t learn entries that are too close to existing entries .
    if self . _Memory . nRows ( ) > 0 :
        dist = None
        # if this vector is a perfect match for one we already learned , then
        # replace the category - it may have changed with online learning on .
        if self . replaceDuplicates :
            dist = self . _calcDistance ( thresholdedInput , distanceNorm = 1 )
            if dist . min ( ) == 0 :
                rowIdx = dist . argmin ( )
                self . _categoryList [ rowIdx ] = int ( inputCategory )
                if self . fixedCapacity :
                    self . _categoryRecencyList [ rowIdx ] = rowID
                addRow = False
        # Don ' t add this vector if it matches closely with another we already
        # added
        if self . distThreshold > 0 :
            if dist is None or self . distanceNorm != 1 :
                dist = self . _calcDistance ( thresholdedInput )
            minDist = dist . min ( )
            addRow = ( minDist >= self . distThreshold )
            if not addRow :
                if self . fixedCapacity :
                    rowIdx = dist . argmin ( )
                    self . _categoryRecencyList [ rowIdx ] = rowID
    # If sparsity is too low , we do not want to add this vector
    if addRow and self . minSparsity > 0.0 :
        if isSparse == 0 :
            sparsity = ( float ( len ( thresholdedInput . nonzero ( ) [ 0 ] ) ) / len ( thresholdedInput ) )
        else :
            sparsity = float ( len ( inputPattern ) ) / isSparse
        if sparsity < self . minSparsity :
            addRow = False
    # Add the new sparse vector to our storage
    if addRow :
        self . _protoSizes = None
        # need to re - compute
        if isSparse == 0 :
            self . _Memory . addRow ( thresholdedInput )
        else :
            self . _Memory . addRowNZ ( inputPattern , [ 1 ] * len ( inputPattern ) )
        self . _numPatterns += 1
        self . _categoryList . append ( int ( inputCategory ) )
        self . _addPartitionId ( self . _numPatterns - 1 , partitionId )
        if self . fixedCapacity :
            self . _categoryRecencyList . append ( rowID )
            if self . _numPatterns > self . maxStoredPatterns and self . maxStoredPatterns > 0 :
                leastRecentlyUsedPattern = numpy . argmin ( self . _categoryRecencyList )
                self . _Memory . deleteRow ( leastRecentlyUsedPattern )
                self . _categoryList . pop ( leastRecentlyUsedPattern )
                self . _categoryRecencyList . pop ( leastRecentlyUsedPattern )
                self . _numPatterns -= 1
if self . numSVDDims is not None and self . numSVDSamples > 0 and self . _numPatterns == self . numSVDSamples :
    self . computeSVD ( )
return self . _numPatterns


signature:
def getOverlaps ( self , inputPattern ) :
    """Return the degree of overlap between an input pattern and each category
 stored in the classifier . The overlap is computed by computing :
 . . code - block : : python
 logical _ and ( inputPattern ! = 0 , trainingPattern ! = 0 ) . sum ( )
 : param inputPattern : pattern to check overlap of
 : returns : ( overlaps , categories ) Two numpy arrays of the same length , where :
 * overlaps : an integer overlap amount for each category
 * categories : category index for each element of overlaps"""

implementation:
assert self . useSparseMemory , "Not implemented yet for dense storage"
overlaps = self . _Memory . rightVecSumAtNZ ( inputPattern )
return ( overlaps , self . _categoryList )


signature:
def getDistances ( self , inputPattern ) :
    """Return the distances between the input pattern and all other
 stored patterns .
 : param inputPattern : pattern to check distance with
 : returns : ( distances , categories ) numpy arrays of the same length .
 - overlaps : an integer overlap amount for each category
 - categories : category index for each element of distances"""

implementation:
dist = self . _getDistances ( inputPattern )
return ( dist , self . _categoryList )


signature:
def infer ( self , inputPattern , computeScores = True , overCategories = True , partitionId = None ) :
    """Finds the category that best matches the input pattern . Returns the
 winning category index as well as a distribution over all categories .
 : param inputPattern : ( list or array ) The pattern to be classified . This
 must be a dense representation of the array ( e . g . [ 0 , 0 , 1 , 1 , 0 , 1 ] ) .
 : param computeScores : NO EFFECT
 : param overCategories : NO EFFECT
 : param partitionId : ( int ) If provided , all training vectors with partitionId
 equal to that of the input pattern are ignored .
 For example , this may be used to perform k - fold cross validation
 without repopulating the classifier . First partition all the data into
 k equal partitions numbered 0 , 1 , 2 , . . . and then call learn ( ) for each
 vector passing in its partitionId . Then , during inference , by passing
 in the partition ID in the call to infer ( ) , all other vectors with the
 same partitionId are ignored simulating the effect of repopulating the
 classifier while ommitting the training vectors in the same partition .
 : returns : 4 - tuple with these keys :
 - ` ` winner ` ` : The category with the greatest number of nearest neighbors
 within the kth nearest neighbors . If the inferenceResult contains no
 neighbors , the value of winner is None . This can happen , for example ,
 in cases of exact matching , if there are no stored vectors , or if
 minSparsity is not met .
 - ` ` inferenceResult ` ` : A list of length numCategories , each entry contains
 the number of neighbors within the top k neighbors that are in that
 category .
 - ` ` dist ` ` : A list of length numPrototypes . Each entry is the distance
 from the unknown to that prototype . All distances are between 0.0 and
 1.0.
 - ` ` categoryDist ` ` : A list of length numCategories . Each entry is the
 distance from the unknown to the nearest prototype of
 that category . All distances are between 0 and 1.0."""

implementation:
# Calculate sparsity . If sparsity is too low , we do not want to run
# inference with this vector
sparsity = 0.0
if self . minSparsity > 0.0 :
    sparsity = ( float ( len ( inputPattern . nonzero ( ) [ 0 ] ) ) / len ( inputPattern ) )
if len ( self . _categoryList ) == 0 or sparsity < self . minSparsity : # No categories learned yet ; i . e . first inference w / online learning or
# insufficient sparsity
    winner = None
    inferenceResult = numpy . zeros ( 1 )
    dist = numpy . ones ( 1 )
    categoryDist = numpy . ones ( 1 )
else :
    maxCategoryIdx = max ( self . _categoryList )
    inferenceResult = numpy . zeros ( maxCategoryIdx + 1 )
    dist = self . _getDistances ( inputPattern , partitionId = partitionId )
    validVectorCount = len ( self . _categoryList ) - self . _categoryList . count ( - 1 )
    # Loop through the indices of the nearest neighbors .
    if self . exact : # Is there an exact match in the distances ?
        exactMatches = numpy . where ( dist < 0.00001 ) [ 0 ]
        if len ( exactMatches ) > 0 :
            for i in exactMatches [ : min ( self . k , validVectorCount ) ] :
                inferenceResult [ self . _categoryList [ i ] ] += 1.0
    else :
        sorted = dist . argsort ( )
        for j in sorted [ : min ( self . k , validVectorCount ) ] :
            inferenceResult [ self . _categoryList [ j ] ] += 1.0
    # Prepare inference results .
    if inferenceResult . any ( ) :
        winner = inferenceResult . argmax ( )
        inferenceResult /= inferenceResult . sum ( )
    else :
        winner = None
    categoryDist = min_score_per_category ( maxCategoryIdx , self . _categoryList , dist )
    categoryDist . clip ( 0 , 1.0 , categoryDist )
if self . verbosity >= 1 :
    print "%s infer:" % ( g_debugPrefix )
    print "  active inputs:" , _labeledInput ( inputPattern , cellsPerCol = self . cellsPerCol )
    print "  winner category:" , winner
    print "  pct neighbors of each category:" , inferenceResult
    print "  dist of each prototype:" , dist
    print "  dist of each category:" , categoryDist
result = ( winner , inferenceResult , dist , categoryDist )
return result


signature:
def getClosest ( self , inputPattern , topKCategories = 3 ) :
    """Returns the index of the pattern that is closest to inputPattern ,
 the distances of all patterns to inputPattern , and the indices of the k
 closest categories ."""

implementation:
inferenceResult = numpy . zeros ( max ( self . _categoryList ) + 1 )
dist = self . _getDistances ( inputPattern )
sorted = dist . argsort ( )
validVectorCount = len ( self . _categoryList ) - self . _categoryList . count ( - 1 )
for j in sorted [ : min ( self . k , validVectorCount ) ] :
    inferenceResult [ self . _categoryList [ j ] ] += 1.0
winner = inferenceResult . argmax ( )
topNCats = [ ]
for i in range ( topKCategories ) :
    topNCats . append ( ( self . _categoryList [ sorted [ i ] ] , dist [ sorted [ i ] ] ) )
return winner , dist , topNCats


signature:
def closestTrainingPattern ( self , inputPattern , cat ) :
    """Returns the closest training pattern to inputPattern that belongs to
 category " cat " .
 : param inputPattern : The pattern whose closest neighbor is sought
 : param cat : The required category of closest neighbor
 : returns : A dense version of the closest training pattern , or None if no
 such patterns exist"""

implementation:
dist = self . _getDistances ( inputPattern )
sorted = dist . argsort ( )
for patIdx in sorted :
    patternCat = self . _categoryList [ patIdx ]
    # If closest pattern belongs to desired category , return it
    if patternCat == cat :
        if self . useSparseMemory :
            closestPattern = self . _Memory . getRow ( int ( patIdx ) )
        else :
            closestPattern = self . _M [ patIdx ]
        return closestPattern
# No patterns were found !
return None


signature:
def getPattern ( self , idx , sparseBinaryForm = False , cat = None ) :
    """Gets a training pattern either by index or category number .
 : param idx : Index of the training pattern
 : param sparseBinaryForm : If true , returns a list of the indices of the
 non - zero bits in the training pattern
 : param cat : If not None , get the first pattern belonging to category cat . If
 this is specified , idx must be None .
 : returns : The training pattern with specified index"""

implementation:
if cat is not None :
    assert idx is None
    idx = self . _categoryList . index ( cat )
if not self . useSparseMemory :
    pattern = self . _Memory [ idx ]
    if sparseBinaryForm :
        pattern = pattern . nonzero ( ) [ 0 ]
else :
    ( nz , values ) = self . _Memory . rowNonZeros ( idx )
    if not sparseBinaryForm :
        pattern = numpy . zeros ( self . _Memory . nCols ( ) )
        numpy . put ( pattern , nz , 1 )
    else :
        pattern = nz
return pattern


signature:
def getPartitionId ( self , i ) :
    """Gets the partition id given an index .
 : param i : index of partition
 : returns : the partition id associated with pattern i . Returns None if no id
 is associated with it ."""

implementation:
if ( i < 0 ) or ( i >= self . _numPatterns ) :
    raise RuntimeError ( "index out of bounds" )
partitionId = self . _partitionIdList [ i ]
if partitionId == numpy . inf :
    return None
else :
    return partitionId


signature:
def _addPartitionId ( self , index , partitionId = None ) :
    """Adds partition id for pattern index"""

implementation:
if partitionId is None :
    self . _partitionIdList . append ( numpy . inf )
else :
    self . _partitionIdList . append ( partitionId )
    indices = self . _partitionIdMap . get ( partitionId , [ ] )
    indices . append ( index )
    self . _partitionIdMap [ partitionId ] = indices


signature:
def _rebuildPartitionIdMap ( self , partitionIdList ) :
    """Rebuilds the partition Id map using the given partitionIdList"""

implementation:
self . _partitionIdMap = { }
for row , partitionId in enumerate ( partitionIdList ) :
    indices = self . _partitionIdMap . get ( partitionId , [ ] )
    indices . append ( row )
    self . _partitionIdMap [ partitionId ] = indices


signature:
def _calcDistance ( self , inputPattern , distanceNorm = None ) :
    """Calculate the distances from inputPattern to all stored patterns . All
 distances are between 0.0 and 1.0
 : param inputPattern The pattern from which distances to all other patterns
 are calculated
 : param distanceNorm Degree of the distance norm"""

implementation:
if distanceNorm is None :
    distanceNorm = self . distanceNorm
# Sparse memory
if self . useSparseMemory :
    if self . _protoSizes is None :
        self . _protoSizes = self . _Memory . rowSums ( )
    overlapsWithProtos = self . _Memory . rightVecSumAtNZ ( inputPattern )
    inputPatternSum = inputPattern . sum ( )
    if self . distanceMethod == "rawOverlap" :
        dist = inputPattern . sum ( ) - overlapsWithProtos
    elif self . distanceMethod == "pctOverlapOfInput" :
        dist = inputPatternSum - overlapsWithProtos
        if inputPatternSum > 0 :
            dist /= inputPatternSum
    elif self . distanceMethod == "pctOverlapOfProto" :
        overlapsWithProtos /= self . _protoSizes
        dist = 1.0 - overlapsWithProtos
    elif self . distanceMethod == "pctOverlapOfLarger" :
        maxVal = numpy . maximum ( self . _protoSizes , inputPatternSum )
        if maxVal . all ( ) > 0 :
            overlapsWithProtos /= maxVal
        dist = 1.0 - overlapsWithProtos
    elif self . distanceMethod == "norm" :
        dist = self . _Memory . vecLpDist ( self . distanceNorm , inputPattern )
        distMax = dist . max ( )
        if distMax > 0 :
            dist /= distMax
    else :
        raise RuntimeError ( "Unimplemented distance method %s" % self . distanceMethod )
# Dense memory
else :
    if self . distanceMethod == "norm" :
        dist = numpy . power ( numpy . abs ( self . _M - inputPattern ) , self . distanceNorm )
        dist = dist . sum ( 1 )
        dist = numpy . power ( dist , 1.0 / self . distanceNorm )
        dist /= dist . max ( )
    else :
        raise RuntimeError ( "Not implemented yet for dense storage...." )
return dist


signature:
def _getDistances ( self , inputPattern , partitionId = None ) :
    """Return the distances from inputPattern to all stored patterns .
 : param inputPattern The pattern from which distances to all other patterns
 are returned
 : param partitionId If provided , ignore all training vectors with this
 partitionId ."""

implementation:
if not self . _finishedLearning :
    self . finishLearning ( )
    self . _finishedLearning = True
if self . _vt is not None and len ( self . _vt ) > 0 :
    inputPattern = numpy . dot ( self . _vt , inputPattern - self . _mean )
sparseInput = self . _sparsifyVector ( inputPattern )
# Compute distances
dist = self . _calcDistance ( sparseInput )
# Invalidate results where category is - 1
if self . _specificIndexTraining :
    dist [ numpy . array ( self . _categoryList ) == - 1 ] = numpy . inf
# Ignore vectors with this partition id by setting their distances to inf
if partitionId is not None :
    dist [ self . _partitionIdMap . get ( partitionId , [ ] ) ] = numpy . inf
return dist


signature:
def computeSVD ( self , numSVDSamples = 0 , finalize = True ) :
    """Compute the singular value decomposition ( SVD ) . The SVD is a factorization
 of a real or complex matrix . It factors the matrix ` a ` as
 ` u * np . diag ( s ) * v ` , where ` u ` and ` v ` are unitary and ` s ` is a 1 - d array
 of ` a ` ' s singular values .
 * * Reason for computing the SVD : * *
 There are cases where you want to feed a lot of vectors to the
 KNNClassifier . However , this can be slow . You can speed up training by ( 1)
 computing the SVD of the input patterns which will give you the
 eigenvectors , ( 2 ) only keeping a fraction of the eigenvectors , and ( 3)
 projecting the input patterns onto the remaining eigenvectors .
 Note that all input patterns are projected onto the eigenvectors in the same
 fashion . Keeping only the highest eigenvectors increases training
 performance since it reduces the dimensionality of the input .
 : param numSVDSamples : ( int ) the number of samples to use for the SVD
 computation .
 : param finalize : ( bool ) whether to apply SVD to the input patterns .
 : returns : ( array ) The singular values for every matrix , sorted in
 descending order ."""

implementation:
if numSVDSamples == 0 :
    numSVDSamples = self . _numPatterns
if not self . useSparseMemory :
    self . _a = self . _Memory [ : self . _numPatterns ]
else :
    self . _a = self . _Memory . toDense ( ) [ : self . _numPatterns ]
self . _mean = numpy . mean ( self . _a , axis = 0 )
self . _a -= self . _mean
u , self . _s , self . _vt = numpy . linalg . svd ( self . _a [ : numSVDSamples ] )
if finalize :
    self . _finalizeSVD ( )
return self . _s


signature:
def getAdaptiveSVDDims ( self , singularValues , fractionOfMax = 0.001 ) :
    """Compute the number of eigenvectors ( singularValues ) to keep .
 : param singularValues :
 : param fractionOfMax :
 : return :"""

implementation:
v = singularValues / singularValues [ 0 ]
idx = numpy . where ( v < fractionOfMax ) [ 0 ]
if len ( idx ) :
    print "Number of PCA dimensions chosen: " , idx [ 0 ] , "out of " , len ( v )
    return idx [ 0 ]
else :
    print "Number of PCA dimensions chosen: " , len ( v ) - 1 , "out of " , len ( v )
    return len ( v ) - 1


signature:
def _finalizeSVD ( self , numSVDDims = None ) :
    """Called by finalizeLearning ( ) . This will project all the patterns onto the
 SVD eigenvectors .
 : param numSVDDims : ( int ) number of egeinvectors used for projection .
 : return :"""

implementation:
if numSVDDims is not None :
    self . numSVDDims = numSVDDims
if self . numSVDDims == "adaptive" :
    if self . fractionOfMax is not None :
        self . numSVDDims = self . getAdaptiveSVDDims ( self . _s , self . fractionOfMax )
    else :
        self . numSVDDims = self . getAdaptiveSVDDims ( self . _s )
if self . _vt . shape [ 0 ] < self . numSVDDims :
    print "******************************************************************"
    print ( "Warning: The requested number of PCA dimensions is more than " "the number of pattern dimensions." )
    print "Setting numSVDDims = " , self . _vt . shape [ 0 ]
    print "******************************************************************"
    self . numSVDDims = self . _vt . shape [ 0 ]
self . _vt = self . _vt [ : self . numSVDDims ]
# Added when svd is not able to decompose vectors - uses raw spare vectors
if len ( self . _vt ) == 0 :
    return
self . _Memory = numpy . zeros ( ( self . _numPatterns , self . numSVDDims ) )
self . _M = self . _Memory
self . useSparseMemory = False
for i in range ( self . _numPatterns ) :
    self . _Memory [ i ] = numpy . dot ( self . _vt , self . _a [ i ] )
self . _a = None


signature:
def remapCategories ( self , mapping ) :
    """Change the category indices .
 Used by the Network Builder to keep the category indices in sync with the
 ImageSensor categoryInfo when the user renames or removes categories .
 : param mapping : List of new category indices . For example , mapping = [ 2,0,1]
 would change all vectors of category 0 to be category 2 , category 1 to
 0 , and category 2 to 1"""

implementation:
categoryArray = numpy . array ( self . _categoryList )
newCategoryArray = numpy . zeros ( categoryArray . shape [ 0 ] )
newCategoryArray . fill ( - 1 )
for i in xrange ( len ( mapping ) ) :
    newCategoryArray [ categoryArray == i ] = mapping [ i ]
self . _categoryList = list ( newCategoryArray )


signature:
def setCategoryOfVectors ( self , vectorIndices , categoryIndices ) :
    """Change the category associated with this vector ( s ) .
 Used by the Network Builder to move vectors between categories , to enable
 categories , and to invalidate vectors by setting the category to - 1.
 : param vectorIndices : Single index or list of indices
 : param categoryIndices : Single index or list of indices . Can also be a
 single index when vectorIndices is a list , in which case the same
 category will be used for all vectors"""

implementation:
if not hasattr ( vectorIndices , "__iter__" ) :
    vectorIndices = [ vectorIndices ]
    categoryIndices = [ categoryIndices ]
elif not hasattr ( categoryIndices , "__iter__" ) :
    categoryIndices = [ categoryIndices ] * len ( vectorIndices )
for i in xrange ( len ( vectorIndices ) ) :
    vectorIndex = vectorIndices [ i ]
    categoryIndex = categoryIndices [ i ]
    # Out - of - bounds is not an error , because the KNN may not have seen the
    # vector yet
    if vectorIndex < len ( self . _categoryList ) :
        self . _categoryList [ vectorIndex ] = categoryIndex


signature:
def getNextRecord ( self ) :
    """Get the next record to encode . Includes getting a record from the
 ` dataSource ` and applying filters . If the filters request more data from the
 ` dataSource ` continue to get data from the ` dataSource ` until all filters
 are satisfied . This method is separate from : meth : ` . RecordSensor . compute ` so that we can
 use a standalone : class : ` . RecordSensor ` to get filtered data ."""

implementation:
allFiltersHaveEnoughData = False
while not allFiltersHaveEnoughData : # Get the data from the dataSource
    data = self . dataSource . getNextRecordDict ( )
    if not data :
        raise StopIteration ( "Datasource has no more data" )
    # temporary check
    if "_reset" not in data :
        data [ "_reset" ] = 0
    if "_sequenceId" not in data :
        data [ "_sequenceId" ] = 0
    if "_category" not in data :
        data [ "_category" ] = [ None ]
    data , allFiltersHaveEnoughData = self . applyFilters ( data )
self . lastRecord = data
return data


signature:
def applyFilters ( self , data ) :
    """Apply pre - encoding filters . These filters may modify or add data . If a
 filter needs another record ( e . g . a delta filter ) it will request another
 record by returning False and the current record will be skipped ( but will
 still be given to all filters ) .
 We have to be very careful about resets . A filter may add a reset ,
 but other filters should not see the added reset , each filter sees
 the original reset value , and we keep track of whether any filter
 adds a reset .
 : param data : ( dict ) The data that will be processed by the filter .
 : returns : ( tuple ) with the data processed by the filter and a boolean to
 know whether or not the filter needs mode data ."""

implementation:
if self . verbosity > 0 :
    print "RecordSensor got data: %s" % data
allFiltersHaveEnoughData = True
if len ( self . preEncodingFilters ) > 0 :
    originalReset = data [ '_reset' ]
    actualReset = originalReset
    for f in self . preEncodingFilters : # if filter needs more data , it returns False
        filterHasEnoughData = f . process ( data )
        allFiltersHaveEnoughData = ( allFiltersHaveEnoughData and filterHasEnoughData )
        actualReset = actualReset or data [ '_reset' ]
        data [ '_reset' ] = originalReset
    data [ '_reset' ] = actualReset
return data , allFiltersHaveEnoughData


signature:
def populateCategoriesOut ( self , categories , output ) :
    """Populate the output array with the category indices .
 . . note : : Non - categories are represented with ` ` - 1 ` ` .
 : param categories : ( list ) of category strings
 : param output : ( list ) category output , will be overwritten"""

implementation:
if categories [ 0 ] is None : # The record has no entry in category field .
    output [ : ] = - 1
else : # Populate category output array by looping over the smaller of the
# output array ( size specified by numCategories ) and the record ' s number
# of categories .
    for i , cat in enumerate ( categories [ : len ( output ) ] ) :
        output [ i ] = cat
    output [ len ( categories ) : ] = - 1


signature:
def compute ( self , inputs , outputs ) :
    """Get a record from the dataSource and encode it .
 Overrides : meth : ` nupic . bindings . regions . PyRegion . PyRegion . compute ` ."""

implementation:
if not self . topDownMode :
    data = self . getNextRecord ( )
    # The private keys in data are standard of RecordStreamIface objects . Any
    # add ' l keys are column headers from the data source .
    reset = data [ "_reset" ]
    sequenceId = data [ "_sequenceId" ]
    categories = data [ "_category" ]
    # Encode the processed records ; populate outputs [ " dataOut " ] in place
    self . encoder . encodeIntoArray ( data , outputs [ "dataOut" ] )
    # If there is a field to predict , set bucketIdxOut and actValueOut .
    # There is a special case where a predicted field might be a vector , as in
    # the CoordinateEncoder . Since this encoder does not provide bucket
    # indices for prediction , we will ignore it .
    if self . predictedField is not None and self . predictedField != "vector" :
        allEncoders = list ( self . encoder . encoders )
        if self . disabledEncoder is not None :
            allEncoders . extend ( self . disabledEncoder . encoders )
        encoders = [ e for e in allEncoders if e [ 0 ] == self . predictedField ]
        if len ( encoders ) == 0 :
            raise ValueError ( "There is no encoder for set for the predicted " "field: %s" % self . predictedField )
        # TODO : Figure out why there are sometimes multiple encoders with the
        # same name .
        # elif len ( encoders ) > 1:
        # raise ValueError ( " There cant ' be more than 1 encoder for the "
        # " predicted field : % s " % self . predictedField )
        else :
            encoder = encoders [ 0 ] [ 1 ]
        actualValue = data [ self . predictedField ]
        outputs [ "bucketIdxOut" ] [ : ] = encoder . getBucketIndices ( actualValue )
        if isinstance ( actualValue , str ) :
            outputs [ "actValueOut" ] [ : ] = encoder . getBucketIndices ( actualValue )
        else :
            outputs [ "actValueOut" ] [ : ] = actualValue
    # Write out the scalar values obtained from they data source .
    outputs [ "sourceOut" ] [ : ] = self . encoder . getScalars ( data )
    self . _outputValues [ "sourceOut" ] = self . encoder . getEncodedValues ( data )
    # Get the encoded bit arrays for each field
    encoders = self . encoder . getEncoderList ( )
    prevOffset = 0
    sourceEncodings = [ ]
    bitData = outputs [ "dataOut" ]
    for encoder in encoders :
        nextOffset = prevOffset + encoder . getWidth ( )
        sourceEncodings . append ( bitData [ prevOffset : nextOffset ] )
        prevOffset = nextOffset
    self . _outputValues [ 'sourceEncodings' ] = sourceEncodings
    # Execute post - encoding filters , if any
    for filter in self . postEncodingFilters :
        filter . process ( encoder = self . encoder , data = outputs [ 'dataOut' ] )
    # Populate the output numpy arrays ; must assign by index .
    outputs [ 'resetOut' ] [ 0 ] = reset
    outputs [ 'sequenceIdOut' ] [ 0 ] = sequenceId
    self . populateCategoriesOut ( categories , outputs [ 'categoryOut' ] )
    # Verbose print ?
    if self . verbosity >= 1 :
        if self . _iterNum == 0 :
            self . encoder . pprintHeader ( prefix = "sensor:" )
        if reset :
            print "RESET - sequenceID:%d" % sequenceId
        if self . verbosity >= 2 :
            print
    # If verbosity > = 2 , print the record fields
    if self . verbosity >= 1 :
        self . encoder . pprint ( outputs [ "dataOut" ] , prefix = "%7d:" % ( self . _iterNum ) )
        scalarValues = self . encoder . getScalars ( data )
        nz = outputs [ "dataOut" ] . nonzero ( ) [ 0 ]
        print "     nz: (%d)" % ( len ( nz ) ) , nz
        print "  encIn:" , self . encoder . scalarsToStr ( scalarValues )
    if self . verbosity >= 2 : # if hasattr ( data , ' header ' ) :
    # header = data . header ( )
    # else :
    # header = ' ' . join ( self . dataSource . names )
    # print " " , header
        print "   data:" , str ( data )
    if self . verbosity >= 3 :
        decoded = self . encoder . decode ( outputs [ "dataOut" ] )
        print "decoded:" , self . encoder . decodedToStr ( decoded )
    self . _iterNum += 1
else : # Spatial
# This is the top down compute in sensor
# We get the spatial pooler ' s topDownOut as spatialTopDownIn
    spatialTopDownIn = inputs [ 'spatialTopDownIn' ]
    spatialTopDownOut = self . encoder . topDownCompute ( spatialTopDownIn )
    # Split topDownOutput into seperate outputs
    values = [ elem . value for elem in spatialTopDownOut ]
    scalars = [ elem . scalar for elem in spatialTopDownOut ]
    encodings = [ elem . encoding for elem in spatialTopDownOut ]
    self . _outputValues [ 'spatialTopDownOut' ] = values
    outputs [ 'spatialTopDownOut' ] [ : ] = numpy . array ( scalars )
    self . _outputValues [ 'spatialTopDownEncodings' ] = encodings
    # Temporal
    # # TODO : Add temporal top - down loop
    # We get the temporal memory ' s topDownOut passed through the spatial
    # pooler as temporalTopDownIn
    temporalTopDownIn = inputs [ 'temporalTopDownIn' ]
    temporalTopDownOut = self . encoder . topDownCompute ( temporalTopDownIn )
    # Split topDownOutput into separate outputs
    values = [ elem . value for elem in temporalTopDownOut ]
    scalars = [ elem . scalar for elem in temporalTopDownOut ]
    encodings = [ elem . encoding for elem in temporalTopDownOut ]
    self . _outputValues [ 'temporalTopDownOut' ] = values
    outputs [ 'temporalTopDownOut' ] [ : ] = numpy . array ( scalars )
    self . _outputValues [ 'temporalTopDownEncodings' ] = encodings
    assert len ( spatialTopDownOut ) == len ( temporalTopDownOut ) , ( "Error: spatialTopDownOut and temporalTopDownOut should be the same " "size" )


signature:
def _convertNonNumericData ( self , spatialOutput , temporalOutput , output ) :
    """Converts all of the non - numeric fields from spatialOutput and temporalOutput
 into their scalar equivalents and records them in the output dictionary .
 : param spatialOutput : The results of topDownCompute ( ) for the spatial input .
 : param temporalOutput : The results of topDownCompute ( ) for the temporal
 input .
 : param output : The main dictionary of outputs passed to compute ( ) . It is
 expected to have keys ' spatialTopDownOut ' and ' temporalTopDownOut ' that
 are mapped to numpy arrays ."""

implementation:
encoders = self . encoder . getEncoderList ( )
types = self . encoder . getDecoderOutputFieldTypes ( )
for i , ( encoder , type ) in enumerate ( zip ( encoders , types ) ) :
    spatialData = spatialOutput [ i ]
    temporalData = temporalOutput [ i ]
    if type != FieldMetaType . integer and type != FieldMetaType . float : # TODO : Make sure that this doesn ' t modify any state
        spatialData = encoder . getScalars ( spatialData ) [ 0 ]
        temporalData = encoder . getScalars ( temporalData ) [ 0 ]
    assert isinstance ( spatialData , ( float , int ) )
    assert isinstance ( temporalData , ( float , int ) )
    output [ 'spatialTopDownOut' ] [ i ] = spatialData
    output [ 'temporalTopDownOut' ] [ i ] = temporalData


signature:
def getOutputElementCount ( self , name ) :
    """Computes the width of dataOut .
 Overrides
 : meth : ` nupic . bindings . regions . PyRegion . PyRegion . getOutputElementCount ` ."""

implementation:
if name == "resetOut" :
    print ( "WARNING: getOutputElementCount should not have been called with " "resetOut" )
    return 1
elif name == "sequenceIdOut" :
    print ( "WARNING: getOutputElementCount should not have been called with " "sequenceIdOut" )
    return 1
elif name == "dataOut" :
    if self . encoder is None :
        raise Exception ( "NuPIC requested output element count for 'dataOut' " "on a RecordSensor node, but the encoder has not " "been set" )
    return self . encoder . getWidth ( )
elif name == "sourceOut" :
    if self . encoder is None :
        raise Exception ( "NuPIC requested output element count for 'sourceOut' " "on a RecordSensor node, " "but the encoder has not been set" )
    return len ( self . encoder . getDescription ( ) )
elif name == "bucketIdxOut" :
    return 1
elif name == "actValueOut" :
    return 1
elif name == "categoryOut" :
    return self . numCategories
elif name == 'spatialTopDownOut' or name == 'temporalTopDownOut' :
    if self . encoder is None :
        raise Exception ( "NuPIC requested output element count for 'sourceOut' " "on a RecordSensor node, " "but the encoder has not been set" )
    return len ( self . encoder . getDescription ( ) )
else :
    raise Exception ( "Unknown output %s" % name )


signature:
def setParameter ( self , parameterName , index , parameterValue ) :
    """Set the value of a Spec parameter . Most parameters are handled
 automatically by PyRegion ' s parameter set mechanism . The ones that need
 special treatment are explicitly handled here ."""

implementation:
if parameterName == 'topDownMode' :
    self . topDownMode = parameterValue
elif parameterName == 'predictedField' :
    self . predictedField = parameterValue
else :
    raise Exception ( 'Unknown parameter: ' + parameterName )


signature:
def writeToProto ( self , proto ) :
    """Overrides : meth : ` nupic . bindings . regions . PyRegion . PyRegion . writeToProto ` ."""

implementation:
self . encoder . write ( proto . encoder )
if self . disabledEncoder is not None :
    self . disabledEncoder . write ( proto . disabledEncoder )
proto . topDownMode = int ( self . topDownMode )
proto . verbosity = self . verbosity
proto . numCategories = self . numCategories


signature:
def readFromProto ( cls , proto ) :
    """Overrides : meth : ` nupic . bindings . regions . PyRegion . PyRegion . readFromProto ` ."""

implementation:
instance = cls ( )
instance . encoder = MultiEncoder . read ( proto . encoder )
if proto . disabledEncoder is not None :
    instance . disabledEncoder = MultiEncoder . read ( proto . disabledEncoder )
instance . topDownMode = bool ( proto . topDownMode )
instance . verbosity = proto . verbosity
instance . numCategories = proto . numCategories
return instance


signature:
def computeAccuracy ( model , size , top ) :
    """Compute prediction accuracy by checking if the next page in the sequence is
 within the top N predictions calculated by the model
 Args :
 model : HTM model
 size : Sample size
 top : top N predictions to use
 Returns : Probability the next page in the sequence is within the top N
 predicted pages"""

implementation:
accuracy = [ ]
# Load MSNBC web data file
filename = os . path . join ( os . path . dirname ( __file__ ) , "msnbc990928.zip" )
with zipfile . ZipFile ( filename ) as archive :
    with archive . open ( "msnbc990928.seq" ) as datafile : # Skip header lines ( first 7 lines )
        for _ in xrange ( 7 ) :
            next ( datafile )
        # Skip learning data and compute accuracy using only new sessions
        for _ in xrange ( LEARNING_RECORDS ) :
            next ( datafile )
        # Compute prediction accuracy by checking if the next page in the sequence
        # is within the top N predictions calculated by the model
        for _ in xrange ( size ) :
            pages = readUserSession ( datafile )
            model . resetSequenceStates ( )
            for i in xrange ( len ( pages ) - 1 ) :
                result = model . run ( { "page" : pages [ i ] } )
                inferences = result . inferences [ "multiStepPredictions" ] [ 1 ]
                # Get top N predictions for the next page
                predicted = sorted ( inferences . items ( ) , key = itemgetter ( 1 ) , reverse = True ) [ : top ]
                # Check if the next page is within the predicted pages
                accuracy . append ( 1 if pages [ i + 1 ] in zip ( * predicted ) [ 0 ] else 0 )
return np . mean ( accuracy )


signature:
def readUserSession ( datafile ) :
    """Reads the user session record from the file ' s cursor position
 Args :
 datafile : Data file whose cursor points at the beginning of the record
 Returns :
 list of pages in the order clicked by the user"""

implementation:
for line in datafile :
    pages = line . split ( )
    total = len ( pages )
    # Select user sessions with 2 or more pages
    if total < 2 :
        continue
    # Exclude outliers by removing extreme long sessions
    if total > 500 :
        continue
    return [ PAGE_CATEGORIES [ int ( i ) - 1 ] for i in pages ]
return [ ]


signature:
def rewind ( self ) :
    """Put us back at the beginning of the file again ."""

implementation:
# Superclass rewind
super ( FileRecordStream , self ) . rewind ( )
self . close ( )
self . _file = open ( self . _filename , self . _mode )
self . _reader = csv . reader ( self . _file , dialect = "excel" )
# Skip header rows
self . _reader . next ( )
self . _reader . next ( )
self . _reader . next ( )
# Reset record count , etc .
self . _recordCount = 0


signature:
def getNextRecord ( self , useCache = True ) :
    """Returns next available data record from the file .
 : returns : a data row ( a list or tuple ) if available ; None , if no more
 records in the table ( End of Stream - EOS ) ; empty sequence ( list
 or tuple ) when timing out while waiting for the next record ."""

implementation:
assert self . _file is not None
assert self . _mode == self . _FILE_READ_MODE
# Read the line
try :
    line = self . _reader . next ( )
except StopIteration :
    if self . rewindAtEOF :
        if self . _recordCount == 0 :
            raise Exception ( "The source configured to reset at EOF but " "'%s' appears to be empty" % self . _filename )
        self . rewind ( )
        line = self . _reader . next ( )
    else :
        return None
# Keep score of how many records were read
self . _recordCount += 1
# Split the line to text fields and convert each text field to a Python
# object if value is missing ( empty string ) encode appropriately for
# upstream consumers in the case of numeric types , this means replacing
# missing data with a sentinel value for string type , we can leave the empty
# string in place
record = [ ]
for i , f in enumerate ( line ) : # print " DEBUG : Evaluating field @ index % s : % r " % ( i , f )
# sys . stdout . flush ( )
    if f in self . _missingValues :
        record . append ( SENTINEL_VALUE_FOR_MISSING_DATA )
    else : # either there is valid data , or the field is string type ,
    # in which case the adapter does the right thing by default
        record . append ( self . _adapters [ i ] ( f ) )
return record


signature:
def appendRecord ( self , record ) :
    """Saves the record in the underlying csv file .
 : param record : a list of Python objects that will be string - ified"""

implementation:
assert self . _file is not None
assert self . _mode == self . _FILE_WRITE_MODE
assert isinstance ( record , ( list , tuple ) ) , "unexpected record type: " + repr ( type ( record ) )
assert len ( record ) == self . _fieldCount , "len(record): %s, fieldCount: %s" % ( len ( record ) , self . _fieldCount )
# Write header if needed
if self . _recordCount == 0 : # Write the header
    names , types , specials = zip ( * self . getFields ( ) )
    for line in names , types , specials :
        self . _writer . writerow ( line )
# Keep track of sequences , make sure time flows forward
self . _updateSequenceInfo ( record )
line = [ self . _adapters [ i ] ( f ) for i , f in enumerate ( record ) ]
self . _writer . writerow ( line )
self . _recordCount += 1


signature:
def appendRecords ( self , records , progressCB = None ) :
    """Saves multiple records in the underlying storage .
 : param records : array of records as in
 : meth : ` ~ . FileRecordStream . appendRecord `
 : param progressCB : ( function ) callback to report progress"""

implementation:
for record in records :
    self . appendRecord ( record )
    if progressCB is not None :
        progressCB ( )


signature:
def getBookmark ( self ) :
    """Gets a bookmark or anchor to the current position .
 : returns : an anchor to the current position in the data . Passing this
 anchor to a constructor makes the current position to be the first
 returned record ."""

implementation:
if self . _write and self . _recordCount == 0 :
    return None
rowDict = dict ( filepath = os . path . realpath ( self . _filename ) , currentRow = self . _recordCount )
return json . dumps ( rowDict )


signature:
def seekFromEnd ( self , numRecords ) :
    """Seeks to ` ` numRecords ` ` from the end and returns a bookmark to the new
 position .
 : param numRecords : how far to seek from end of file .
 : return : bookmark to desired location ."""

implementation:
self . _file . seek ( self . _getTotalLineCount ( ) - numRecords )
return self . getBookmark ( )


signature:
def getStats ( self ) :
    """Parse the file using dedicated reader and collect fields stats . Never
 called if user of : class : ` ~ . FileRecordStream ` does not invoke
 : meth : ` ~ . FileRecordStream . getStats ` method .
 : returns :
 a dictionary of stats . In the current implementation , min and max
 fields are supported . Example of the return dictionary is :
 . . code - block : : python
 ' min ' : [ f1 _ min , f2 _ min , None , None , fn _ min ] ,
 ' max ' : [ f1 _ max , f2 _ max , None , None , fn _ max ]
 ( where fx _ min / fx _ max are set for scalar fields , or None if not )"""

implementation:
# Collect stats only once per File object , use fresh csv iterator
# to keep the next ( ) method returning sequential records no matter when
# caller asks for stats
if self . _stats == None : # Stats are only available when reading csv file
    assert self . _mode == self . _FILE_READ_MODE
    inFile = open ( self . _filename , self . _FILE_READ_MODE )
    # Create a new reader ; read names , types , specials
    reader = csv . reader ( inFile , dialect = "excel" )
    names = [ n . strip ( ) for n in reader . next ( ) ]
    types = [ t . strip ( ) for t in reader . next ( ) ]
    # Skip over specials
    reader . next ( )
    # Initialize stats to all None
    self . _stats = dict ( )
    self . _stats [ 'min' ] = [ ]
    self . _stats [ 'max' ] = [ ]
    for i in xrange ( len ( names ) ) :
        self . _stats [ 'min' ] . append ( None )
        self . _stats [ 'max' ] . append ( None )
    # Read the file , collect stats
    while True :
        try :
            line = reader . next ( )
            for i , f in enumerate ( line ) :
                if ( len ( types ) > i and types [ i ] in [ FieldMetaType . integer , FieldMetaType . float ] and f not in self . _missingValues ) :
                    value = self . _adapters [ i ] ( f )
                    if self . _stats [ 'max' ] [ i ] == None or self . _stats [ 'max' ] [ i ] < value :
                        self . _stats [ 'max' ] [ i ] = value
                    if self . _stats [ 'min' ] [ i ] == None or self . _stats [ 'min' ] [ i ] > value :
                        self . _stats [ 'min' ] [ i ] = value
        except StopIteration :
            break
return self . _stats


signature:
def _updateSequenceInfo ( self , r ) :
    """Keep track of sequence and make sure time goes forward
 Check if the current record is the beginning of a new sequence
 A new sequence starts in 2 cases :
 1 . The sequence id changed ( if there is a sequence id field )
 2 . The reset field is 1 ( if there is a reset field )
 Note that if there is no sequenceId field or resetId field then the entire
 dataset is technically one big sequence . The function will not return True
 for the first record in this case . This is Ok because it is important to
 detect new sequences only when there are multiple sequences in the file ."""

implementation:
# Get current sequence id ( if any )
newSequence = False
sequenceId = ( r [ self . _sequenceIdIdx ] if self . _sequenceIdIdx is not None else None )
if sequenceId != self . _currSequence : # verify that the new sequence didn ' t show up before
    if sequenceId in self . _sequences :
        raise Exception ( 'Broken sequence: %s, record: %s' % ( sequenceId , r ) )
    # add the finished sequence to the set of sequence
    self . _sequences . add ( self . _currSequence )
    self . _currSequence = sequenceId
    # Verify that the reset is consistent ( if there is one )
    if self . _resetIdx :
        assert r [ self . _resetIdx ] == 1
    newSequence = True
else : # Check the reset
    reset = False
    if self . _resetIdx :
        reset = r [ self . _resetIdx ]
        if reset == 1 :
            newSequence = True
# If it ' s still the same old sequence make sure the time flows forward
if not newSequence :
    if self . _timeStampIdx and self . _currTime is not None :
        t = r [ self . _timeStampIdx ]
        if t < self . _currTime :
            raise Exception ( 'No time travel. Early timestamp for record: %s' % r )
if self . _timeStampIdx :
    self . _currTime = r [ self . _timeStampIdx ]


signature:
def _getStartRow ( self , bookmark ) :
    """Extracts start row from the bookmark information"""

implementation:
bookMarkDict = json . loads ( bookmark )
realpath = os . path . realpath ( self . _filename )
bookMarkFile = bookMarkDict . get ( 'filepath' , None )
if bookMarkFile != realpath :
    print ( "Ignoring bookmark due to mismatch between File's " "filename realpath vs. bookmark; realpath: %r; bookmark: %r" ) % ( realpath , bookMarkDict )
    return 0
else :
    return bookMarkDict [ 'currentRow' ]


signature:
def _getTotalLineCount ( self ) :
    """Returns : count of ALL lines in dataset , including header lines"""

implementation:
# Flush the file before we open it again to count lines
if self . _mode == self . _FILE_WRITE_MODE :
    self . _file . flush ( )
return sum ( 1 for line in open ( self . _filename , self . _FILE_READ_MODE ) )


signature:
def getDataRowCount ( self ) :
    """: returns : ( int ) count of data rows in dataset ( excluding header lines )"""

implementation:
numLines = self . _getTotalLineCount ( )
if numLines == 0 : # this may be the case in a file opened for write before the
# header rows are written out
    assert self . _mode == self . _FILE_WRITE_MODE and self . _recordCount == 0
    numDataRows = 0
else :
    numDataRows = numLines - self . _NUM_HEADER_ROWS
assert numDataRows >= 0
return numDataRows


signature:
def runIoThroughNupic ( inputData , model , gymName , plot ) :
    """Handles looping over the input data and passing each row into the given model
 object , as well as extracting the result object and passing it into an output
 handler .
 : param inputData : file path to input data CSV
 : param model : OPF Model object
 : param gymName : Gym name , used for output handler naming
 : param plot : Whether to use matplotlib or not . If false , uses file output ."""

implementation:
inputFile = open ( inputData , "rb" )
csvReader = csv . reader ( inputFile )
# skip header rows
csvReader . next ( )
csvReader . next ( )
csvReader . next ( )
shifter = InferenceShifter ( )
if plot :
    output = nupic_anomaly_output . NuPICPlotOutput ( gymName )
else :
    output = nupic_anomaly_output . NuPICFileOutput ( gymName )
counter = 0
for row in csvReader :
    counter += 1
    if ( counter % 100 == 0 ) :
        print "Read %i lines..." % counter
    timestamp = datetime . datetime . strptime ( row [ 0 ] , DATE_FORMAT )
    consumption = float ( row [ 1 ] )
    result = model . run ( { "timestamp" : timestamp , "kw_energy_consumption" : consumption } )
    if plot :
        result = shifter . shift ( result )
    prediction = result . inferences [ "multiStepBestPredictions" ] [ 1 ]
    anomalyScore = result . inferences [ "anomalyScore" ]
    output . write ( timestamp , consumption , prediction , anomalyScore )
inputFile . close ( )
output . close ( )


signature:
def topDownCompute ( self , encoded ) :
    """[ ScalarEncoder class method override ]"""

implementation:
# Decode to delta scalar
if self . _prevAbsolute == None or self . _prevDelta == None :
    return [ EncoderResult ( value = 0 , scalar = 0 , encoding = numpy . zeros ( self . n ) ) ]
ret = self . _adaptiveScalarEnc . topDownCompute ( encoded )
if self . _prevAbsolute != None :
    ret = [ EncoderResult ( value = ret [ 0 ] . value + self . _prevAbsolute , scalar = ret [ 0 ] . scalar + self . _prevAbsolute , encoding = ret [ 0 ] . encoding ) ]
# ret [ 0 ] . value + = self . _ prevAbsolute
# ret [ 0 ] . scalar + = self . _ prevAbsolute
return ret


signature:
def isTemporal ( inferenceElement ) :
    """Returns True if the inference from this timestep is predicted the input
 for the NEXT timestep .
 NOTE : This should only be checked IF THE MODEL ' S INFERENCE TYPE IS ALSO
 TEMPORAL . That is , a temporal model CAN have non - temporal inference elements ,
 but a non - temporal model CANNOT have temporal inference elements"""

implementation:
if InferenceElement . __temporalInferenceElements is None :
    InferenceElement . __temporalInferenceElements = set ( [ InferenceElement . prediction ] )
return inferenceElement in InferenceElement . __temporalInferenceElements


signature:
def getTemporalDelay ( inferenceElement , key = None ) :
    """Returns the number of records that elapse between when an inference is
 made and when the corresponding input record will appear . For example , a
 multistep prediction for 3 timesteps out will have a delay of 3
 Parameters :
 inferenceElement : The InferenceElement value being delayed
 key : If the inference is a dictionary type , this specifies
 key for the sub - inference that is being delayed"""

implementation:
# For next step prediction , we shift by 1
if inferenceElement in ( InferenceElement . prediction , InferenceElement . encodings ) :
    return 1
# For classification , anomaly scores , the inferences immediately succeed the
# inputs
if inferenceElement in ( InferenceElement . anomalyScore , InferenceElement . anomalyLabel , InferenceElement . classification , InferenceElement . classConfidences ) :
    return 0
# For multistep prediction , the delay is based on the key in the inference
# dictionary
if inferenceElement in ( InferenceElement . multiStepPredictions , InferenceElement . multiStepBestPredictions ) :
    return int ( key )
# default : return 0
return 0


signature:
def getMaxDelay ( inferences ) :
    """Returns the maximum delay for the InferenceElements in the inference
 dictionary
 Parameters :
 inferences : A dictionary where the keys are InferenceElements"""

implementation:
maxDelay = 0
for inferenceElement , inference in inferences . iteritems ( ) :
    if isinstance ( inference , dict ) :
        for key in inference . iterkeys ( ) :
            maxDelay = max ( InferenceElement . getTemporalDelay ( inferenceElement , key ) , maxDelay )
    else :
        maxDelay = max ( InferenceElement . getTemporalDelay ( inferenceElement ) , maxDelay )
return maxDelay


signature:
def isTemporal ( inferenceType ) :
    """Returns True if the inference type is ' temporal ' , i . e . requires a
 temporal memory in the network ."""

implementation:
if InferenceType . __temporalInferenceTypes is None :
    InferenceType . __temporalInferenceTypes = set ( [ InferenceType . TemporalNextStep , InferenceType . TemporalClassification , InferenceType . TemporalAnomaly , InferenceType . TemporalMultiStep , InferenceType . NontemporalMultiStep ] )
return inferenceType in InferenceType . __temporalInferenceTypes


signature:
def makeDirectoryFromAbsolutePath ( absDirPath ) :
    """Makes directory for the given directory path with default permissions .
 If the directory already exists , it is treated as success .
 absDirPath : absolute path of the directory to create .
 Returns : absDirPath arg
 Exceptions : OSError if directory creation fails"""

implementation:
assert os . path . isabs ( absDirPath )
try :
    os . makedirs ( absDirPath )
except OSError , e :
    if e . errno != os . errno . EEXIST :
        raise
return absDirPath


signature:
def _readConfigFile ( cls , filename , path = None ) :
    """Parse the given XML file and return a dict describing the file .
 Parameters :
 filename : name of XML file to parse ( no path )
 path : path of the XML file . If None , then use the standard
 configuration search path .
 retval : returns a dict with each property as a key and a dict of all
 the property ' s attributes as value"""

implementation:
outputProperties = dict ( )
# Get the path to the config files .
if path is None :
    filePath = cls . findConfigFile ( filename )
else :
    filePath = os . path . join ( path , filename )
# Read in the config file
try :
    if filePath is not None :
        try : # Use warn since console log level is set to warning
            _getLoggerBase ( ) . debug ( "Loading config file: %s" , filePath )
            with open ( filePath , 'r' ) as inp :
                contents = inp . read ( )
        except Exception :
            raise RuntimeError ( "Expected configuration file at %s" % filePath )
    else : # If the file was not found in the normal search paths , which includes
    # checking the NTA _ CONF _ PATH , we ' ll try loading it from pkg _ resources .
        try :
            contents = resource_string ( "nupic.support" , filename )
        except Exception as resourceException : # We expect these to be read , and if they don ' t exist we ' ll just use
        # an empty configuration string .
            if filename in [ USER_CONFIG , CUSTOM_CONFIG ] :
                contents = '<configuration/>'
            else :
                raise resourceException
    elements = ElementTree . XML ( contents )
    if elements . tag != 'configuration' :
        raise RuntimeError ( "Expected top-level element to be 'configuration' " "but got '%s'" % ( elements . tag ) )
    # Add in each property found
    propertyElements = elements . findall ( './property' )
    for propertyItem in propertyElements :
        propInfo = dict ( )
        # Parse this property element
        propertyAttributes = list ( propertyItem )
        for propertyAttribute in propertyAttributes :
            propInfo [ propertyAttribute . tag ] = propertyAttribute . text
        # Get the name
        name = propInfo . get ( 'name' , None )
        # value is allowed to be empty string
        if 'value' in propInfo and propInfo [ 'value' ] is None :
            value = ''
        else :
            value = propInfo . get ( 'value' , None )
            if value is None :
                if 'novalue' in propInfo : # Placeholder " novalue " properties are intended to be overridden
                # via dynamic configuration or another configuration layer .
                    continue
                else :
                    raise RuntimeError ( "Missing 'value' element within the property " "element: => %s " % ( str ( propInfo ) ) )
        # The value is allowed to contain substitution tags of the form
        # $ { env . VARNAME } , which should be substituted with the corresponding
        # environment variable values
        restOfValue = value
        value = ''
        while True : # Find the beginning of substitution tag
            pos = restOfValue . find ( '${env.' )
            if pos == - 1 : # No more environment variable substitutions
                value += restOfValue
                break
            # Append prefix to value accumulator
            value += restOfValue [ 0 : pos ]
            # Find the end of current substitution tag
            varTailPos = restOfValue . find ( '}' , pos )
            if varTailPos == - 1 :
                raise RuntimeError ( "Trailing environment variable tag delimiter '}'" " not found in %r" % ( restOfValue ) )
            # Extract environment variable name from tag
            varname = restOfValue [ pos + 6 : varTailPos ]
            if varname not in os . environ :
                raise RuntimeError ( "Attempting to use the value of the environment" " variable %r, which is not defined" % ( varname ) )
            envVarValue = os . environ [ varname ]
            value += envVarValue
            restOfValue = restOfValue [ varTailPos + 1 : ]
        # Check for errors
        if name is None :
            raise RuntimeError ( "Missing 'name' element within following property " "element:\n => %s " % ( str ( propInfo ) ) )
        propInfo [ 'value' ] = value
        outputProperties [ name ] = propInfo
    return outputProperties
except Exception :
    _getLoggerBase ( ) . exception ( "Error while parsing configuration file: %s." , filePath )
    raise


signature:
def setCustomProperties ( cls , properties ) :
    """Set multiple custom properties and persist them to the custom
 configuration store .
 Parameters :
 properties : a dict of property name / value pairs to set"""

implementation:
_getLogger ( ) . info ( "Setting custom configuration properties=%r; caller=%r" , properties , traceback . format_stack ( ) )
_CustomConfigurationFileWrapper . edit ( properties )
for propertyName , value in properties . iteritems ( ) :
    cls . set ( propertyName , value )


signature:
def clear ( cls ) :
    """Clear all configuration properties from in - memory cache , but do NOT
 alter the custom configuration file . Used in unit - testing ."""

implementation:
# Clear the in - memory settings cache , forcing reload upon subsequent " get "
# request .
super ( Configuration , cls ) . clear ( )
# Reset in - memory custom configuration info .
_CustomConfigurationFileWrapper . clear ( persistent = False )


signature:
def resetCustomConfig ( cls ) :
    """Clear all custom configuration settings and delete the persistent
 custom configuration store ."""

implementation:
_getLogger ( ) . info ( "Resetting all custom configuration properties; " "caller=%r" , traceback . format_stack ( ) )
# Clear the in - memory settings cache , forcing reload upon subsequent " get "
# request .
super ( Configuration , cls ) . clear ( )
# Delete the persistent custom configuration store and reset in - memory
# custom configuration info
_CustomConfigurationFileWrapper . clear ( persistent = True )


signature:
def clear ( cls , persistent = False ) :
    """If persistent is True , delete the temporary file
 Parameters :
 persistent : if True , custom configuration file is deleted"""

implementation:
if persistent :
    try :
        os . unlink ( cls . getPath ( ) )
    except OSError , e :
        if e . errno != errno . ENOENT :
            _getLogger ( ) . exception ( "Error %s while trying to remove dynamic " "configuration file: %s" , e . errno , cls . getPath ( ) )
            raise
cls . _path = None


signature:
def getCustomDict ( cls ) :
    """Returns a dict of all temporary values in custom configuration file"""

implementation:
if not os . path . exists ( cls . getPath ( ) ) :
    return dict ( )
properties = Configuration . _readConfigFile ( os . path . basename ( cls . getPath ( ) ) , os . path . dirname ( cls . getPath ( ) ) )
values = dict ( )
for propName in properties :
    if 'value' in properties [ propName ] :
        values [ propName ] = properties [ propName ] [ 'value' ]
return values


signature:
def edit ( cls , properties ) :
    """Edits the XML configuration file with the parameters specified by
 properties
 Parameters :
 properties : dict of settings to be applied to the custom configuration store
 ( key is property name , value is value )"""

implementation:
copyOfProperties = copy ( properties )
configFilePath = cls . getPath ( )
try :
    with open ( configFilePath , 'r' ) as fp :
        contents = fp . read ( )
except IOError , e :
    if e . errno != errno . ENOENT :
        _getLogger ( ) . exception ( "Error %s reading custom configuration store " "from %s, while editing properties %s." , e . errno , configFilePath , properties )
        raise
    contents = '<configuration/>'
try :
    elements = ElementTree . XML ( contents )
    ElementTree . tostring ( elements )
except Exception , e : # Raising error as RuntimeError with custom message since ElementTree
# exceptions aren ' t clear .
    msg = "File contents of custom configuration is corrupt.  File " "location: %s; Contents: '%s'. Original Error (%s): %s." % ( configFilePath , contents , type ( e ) , e )
    _getLogger ( ) . exception ( msg )
    raise RuntimeError ( msg ) , None , sys . exc_info ( ) [ 2 ]
if elements . tag != 'configuration' :
    e = "Expected top-level element to be 'configuration' but got '%s'" % ( elements . tag )
    _getLogger ( ) . error ( e )
    raise RuntimeError ( e )
# Apply new properties to matching settings in the custom config store ;
# pop matching properties from our copy of the properties dict
for propertyItem in elements . findall ( './property' ) :
    propInfo = dict ( ( attr . tag , attr . text ) for attr in propertyItem )
    name = propInfo [ 'name' ]
    if name in copyOfProperties :
        foundValues = propertyItem . findall ( './value' )
        if len ( foundValues ) > 0 :
            foundValues [ 0 ] . text = str ( copyOfProperties . pop ( name ) )
            if not copyOfProperties :
                break
        else :
            e = "Property %s missing value tag." % ( name , )
            _getLogger ( ) . error ( e )
            raise RuntimeError ( e )
# Add unmatched remaining properties to custom config store
for propertyName , value in copyOfProperties . iteritems ( ) :
    newProp = ElementTree . Element ( 'property' )
    nameTag = ElementTree . Element ( 'name' )
    nameTag . text = propertyName
    newProp . append ( nameTag )
    valueTag = ElementTree . Element ( 'value' )
    valueTag . text = str ( value )
    newProp . append ( valueTag )
    elements . append ( newProp )
try :
    makeDirectoryFromAbsolutePath ( os . path . dirname ( configFilePath ) )
    with open ( configFilePath , 'w' ) as fp :
        fp . write ( ElementTree . tostring ( elements ) )
except Exception , e :
    _getLogger ( ) . exception ( "Error while saving custom configuration " "properties %s in %s." , properties , configFilePath )
    raise


signature:
def _setPath ( cls ) :
    """Sets the path of the custom configuration file"""

implementation: cls . _path = os . path . join ( os . environ [ 'NTA_DYNAMIC_CONF_DIR' ] , cls . customFileName )


signature:
def getState ( self ) :
    """Get the particle state as a dict . This is enough information to
 instantiate this particle on another worker ."""

implementation:
varStates = dict ( )
for varName , var in self . permuteVars . iteritems ( ) :
    varStates [ varName ] = var . getState ( )
return dict ( id = self . particleId , genIdx = self . genIdx , swarmId = self . swarmId , varStates = varStates )


signature:
def initStateFrom ( self , particleId , particleState , newBest ) :
    """Init all of our variable positions , velocities , and optionally the best
 result and best position from the given particle .
 If newBest is true , we get the best result and position for this new
 generation from the resultsDB , This is used when evoloving a particle
 because the bestResult and position as stored in was the best AT THE TIME
 THAT PARTICLE STARTED TO RUN and does not include the best since that
 particle completed ."""

implementation:
# Get the update best position and result ?
if newBest :
    ( bestResult , bestPosition ) = self . _resultsDB . getParticleBest ( particleId )
else :
    bestResult = bestPosition = None
# Replace with the position and velocity of each variable from
# saved state
varStates = particleState [ 'varStates' ]
for varName in varStates . keys ( ) :
    varState = copy . deepcopy ( varStates [ varName ] )
    if newBest :
        varState [ 'bestResult' ] = bestResult
    if bestPosition is not None :
        varState [ 'bestPosition' ] = bestPosition [ varName ]
    self . permuteVars [ varName ] . setState ( varState )


signature:
def copyVarStatesFrom ( self , particleState , varNames ) :
    """Copy specific variables from particleState into this particle .
 Parameters :
 particleState : dict produced by a particle ' s getState ( ) method
 varNames : which variables to copy"""

implementation:
# Set this to false if you don ' t want the variable to move anymore
# after we set the state
allowedToMove = True
for varName in particleState [ 'varStates' ] :
    if varName in varNames : # If this particle doesn ' t include this field , don ' t copy it
        if varName not in self . permuteVars :
            continue
        # Set the best position to the copied position
        state = copy . deepcopy ( particleState [ 'varStates' ] [ varName ] )
        state [ '_position' ] = state [ 'position' ]
        state [ 'bestPosition' ] = state [ 'position' ]
        if not allowedToMove :
            state [ 'velocity' ] = 0
        # Set the state now
        self . permuteVars [ varName ] . setState ( state )
        if allowedToMove : # Let the particle move in both directions from the best position
        # it found previously and set it ' s initial velocity to a known
        # fraction of the total distance .
            self . permuteVars [ varName ] . resetVelocity ( self . _rng )


signature:
def getPosition ( self ) :
    """Return the position of this particle . This returns a dict ( ) of key
 value pairs where each key is the name of the flattened permutation
 variable and the value is its chosen value .
 Parameters :
 retval : dict ( ) of flattened permutation choices"""

implementation:
result = dict ( )
for ( varName , value ) in self . permuteVars . iteritems ( ) :
    result [ varName ] = value . getPosition ( )
return result


signature:
def getPositionFromState ( pState ) :
    """Return the position of a particle given its state dict .
 Parameters :
 retval : dict ( ) of particle position , keys are the variable names ,
 values are their positions"""

implementation:
result = dict ( )
for ( varName , value ) in pState [ 'varStates' ] . iteritems ( ) :
    result [ varName ] = value [ 'position' ]
return result


signature:
def agitate ( self ) :
    """Agitate this particle so that it is likely to go to a new position .
 Every time agitate is called , the particle is jiggled an even greater
 amount .
 Parameters :
 retval : None"""

implementation:
for ( varName , var ) in self . permuteVars . iteritems ( ) :
    var . agitate ( )
self . newPosition ( )


signature:
def newPosition ( self , whichVars = None ) : # TODO : incorporate data from choice variables . . . .
# TODO : make sure we ' re calling this when appropriate .
    """Choose a new position based on results obtained so far from all other
 particles .
 Parameters :
 whichVars : If not None , only move these variables
 retval : new position"""

implementation:
# Get the global best position for this swarm generation
globalBestPosition = None
# If speculative particles are enabled , use the global best considering
# even particles in the current generation . This gives better results
# but does not provide repeatable results because it depends on
# worker timing
if self . _hsObj . _speculativeParticles :
    genIdx = self . genIdx
else :
    genIdx = self . genIdx - 1
if genIdx >= 0 :
    ( bestModelId , _ ) = self . _resultsDB . bestModelIdAndErrScore ( self . swarmId , genIdx )
    if bestModelId is not None :
        ( particleState , _ , _ , _ , _ ) = self . _resultsDB . getParticleInfo ( bestModelId )
        globalBestPosition = Particle . getPositionFromState ( particleState )
# Update each variable
for ( varName , var ) in self . permuteVars . iteritems ( ) :
    if whichVars is not None and varName not in whichVars :
        continue
    if globalBestPosition is None :
        var . newPosition ( None , self . _rng )
    else :
        var . newPosition ( globalBestPosition [ varName ] , self . _rng )
# get the new position
position = self . getPosition ( )
# Log the new position
if self . logger . getEffectiveLevel ( ) <= logging . DEBUG :
    msg = StringIO . StringIO ( )
    print >> msg , "New particle position: \n%s" % ( pprint . pformat ( position , indent = 4 ) )
    print >> msg , "Particle variables:"
    for ( varName , var ) in self . permuteVars . iteritems ( ) :
        print >> msg , "  %s: %s" % ( varName , str ( var ) )
    self . logger . debug ( msg . getvalue ( ) )
    msg . close ( )
return position


signature:
def propose ( self , current , r ) :
    """Generates a random sample from the discrete probability distribution and
 returns its value , the log of the probability of sampling that value and the
 log of the probability of sampling the current value ( passed in ) ."""

implementation:
stay = ( r . uniform ( 0 , 1 ) < self . kernel )
if stay :
    logKernel = numpy . log ( self . kernel )
    return current , logKernel , logKernel
else : # Choose uniformly , not according to the pmf .
    curIndex = self . keyMap [ current ]
    ri = r . randint ( 0 , self . nKeys - 1 )
    logKernel = numpy . log ( 1.0 - self . kernel )
    lp = logKernel + self . logp
    if ri < curIndex :
        return self . keys [ ri ] , lp , lp
    else :
        return self . keys [ ri + 1 ] , lp , lp


signature:
def propose ( self , current , r ) :
    """Generates a random sample from the Poisson probability distribution with
 with location and scale parameter equal to the current value ( passed in ) .
 Returns the value of the random sample , the log of the probability of
 sampling that value , and the log of the probability of sampling the current
 value if the roles of the new sample and the current sample were reversed
 ( the log of the backward proposal probability ) ."""

implementation:
curLambda = current + self . offset
x , logProb = PoissonDistribution ( curLambda ) . sample ( r )
logBackward = PoissonDistribution ( x + self . offset ) . logDensity ( current )
return x , logProb , logBackward


signature:
def __getLogger ( cls ) :
    """Get the logger for this object .
 : returns : ( Logger ) A Logger object ."""

implementation:
if cls . __logger is None :
    cls . __logger = opf_utils . initLogger ( cls )
return cls . __logger


signature:
def create ( modelConfig , logLevel = logging . ERROR ) :
    """Create a new model instance , given a description dictionary .
 : param modelConfig : ( dict )
 A dictionary describing the current model ,
 ` described here < . . / . . / quick - start / example - model - params . html > ` _ .
 : param logLevel : ( int ) The level of logging output that should be generated
 : raises Exception : Unsupported model type
 : returns : : class : ` nupic . frameworks . opf . model . Model `"""

implementation:
logger = ModelFactory . __getLogger ( )
logger . setLevel ( logLevel )
logger . debug ( "ModelFactory returning Model from dict: %s" , modelConfig )
modelClass = None
if modelConfig [ 'model' ] == "HTMPrediction" :
    modelClass = HTMPredictionModel
elif modelConfig [ 'model' ] == "TwoGram" :
    modelClass = TwoGramModel
elif modelConfig [ 'model' ] == "PreviousValue" :
    modelClass = PreviousValueModel
else :
    raise Exception ( "ModelFactory received unsupported Model type: %s" % modelConfig [ 'model' ] )
return modelClass ( ** modelConfig [ 'modelParams' ] )


signature:
def loadFromCheckpoint ( savedModelDir , newSerialization = False ) :
    """Load saved model .
 : param savedModelDir : ( string )
 Directory of where the experiment is to be or was saved
 : returns : ( : class : ` nupic . frameworks . opf . model . Model ` ) The loaded model
 instance ."""

implementation:
if newSerialization :
    return HTMPredictionModel . readFromCheckpoint ( savedModelDir )
else :
    return Model . load ( savedModelDir )


signature:
def compute ( self , activeColumns , learn = True ) :
    """Perform one time step of the Temporal Memory algorithm .
 This method calls : meth : ` activateCells ` , then calls
 : meth : ` activateDendrites ` . Using : class : ` TemporalMemory ` via its
 : meth : ` compute ` method ensures that you ' ll always be able to call
 : meth : ` getPredictiveCells ` to get predictions for the next time step .
 : param activeColumns : ( iter ) Indices of active columns .
 : param learn : ( bool ) Whether or not learning is enabled ."""

implementation:
self . activateCells ( sorted ( activeColumns ) , learn )
self . activateDendrites ( learn )


signature:
def activateCells ( self , activeColumns , learn = True ) :
    """Calculate the active cells , using the current active columns and dendrite
 segments . Grow and reinforce synapses .
 : param activeColumns : ( iter ) A sorted list of active column indices .
 : param learn : ( bool ) If true , reinforce / punish / grow synapses .
 * * Pseudocode : * *
 for each column
 if column is active and has active distal dendrite segments
 call activatePredictedColumn
 if column is active and doesn ' t have active distal dendrite segments
 call burstColumn
 if column is inactive and has matching distal dendrite segments
 call punishPredictedColumn"""

implementation:
prevActiveCells = self . activeCells
prevWinnerCells = self . winnerCells
self . activeCells = [ ]
self . winnerCells = [ ]
segToCol = lambda segment : int ( segment . cell / self . cellsPerColumn )
identity = lambda x : x
for columnData in groupby2 ( activeColumns , identity , self . activeSegments , segToCol , self . matchingSegments , segToCol ) :
    ( column , activeColumns , columnActiveSegments , columnMatchingSegments ) = columnData
    if activeColumns is not None :
        if columnActiveSegments is not None :
            cellsToAdd = self . activatePredictedColumn ( column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells , learn )
            self . activeCells += cellsToAdd
            self . winnerCells += cellsToAdd
        else :
            ( cellsToAdd , winnerCell ) = self . burstColumn ( column , columnMatchingSegments , prevActiveCells , prevWinnerCells , learn )
            self . activeCells += cellsToAdd
            self . winnerCells . append ( winnerCell )
    else :
        if learn :
            self . punishPredictedColumn ( column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells )


signature:
def activateDendrites ( self , learn = True ) :
    """Calculate dendrite segment activity , using the current active cells .
 : param learn : ( bool ) If true , segment activations will be recorded . This
 information is used during segment cleanup .
 * * Pseudocode : * *
 for each distal dendrite segment with activity > = activationThreshold
 mark the segment as active
 for each distal dendrite segment with unconnected activity > = minThreshold
 mark the segment as matching"""

implementation:
( numActiveConnected , numActivePotential ) = self . connections . computeActivity ( self . activeCells , self . connectedPermanence )
activeSegments = ( self . connections . segmentForFlatIdx ( i ) for i in xrange ( len ( numActiveConnected ) ) if numActiveConnected [ i ] >= self . activationThreshold )
matchingSegments = ( self . connections . segmentForFlatIdx ( i ) for i in xrange ( len ( numActivePotential ) ) if numActivePotential [ i ] >= self . minThreshold )
self . activeSegments = sorted ( activeSegments , key = self . connections . segmentPositionSortKey )
self . matchingSegments = sorted ( matchingSegments , key = self . connections . segmentPositionSortKey )
self . numActiveConnectedSynapsesForSegment = numActiveConnected
self . numActivePotentialSynapsesForSegment = numActivePotential
if learn :
    for segment in self . activeSegments :
        self . lastUsedIterationForSegment [ segment . flatIdx ] = self . iteration
    self . iteration += 1


signature:
def reset ( self ) :
    """Indicates the start of a new sequence . Clears any predictions and makes sure
 synapses don ' t grow to the currently active cells in the next time step ."""

implementation:
self . activeCells = [ ]
self . winnerCells = [ ]
self . activeSegments = [ ]
self . matchingSegments = [ ]


signature:
def activatePredictedColumn ( self , column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells , learn ) :
    """Determines which cells in a predicted column should be added to winner cells
 list , and learns on the segments that correctly predicted this column .
 : param column : ( int ) Index of bursting column .
 : param columnActiveSegments : ( iter ) Active segments in this column .
 : param columnMatchingSegments : ( iter ) Matching segments in this column .
 : param prevActiveCells : ( list ) Active cells in ` ` t - 1 ` ` .
 : param prevWinnerCells : ( list ) Winner cells in ` ` t - 1 ` ` .
 : param learn : ( bool ) If true , grow and reinforce synapses .
 : returns : ( list ) A list of predicted cells that will be added to
 active cells and winner cells ."""

implementation: return self . _activatePredictedColumn ( self . connections , self . _random , columnActiveSegments , prevActiveCells , prevWinnerCells , self . numActivePotentialSynapsesForSegment , self . maxNewSynapseCount , self . initialPermanence , self . permanenceIncrement , self . permanenceDecrement , self . maxSynapsesPerSegment , learn )


signature:
def punishPredictedColumn ( self , column , columnActiveSegments , columnMatchingSegments , prevActiveCells , prevWinnerCells ) :
    """Punishes the Segments that incorrectly predicted a column to be active .
 : param column : ( int ) Index of bursting column .
 : param columnActiveSegments : ( iter ) Active segments for this column , or None
 if there aren ' t any .
 : param columnMatchingSegments : ( iter ) Matching segments for this column , or
 None if there aren ' t any .
 : param prevActiveCells : ( list ) Active cells in ` ` t - 1 ` ` .
 : param prevWinnerCells : ( list ) Winner cells in ` ` t - 1 ` ` ."""

implementation: self . _punishPredictedColumn ( self . connections , columnMatchingSegments , prevActiveCells , self . predictedSegmentDecrement )


signature:
def createSegment ( self , cell ) :
    """Create a : class : ` ~ nupic . algorithms . connections . Segment ` on the specified
 cell . This method calls
 : meth : ` ~ nupic . algorithms . connections . Connections . createSegment ` on the
 underlying : class : ` ~ nupic . algorithms . connections . Connections ` , and it does
 some extra bookkeeping . Unit tests should call this method , and not
 : meth : ` ~ nupic . algorithms . connections . Connections . createSegment ` .
 : param cell : ( int ) Index of cell to create a segment on .
 : returns : ( : class : ` ~ nupic . algorithms . connections . Segment ` ) The created
 segment ."""

implementation: return self . _createSegment ( self . connections , self . lastUsedIterationForSegment , cell , self . iteration , self . maxSegmentsPerCell )


signature:
def _activatePredictedColumn ( cls , connections , random , columnActiveSegments , prevActiveCells , prevWinnerCells , numActivePotentialSynapsesForSegment , maxNewSynapseCount , initialPermanence , permanenceIncrement , permanenceDecrement , maxSynapsesPerSegment , learn ) :
    """: param connections : ( Object )
 Connections for the TM . Gets mutated .
 : param random : ( Object )
 Random number generator . Gets mutated .
 : param columnActiveSegments : ( iter )
 Active segments in this column .
 : param prevActiveCells : ( list )
 Active cells in ` t - 1 ` .
 : param prevWinnerCells : ( list )
 Winner cells in ` t - 1 ` .
 : param numActivePotentialSynapsesForSegment : ( list )
 Number of active potential synapses per segment , indexed by the segment ' s
 flatIdx .
 : param maxNewSynapseCount : ( int )
 The maximum number of synapses added to a segment during learning
 : param initialPermanence : ( float )
 Initial permanence of a new synapse .
 @ permanenceIncrement ( float )
 Amount by which permanences of synapses are incremented during learning .
 @ permanenceDecrement ( float )
 Amount by which permanences of synapses are decremented during learning .
 : param maxSynapsesPerSegment : ( int )
 The maximum number of synapses per segment .
 : param learn : ( bool )
 If true , grow and reinforce synapses .
 : returns : cellsToAdd ( list )
 A list of predicted cells that will be added to active cells and winner
 cells .
 Pseudocode :
 for each cell in the column that has an active distal dendrite segment
 mark the cell as active
 mark the cell as a winner cell
 ( learning ) for each active distal dendrite segment
 strengthen active synapses
 weaken inactive synapses
 grow synapses to previous winner cells"""

implementation:
cellsToAdd = [ ]
previousCell = None
for segment in columnActiveSegments :
    if segment . cell != previousCell :
        cellsToAdd . append ( segment . cell )
        previousCell = segment . cell
    if learn :
        cls . _adaptSegment ( connections , segment , prevActiveCells , permanenceIncrement , permanenceDecrement )
        active = numActivePotentialSynapsesForSegment [ segment . flatIdx ]
        nGrowDesired = maxNewSynapseCount - active
        if nGrowDesired > 0 :
            cls . _growSynapses ( connections , random , segment , nGrowDesired , prevWinnerCells , initialPermanence , maxSynapsesPerSegment )
return cellsToAdd


signature:
def _burstColumn ( cls , connections , random , lastUsedIterationForSegment , column , columnMatchingSegments , prevActiveCells , prevWinnerCells , cellsForColumn , numActivePotentialSynapsesForSegment , iteration , maxNewSynapseCount , initialPermanence , permanenceIncrement , permanenceDecrement , maxSegmentsPerCell , maxSynapsesPerSegment , learn ) :
    """: param connections : ( Object )
 Connections for the TM . Gets mutated .
 : param random : ( Object )
 Random number generator . Gets mutated .
 : param lastUsedIterationForSegment : ( list )
 Last used iteration for each segment , indexed by the segment ' s flatIdx .
 Gets mutated .
 : param column : ( int )
 Index of bursting column .
 : param columnMatchingSegments : ( iter )
 Matching segments in this column .
 : param prevActiveCells : ( list )
 Active cells in ` t - 1 ` .
 : param prevWinnerCells : ( list )
 Winner cells in ` t - 1 ` .
 : param cellsForColumn : ( sequence )
 Range of cell indices on which to operate .
 : param numActivePotentialSynapsesForSegment : ( list )
 Number of active potential synapses per segment , indexed by the segment ' s
 flatIdx .
 : param iteration : ( int )
 The current timestep .
 : param maxNewSynapseCount : ( int )
 The maximum number of synapses added to a segment during learning .
 : param initialPermanence : ( float )
 Initial permanence of a new synapse .
 : param permanenceIncrement : ( float )
 Amount by which permanences of synapses are incremented during learning .
 : param permanenceDecrement : ( float )
 Amount by which permanences of synapses are decremented during learning .
 : param maxSegmentsPerCell : ( int )
 The maximum number of segments per cell .
 : param maxSynapsesPerSegment : ( int )
 The maximum number of synapses per segment .
 : param learn : ( bool )
 Whether or not learning is enabled .
 : returns : ( tuple ) Contains :
 ` cells ` ( iter ) ,
 ` winnerCell ` ( int ) ,
 Pseudocode :
 mark all cells as active
 if there are any matching distal dendrite segments
 find the most active matching segment
 mark its cell as a winner cell
 ( learning )
 grow and reinforce synapses to previous winner cells
 else
 find the cell with the least segments , mark it as a winner cell
 ( learning )
 ( optimization ) if there are prev winner cells
 add a segment to this winner cell
 grow synapses to previous winner cells"""

implementation:
if columnMatchingSegments is not None :
    numActive = lambda s : numActivePotentialSynapsesForSegment [ s . flatIdx ]
    bestMatchingSegment = max ( columnMatchingSegments , key = numActive )
    winnerCell = bestMatchingSegment . cell
    if learn :
        cls . _adaptSegment ( connections , bestMatchingSegment , prevActiveCells , permanenceIncrement , permanenceDecrement )
        nGrowDesired = maxNewSynapseCount - numActive ( bestMatchingSegment )
        if nGrowDesired > 0 :
            cls . _growSynapses ( connections , random , bestMatchingSegment , nGrowDesired , prevWinnerCells , initialPermanence , maxSynapsesPerSegment )
else :
    winnerCell = cls . _leastUsedCell ( random , cellsForColumn , connections )
    if learn :
        nGrowExact = min ( maxNewSynapseCount , len ( prevWinnerCells ) )
        if nGrowExact > 0 :
            segment = cls . _createSegment ( connections , lastUsedIterationForSegment , winnerCell , iteration , maxSegmentsPerCell )
            cls . _growSynapses ( connections , random , segment , nGrowExact , prevWinnerCells , initialPermanence , maxSynapsesPerSegment )
return cellsForColumn , winnerCell


signature:
def _punishPredictedColumn ( cls , connections , columnMatchingSegments , prevActiveCells , predictedSegmentDecrement ) :
    """: param connections : ( Object )
 Connections for the TM . Gets mutated .
 : param columnMatchingSegments : ( iter )
 Matching segments for this column .
 : param prevActiveCells : ( list )
 Active cells in ` t - 1 ` .
 : param predictedSegmentDecrement : ( float )
 Amount by which segments are punished for incorrect predictions .
 Pseudocode :
 for each matching segment in the column
 weaken active synapses"""

implementation:
if predictedSegmentDecrement > 0.0 and columnMatchingSegments is not None :
    for segment in columnMatchingSegments :
        cls . _adaptSegment ( connections , segment , prevActiveCells , - predictedSegmentDecrement , 0.0 )


signature:
def _createSegment ( cls , connections , lastUsedIterationForSegment , cell , iteration , maxSegmentsPerCell ) :
    """Create a segment on the connections , enforcing the maxSegmentsPerCell
 parameter ."""

implementation:
# Enforce maxSegmentsPerCell .
while connections . numSegments ( cell ) >= maxSegmentsPerCell :
    leastRecentlyUsedSegment = min ( connections . segmentsForCell ( cell ) , key = lambda segment : lastUsedIterationForSegment [ segment . flatIdx ] )
    connections . destroySegment ( leastRecentlyUsedSegment )
# Create the segment .
segment = connections . createSegment ( cell )
# Do TM - specific bookkeeping for the segment .
if segment . flatIdx == len ( lastUsedIterationForSegment ) :
    lastUsedIterationForSegment . append ( iteration )
elif segment . flatIdx < len ( lastUsedIterationForSegment ) : # A flatIdx was recycled .
    lastUsedIterationForSegment [ segment . flatIdx ] = iteration
else :
    raise AssertionError ( "All segments should be created with the TM createSegment method." )
return segment


signature:
def _destroyMinPermanenceSynapses ( cls , connections , random , segment , nDestroy , excludeCells ) :
    """Destroy nDestroy synapses on the specified segment , but don ' t destroy
 synapses to the " excludeCells " ."""

implementation:
destroyCandidates = sorted ( ( synapse for synapse in connections . synapsesForSegment ( segment ) if synapse . presynapticCell not in excludeCells ) , key = lambda s : s . _ordinal )
for _ in xrange ( nDestroy ) :
    if len ( destroyCandidates ) == 0 :
        break
    minSynapse = None
    minPermanence = float ( "inf" )
    for synapse in destroyCandidates :
        if synapse . permanence < minPermanence - EPSILON :
            minSynapse = synapse
            minPermanence = synapse . permanence
    connections . destroySynapse ( minSynapse )
    destroyCandidates . remove ( minSynapse )


signature:
def _leastUsedCell ( cls , random , cells , connections ) :
    """Gets the cell with the smallest number of segments .
 Break ties randomly .
 : param random : ( Object )
 Random number generator . Gets mutated .
 : param cells : ( list )
 Indices of cells .
 : param connections : ( Object )
 Connections instance for the TM .
 : returns : ( int ) Cell index ."""

implementation:
leastUsedCells = [ ]
minNumSegments = float ( "inf" )
for cell in cells :
    numSegments = connections . numSegments ( cell )
    if numSegments < minNumSegments :
        minNumSegments = numSegments
        leastUsedCells = [ ]
    if numSegments == minNumSegments :
        leastUsedCells . append ( cell )
i = random . getUInt32 ( len ( leastUsedCells ) )
return leastUsedCells [ i ]


signature:
def _growSynapses ( cls , connections , random , segment , nDesiredNewSynapes , prevWinnerCells , initialPermanence , maxSynapsesPerSegment ) :
    """Creates nDesiredNewSynapes synapses on the segment passed in if
 possible , choosing random cells from the previous winner cells that are
 not already on the segment .
 : param connections : ( Object ) Connections instance for the tm
 : param random : ( Object ) TM object used to generate random
 numbers
 : param segment : ( int ) Segment to grow synapses on .
 : param nDesiredNewSynapes : ( int ) Desired number of synapses to grow
 : param prevWinnerCells : ( list ) Winner cells in ` t - 1 `
 : param initialPermanence : ( float ) Initial permanence of a new synapse ."""

implementation:
candidates = list ( prevWinnerCells )
for synapse in connections . synapsesForSegment ( segment ) :
    i = binSearch ( candidates , synapse . presynapticCell )
    if i != - 1 :
        del candidates [ i ]
nActual = min ( nDesiredNewSynapes , len ( candidates ) )
# Check if we ' re going to surpass the maximum number of synapses .
overrun = connections . numSynapses ( segment ) + nActual - maxSynapsesPerSegment
if overrun > 0 :
    cls . _destroyMinPermanenceSynapses ( connections , random , segment , overrun , prevWinnerCells )
# Recalculate in case we weren ' t able to destroy as many synapses as needed .
nActual = min ( nActual , maxSynapsesPerSegment - connections . numSynapses ( segment ) )
for _ in range ( nActual ) :
    i = random . getUInt32 ( len ( candidates ) )
    connections . createSynapse ( segment , candidates [ i ] , initialPermanence )
    del candidates [ i ]


signature:
def _adaptSegment ( cls , connections , segment , prevActiveCells , permanenceIncrement , permanenceDecrement ) :
    """Updates synapses on segment .
 Strengthens active synapses ; weakens inactive synapses .
 : param connections : ( Object ) Connections instance for the tm
 : param segment : ( int ) Segment to adapt
 : param prevActiveCells : ( list ) Active cells in ` t - 1 `
 : param permanenceIncrement : ( float ) Amount to increment active synapses
 : param permanenceDecrement : ( float ) Amount to decrement inactive synapses"""

implementation:
# Destroying a synapse modifies the set that we ' re iterating through .
synapsesToDestroy = [ ]
for synapse in connections . synapsesForSegment ( segment ) :
    permanence = synapse . permanence
    if binSearch ( prevActiveCells , synapse . presynapticCell ) != - 1 :
        permanence += permanenceIncrement
    else :
        permanence -= permanenceDecrement
    # Keep permanence within min / max bounds
    permanence = max ( 0.0 , min ( 1.0 , permanence ) )
    if permanence < EPSILON :
        synapsesToDestroy . append ( synapse )
    else :
        connections . updateSynapsePermanence ( synapse , permanence )
for synapse in synapsesToDestroy :
    connections . destroySynapse ( synapse )
if connections . numSynapses ( segment ) == 0 :
    connections . destroySegment ( segment )


signature:
def columnForCell ( self , cell ) :
    """Returns the index of the column that a cell belongs to .
 : param cell : ( int ) Cell index
 : returns : ( int ) Column index"""

implementation:
self . _validateCell ( cell )
return int ( cell / self . cellsPerColumn )


signature:
def cellsForColumn ( self , column ) :
    """Returns the indices of cells that belong to a column .
 : param column : ( int ) Column index
 : returns : ( list ) Cell indices"""

implementation:
self . _validateColumn ( column )
start = self . cellsPerColumn * column
end = start + self . cellsPerColumn
return range ( start , end )


signature:
def mapCellsToColumns ( self , cells ) :
    """Maps cells to the columns they belong to .
 : param cells : ( set ) Cells
 : returns : ( dict ) Mapping from columns to their cells in ` cells `"""

implementation:
cellsForColumns = defaultdict ( set )
for cell in cells :
    column = self . columnForCell ( cell )
    cellsForColumns [ column ] . add ( cell )
return cellsForColumns


signature:
def getPredictiveCells ( self ) :
    """Returns the indices of the predictive cells .
 : returns : ( list ) Indices of predictive cells ."""

implementation:
previousCell = None
predictiveCells = [ ]
for segment in self . activeSegments :
    if segment . cell != previousCell :
        predictiveCells . append ( segment . cell )
        previousCell = segment . cell
return predictiveCells