signature:
def clear_all_placeholder_variables ( printable = True ) :
    """Clears all the placeholder variables of keep prob ,
 including keeping probabilities of all dropout , denoising , dropconnect etc .
 Parameters
 printable : boolean
 If True , print all deleted variables ."""

implementation:
tl . logging . info ( 'clear all .....................................' )
gl = globals ( ) . copy ( )
for var in gl :
    if var [ 0 ] == '_' :
        continue
    if 'func' in str ( globals ( ) [ var ] ) :
        continue
    if 'module' in str ( globals ( ) [ var ] ) :
        continue
    if 'class' in str ( globals ( ) [ var ] ) :
        continue
    if printable :
        tl . logging . info ( " clear_all ------- %s" % str ( globals ( ) [ var ] ) )
    del globals ( ) [ var ]


signature:
def set_gpu_fraction ( gpu_fraction = 0.3 ) :
    """Set the GPU memory fraction for the application .
 Parameters
 gpu _ fraction : float
 Fraction of GPU memory , ( 0 ~ 1]
 References
 - ` TensorFlow using GPU < https : / / www . tensorflow . org / versions / r0.9 / how _ tos / using _ gpu / index . html > ` _ _"""

implementation:
tl . logging . info ( "[TL]: GPU MEM Fraction %f" % gpu_fraction )
gpu_options = tf . GPUOptions ( per_process_gpu_memory_fraction = gpu_fraction )
sess = tf . Session ( config = tf . ConfigProto ( gpu_options = gpu_options ) )
return sess


signature:
def generate_skip_gram_batch ( data , batch_size , num_skips , skip_window , data_index = 0 ) :
    """Generate a training batch for the Skip - Gram model .
 See ` Word2Vec example < https : / / github . com / tensorlayer / tensorlayer / blob / master / example / tutorial _ word2vec _ basic . py > ` _ _ .
 Parameters
 data : list of data
 To present context , usually a list of integers .
 batch _ size : int
 Batch size to return .
 num _ skips : int
 How many times to reuse an input to generate a label .
 skip _ window : int
 How many words to consider left and right .
 data _ index : int
 Index of the context location . This code use ` data _ index ` to instead of yield like ` ` tl . iterate ` ` .
 Returns
 batch : list of data
 Inputs .
 labels : list of data
 Labels
 data _ index : int
 Index of the context location .
 Examples
 Setting num _ skips = 2 , skip _ window = 1 , use the right and left words .
 In the same way , num _ skips = 4 , skip _ window = 2 means use the nearby 4 words .
 > > > data = [ 1,2,3,4,5,6,7,8,9,10,11]
 > > > batch , labels , data _ index = tl . nlp . generate _ skip _ gram _ batch ( data = data , batch _ size = 8 , num _ skips = 2 , skip _ window = 1 , data _ index = 0)
 > > > print ( batch )
 [2 2 3 3 4 4 5 5]
 > > > print ( labels )"""

implementation:
# global data _ index # you can put data _ index outside the function , then
# modify the global data _ index in the function without return it .
# note : without using yield , this code use data _ index to instead .
if batch_size % num_skips != 0 :
    raise Exception ( "batch_size should be able to be divided by num_skips." )
if num_skips > 2 * skip_window :
    raise Exception ( "num_skips <= 2 * skip_window" )
batch = np . ndarray ( shape = ( batch_size ) , dtype = np . int32 )
labels = np . ndarray ( shape = ( batch_size , 1 ) , dtype = np . int32 )
span = 2 * skip_window + 1
# [ skip _ window target skip _ window ]
buffer = collections . deque ( maxlen = span )
for _ in range ( span ) :
    buffer . append ( data [ data_index ] )
    data_index = ( data_index + 1 ) % len ( data )
for i in range ( batch_size // num_skips ) :
    target = skip_window
    # target label at the center of the buffer
    targets_to_avoid = [ skip_window ]
    for j in range ( num_skips ) :
        while target in targets_to_avoid :
            target = random . randint ( 0 , span - 1 )
        targets_to_avoid . append ( target )
        batch [ i * num_skips + j ] = buffer [ skip_window ]
        labels [ i * num_skips + j , 0 ] = buffer [ target ]
    buffer . append ( data [ data_index ] )
    data_index = ( data_index + 1 ) % len ( data )
return batch , labels , data_index


signature:
def sample ( a = None , temperature = 1.0 ) :
    """Sample an index from a probability array .
 Parameters
 a : list of float
 List of probabilities .
 temperature : float or None
 The higher the more uniform . When a = [ 0.1 , 0.2 , 0.7 ] ,
 - temperature = 0.7 , the distribution will be sharpen [ 0.05048273 , 0.13588945 , 0.81362782]
 - temperature = 1.0 , the distribution will be the same [ 0.1 , 0.2 , 0.7]
 - temperature = 1.5 , the distribution will be filtered [ 0.16008435 , 0.25411807 , 0.58579758]
 - If None , it will be ` ` np . argmax ( a ) ` `
 Notes
 - No matter what is the temperature and input list , the sum of all probabilities will be one . Even if input list = [ 1 , 100 , 200 ] , the sum of all probabilities will still be one .
 - For large vocabulary size , choice a higher temperature or ` ` tl . nlp . sample _ top ` ` to avoid error ."""

implementation:
if a is None :
    raise Exception ( "a : list of float" )
b = np . copy ( a )
try :
    if temperature == 1 :
        return np . argmax ( np . random . multinomial ( 1 , a , 1 ) )
    if temperature is None :
        return np . argmax ( a )
    else :
        a = np . log ( a ) / temperature
        a = np . exp ( a ) / np . sum ( np . exp ( a ) )
        return np . argmax ( np . random . multinomial ( 1 , a , 1 ) )
except Exception : # np . set _ printoptions ( threshold = np . nan )
# tl . logging . info ( a )
# tl . logging . info ( np . sum ( a ) )
# tl . logging . info ( np . max ( a ) )
# tl . logging . info ( np . min ( a ) )
# exit ( )
    message = "For large vocabulary_size, choice a higher temperature\
         to avoid log error. Hint : use ``sample_top``. "
    warnings . warn ( message , Warning )
    # tl . logging . info ( a )
    # tl . logging . info ( b )
    return np . argmax ( np . random . multinomial ( 1 , b , 1 ) )


signature:
def sample_top ( a = None , top_k = 10 ) :
    """Sample from ` ` top _ k ` ` probabilities .
 Parameters
 a : list of float
 List of probabilities .
 top _ k : int
 Number of candidates to be considered ."""

implementation:
if a is None :
    a = [ ]
idx = np . argpartition ( a , - top_k ) [ - top_k : ]
probs = a [ idx ]
# tl . logging . info ( " new % f " % probs )
probs = probs / np . sum ( probs )
choice = np . random . choice ( idx , p = probs )
return choice


signature:
def process_sentence ( sentence , start_word = "<S>" , end_word = "</S>" ) :
    """Seperate a sentence string into a list of string words , add start _ word and end _ word ,
 see ` ` create _ vocab ( ) ` ` and ` ` tutorial _ tfrecord3 . py ` ` .
 Parameters
 sentence : str
 A sentence .
 start _ word : str or None
 The start word . If None , no start word will be appended .
 end _ word : str or None
 The end word . If None , no end word will be appended .
 Returns
 list of str
 A list of strings that separated into words .
 Examples
 > > > c = " how are you ? "
 > > > c = tl . nlp . process _ sentence ( c )
 > > > print ( c )
 [ ' < S > ' , ' how ' , ' are ' , ' you ' , ' ? ' , ' < / S > ' ]
 Notes
 - You have to install the following package .
 - ` Installing NLTK < http : / / www . nltk . org / install . html > ` _ _
 - ` Installing NLTK data < http : / / www . nltk . org / data . html > ` _ _"""

implementation:
if start_word is not None :
    process_sentence = [ start_word ]
else :
    process_sentence = [ ]
process_sentence . extend ( nltk . tokenize . word_tokenize ( sentence . lower ( ) ) )
if end_word is not None :
    process_sentence . append ( end_word )
return process_sentence


signature:
def create_vocab ( sentences , word_counts_output_file , min_word_count = 1 ) :
    """Creates the vocabulary of word to word _ id .
 See ` ` tutorial _ tfrecord3 . py ` ` .
 The vocabulary is saved to disk in a text file of word counts . The id of each
 word in the file is its corresponding 0 - based line number .
 Parameters
 sentences : list of list of str
 All sentences for creating the vocabulary .
 word _ counts _ output _ file : str
 The file name .
 min _ word _ count : int
 Minimum number of occurrences for a word .
 Returns
 : class : ` SimpleVocabulary `
 The simple vocabulary object , see : class : ` Vocabulary ` for more .
 Examples
 Pre - process sentences
 > > > captions = [ " one two , three " , " four five five " ]
 > > > processed _ capts = [ ]
 > > > for c in captions :
 > > > c = tl . nlp . process _ sentence ( c , start _ word = " < S > " , end _ word = " < / S > " )
 > > > processed _ capts . append ( c )
 > > > print ( processed _ capts )
 . . . [ [ ' < S > ' , ' one ' , ' two ' , ' , ' , ' three ' , ' < / S > ' ] , [ ' < S > ' , ' four ' , ' five ' , ' five ' , ' < / S > ' ] ]
 Create vocabulary
 > > > tl . nlp . create _ vocab ( processed _ capts , word _ counts _ output _ file = ' vocab . txt ' , min _ word _ count = 1)
 Creating vocabulary .
 Total words : 8
 Words in vocabulary : 8
 Wrote vocabulary file : vocab . txt
 Get vocabulary object
 > > > vocab = tl . nlp . Vocabulary ( ' vocab . txt ' , start _ word = " < S > " , end _ word = " < / S > " , unk _ word = " < UNK > " )
 INFO : tensorflow : Initializing vocabulary from file : vocab . txt
 [ TL ] Vocabulary from vocab . txt : < S > < / S > < UNK >
 vocabulary with 10 words ( includes start _ word , end _ word , unk _ word )
 start _ id : 2
 end _ id : 3
 unk _ id : 9
 pad _ id : 0"""

implementation:
tl . logging . info ( "Creating vocabulary." )
counter = Counter ( )
for c in sentences :
    counter . update ( c )
    # tl . logging . info ( ' c ' , c )
tl . logging . info ( "    Total words: %d" % len ( counter ) )
# Filter uncommon words and sort by descending count .
word_counts = [ x for x in counter . items ( ) if x [ 1 ] >= min_word_count ]
word_counts . sort ( key = lambda x : x [ 1 ] , reverse = True )
word_counts = [ ( "<PAD>" , 0 ) ] + word_counts
# 1st id should be reserved for padding
# tl . logging . info ( word _ counts )
tl . logging . info ( "    Words in vocabulary: %d" % len ( word_counts ) )
# Write out the word counts file .
with tf . gfile . FastGFile ( word_counts_output_file , "w" ) as f :
    f . write ( "\n" . join ( [ "%s %d" % ( w , c ) for w , c in word_counts ] ) )
tl . logging . info ( "    Wrote vocabulary file: %s" % word_counts_output_file )
# Create the vocabulary dictionary .
reverse_vocab = [ x [ 0 ] for x in word_counts ]
unk_id = len ( reverse_vocab )
vocab_dict = dict ( [ ( x , y ) for ( y , x ) in enumerate ( reverse_vocab ) ] )
vocab = SimpleVocabulary ( vocab_dict , unk_id )
return vocab


signature:
def read_words ( filename = "nietzsche.txt" , replace = None ) :
    """Read list format context from a file .
 For customized read _ words method , see ` ` tutorial _ generate _ text . py ` ` .
 Parameters
 filename : str
 a file path .
 replace : list of str
 replace original string by target string .
 Returns
 list of str
 The context in a list ( split using space ) ."""

implementation:
if replace is None :
    replace = [ '\n' , '<eos>' ]
with tf . gfile . GFile ( filename , "r" ) as f :
    try : # python 3.4 or older
        context_list = f . read ( ) . replace ( * replace ) . split ( )
    except Exception : # python 3.5
        f . seek ( 0 )
        replace = [ x . encode ( 'utf-8' ) for x in replace ]
        context_list = f . read ( ) . replace ( * replace ) . split ( )
    return context_list


signature:
def read_analogies_file ( eval_file = 'questions-words.txt' , word2id = None ) :
    """Reads through an analogy question file , return its id format .
 Parameters
 eval _ file : str
 The file name .
 word2id : dictionary
 a dictionary that maps word to ID .
 Returns
 numpy . array
 A ` ` [ n _ examples , 4 ] ` ` numpy array containing the analogy question ' s word IDs .
 Examples
 The file should be in this format
 > > > : capital - common - countries
 > > > Athens Greece Baghdad Iraq
 > > > Athens Greece Bangkok Thailand
 > > > Athens Greece Beijing China
 > > > Athens Greece Berlin Germany
 > > > Athens Greece Bern Switzerland
 > > > Athens Greece Cairo Egypt
 > > > Athens Greece Canberra Australia
 > > > Athens Greece Hanoi Vietnam
 > > > Athens Greece Havana Cuba
 Get the tokenized analogy question data
 > > > words = tl . files . load _ matt _ mahoney _ text8 _ dataset ( )
 > > > data , count , dictionary , reverse _ dictionary = tl . nlp . build _ words _ dataset ( words , vocabulary _ size , True )
 > > > analogy _ questions = tl . nlp . read _ analogies _ file ( eval _ file = ' questions - words . txt ' , word2id = dictionary )
 > > > print ( analogy _ questions )
 [ [ 3068 1248 7161 1581]
 [ 3068 1248 28683 5642]
 [ 3068 1248 3878 486]
 [ 1216 4309 19982 25506]
 [ 1216 4309 3194 8650]
 [ 1216 4309 140 312 ] ]"""

implementation:
if word2id is None :
    word2id = { }
questions = [ ]
questions_skipped = 0
with open ( eval_file , "rb" ) as analogy_f :
    for line in analogy_f :
        if line . startswith ( b":" ) : # Skip comments .
            continue
        words = line . strip ( ) . lower ( ) . split ( b" " )
        # lowercase
        ids = [ word2id . get ( w . strip ( ) ) for w in words ]
        if None in ids or len ( ids ) != 4 :
            questions_skipped += 1
        else :
            questions . append ( np . array ( ids ) )
tl . logging . info ( "Eval analogy file: %s" % eval_file )
tl . logging . info ( "Questions: %d" , len ( questions ) )
tl . logging . info ( "Skipped: %d" , questions_skipped )
analogy_questions = np . array ( questions , dtype = np . int32 )
return analogy_questions


signature:
def build_reverse_dictionary ( word_to_id ) :
    """Given a dictionary that maps word to integer id .
 Returns a reverse dictionary that maps a id to word .
 Parameters
 word _ to _ id : dictionary
 that maps word to ID .
 Returns
 dictionary
 A dictionary that maps IDs to words ."""

implementation:
reverse_dictionary = dict ( zip ( word_to_id . values ( ) , word_to_id . keys ( ) ) )
return reverse_dictionary


signature:
def build_words_dataset ( words = None , vocabulary_size = 50000 , printable = True , unk_key = 'UNK' ) :
    """Build the words dictionary and replace rare words with ' UNK ' token .
 The most common word has the smallest integer id .
 Parameters
 words : list of str or byte
 The context in list format . You may need to do preprocessing on the words , such as lower case , remove marks etc .
 vocabulary _ size : int
 The maximum vocabulary size , limiting the vocabulary size . Then the script replaces rare words with ' UNK ' token .
 printable : boolean
 Whether to print the read vocabulary size of the given words .
 unk _ key : str
 Represent the unknown words .
 Returns
 data : list of int
 The context in a list of ID .
 count : list of tuple and list
 Pair words and IDs .
 - count [ 0 ] is a list : the number of rare words
 - count [ 1 : ] are tuples : the number of occurrence of each word
 - e . g . [ [ ' UNK ' , 418391 ] , ( b ' the ' , 1061396 ) , ( b ' of ' , 593677 ) , ( b ' and ' , 416629 ) , ( b ' one ' , 411764 ) ]
 dictionary : dictionary
 It is ` word _ to _ id ` that maps word to ID .
 reverse _ dictionary : a dictionary
 It is ` id _ to _ word ` that maps ID to word .
 Examples
 > > > words = tl . files . load _ matt _ mahoney _ text8 _ dataset ( )
 > > > vocabulary _ size = 50000
 > > > data , count , dictionary , reverse _ dictionary = tl . nlp . build _ words _ dataset ( words , vocabulary _ size )
 References
 - ` tensorflow / examples / tutorials / word2vec / word2vec _ basic . py < https : / / github . com / tensorflow / tensorflow / blob / r0.7 / tensorflow / examples / tutorials / word2vec / word2vec _ basic . py > ` _ _"""

implementation:
if words is None :
    raise Exception ( "words : list of str or byte" )
count = [ [ unk_key , - 1 ] ]
count . extend ( collections . Counter ( words ) . most_common ( vocabulary_size - 1 ) )
dictionary = dict ( )
for word , _ in count :
    dictionary [ word ] = len ( dictionary )
data = list ( )
unk_count = 0
for word in words :
    if word in dictionary :
        index = dictionary [ word ]
    else :
        index = 0
        # dictionary [ ' UNK ' ]
        unk_count += 1
    data . append ( index )
count [ 0 ] [ 1 ] = unk_count
reverse_dictionary = dict ( zip ( dictionary . values ( ) , dictionary . keys ( ) ) )
if printable :
    tl . logging . info ( 'Real vocabulary size    %d' % len ( collections . Counter ( words ) . keys ( ) ) )
    tl . logging . info ( 'Limited vocabulary size {}' . format ( vocabulary_size ) )
if len ( collections . Counter ( words ) . keys ( ) ) < vocabulary_size :
    raise Exception ( "len(collections.Counter(words).keys()) >= vocabulary_size , the limited vocabulary_size must be less than or equal to the read vocabulary_size" )
return data , count , dictionary , reverse_dictionary


signature:
def words_to_word_ids ( data = None , word_to_id = None , unk_key = 'UNK' ) :
    """Convert a list of string ( words ) to IDs .
 Parameters
 data : list of string or byte
 The context in list format
 word _ to _ id : a dictionary
 that maps word to ID .
 unk _ key : str
 Represent the unknown words .
 Returns
 list of int
 A list of IDs to represent the context .
 Examples
 > > > words = tl . files . load _ matt _ mahoney _ text8 _ dataset ( )
 > > > vocabulary _ size = 50000
 > > > data , count , dictionary , reverse _ dictionary = tl . nlp . build _ words _ dataset ( words , vocabulary _ size , True )
 > > > context = [ b ' hello ' , b ' how ' , b ' are ' , b ' you ' ]
 > > > ids = tl . nlp . words _ to _ word _ ids ( words , dictionary )
 > > > context = tl . nlp . word _ ids _ to _ words ( ids , reverse _ dictionary )
 > > > print ( ids )
 [6434 , 311 , 26 , 207]
 > > > print ( context )
 [ b ' hello ' , b ' how ' , b ' are ' , b ' you ' ]
 References
 - ` tensorflow . models . rnn . ptb . reader < https : / / github . com / tensorflow / tensorflow / tree / master / tensorflow / models / rnn / ptb > ` _ _"""

implementation:
if data is None :
    raise Exception ( "data : list of string or byte" )
if word_to_id is None :
    raise Exception ( "word_to_id : a dictionary" )
# if isinstance ( data [ 0 ] , six . string _ types ) :
# tl . logging . info ( type ( data [ 0 ] ) )
# # exit ( )
# tl . logging . info ( data [ 0 ] )
# tl . logging . info ( word _ to _ id )
# return [ word _ to _ id [ str ( word ) ] for word in data ]
# else :
word_ids = [ ]
for word in data :
    if word_to_id . get ( word ) is not None :
        word_ids . append ( word_to_id [ word ] )
    else :
        word_ids . append ( word_to_id [ unk_key ] )
return word_ids


signature:
def save_vocab ( count = None , name = 'vocab.txt' ) :
    """Save the vocabulary to a file so the model can be reloaded .
 Parameters
 count : a list of tuple and list
 count [ 0 ] is a list : the number of rare words ,
 count [ 1 : ] are tuples : the number of occurrence of each word ,
 e . g . [ [ ' UNK ' , 418391 ] , ( b ' the ' , 1061396 ) , ( b ' of ' , 593677 ) , ( b ' and ' , 416629 ) , ( b ' one ' , 411764 ) ]
 Examples
 > > > words = tl . files . load _ matt _ mahoney _ text8 _ dataset ( )
 > > > vocabulary _ size = 50000
 > > > data , count , dictionary , reverse _ dictionary = tl . nlp . build _ words _ dataset ( words , vocabulary _ size , True )
 > > > tl . nlp . save _ vocab ( count , name = ' vocab _ text8 . txt ' )
 > > > vocab _ text8 . txt
 UNK 418391
 the 1061396
 of 593677
 and 416629
 one 411764
 in 372201
 a 325873
 to 316376"""

implementation:
if count is None :
    count = [ ]
pwd = os . getcwd ( )
vocabulary_size = len ( count )
with open ( os . path . join ( pwd , name ) , "w" ) as f :
    for i in xrange ( vocabulary_size ) :
        f . write ( "%s %d\n" % ( tf . compat . as_text ( count [ i ] [ 0 ] ) , count [ i ] [ 1 ] ) )
tl . logging . info ( "%d vocab saved to %s in %s" % ( vocabulary_size , name , pwd ) )


signature:
def basic_tokenizer ( sentence , _WORD_SPLIT = re . compile ( b"([.,!?\"':;)(])" ) ) :
    """Very basic tokenizer : split the sentence into a list of tokens .
 Parameters
 sentence : tensorflow . python . platform . gfile . GFile Object
 _ WORD _ SPLIT : regular expression for word spliting .
 Examples
 > > > see create _ vocabulary
 > > > from tensorflow . python . platform import gfile
 > > > train _ path = " wmt / giga - fren . release2"
 > > > with gfile . GFile ( train _ path + " . en " , mode = " rb " ) as f :
 > > > for line in f :
 > > > tokens = tl . nlp . basic _ tokenizer ( line )
 > > > tl . logging . info ( tokens )
 > > > exit ( )
 [ b ' Changing ' , b ' Lives ' , b ' | ' , b ' Changing ' , b ' Society ' , b ' | ' , b ' How ' ,
 b ' It ' , b ' Works ' , b ' | ' , b ' Technology ' , b ' Drives ' , b ' Change ' , b ' Home ' ,
 b ' | ' , b ' Concepts ' , b ' | ' , b ' Teachers ' , b ' | ' , b ' Search ' , b ' | ' , b ' Overview ' ,
 b ' | ' , b ' Credits ' , b ' | ' , b ' HHCC ' , b ' Web ' , b ' | ' , b ' Reference ' , b ' | ' ,
 b ' Feedback ' , b ' Virtual ' , b ' Museum ' , b ' of ' , b ' Canada ' , b ' Home ' , b ' Page ' ]
 References
 - Code from ` ` / tensorflow / models / rnn / translation / data _ utils . py ` `"""

implementation:
words = [ ]
sentence = tf . compat . as_bytes ( sentence )
for space_separated_fragment in sentence . strip ( ) . split ( ) :
    words . extend ( re . split ( _WORD_SPLIT , space_separated_fragment ) )
return [ w for w in words if w ]


signature:
def create_vocabulary ( vocabulary_path , data_path , max_vocabulary_size , tokenizer = None , normalize_digits = True , _DIGIT_RE = re . compile ( br"\d" ) , _START_VOCAB = None ) :
    r"""Create vocabulary file ( if it does not exist yet ) from data file .
 Data file is assumed to contain one sentence per line . Each sentence is
 tokenized and digits are normalized ( if normalize _ digits is set ) .
 Vocabulary contains the most - frequent tokens up to max _ vocabulary _ size .
 We write it to vocabulary _ path in a one - token - per - line format , so that later
 token in the first line gets id = 0 , second line gets id = 1 , and so on .
 Parameters
 vocabulary _ path : str
 Path where the vocabulary will be created .
 data _ path : str
 Data file that will be used to create vocabulary .
 max _ vocabulary _ size : int
 Limit on the size of the created vocabulary .
 tokenizer : function
 A function to use to tokenize each data sentence . If None , basic _ tokenizer will be used .
 normalize _ digits : boolean
 If true , all digits are replaced by ` 0 ` .
 _ DIGIT _ RE : regular expression function
 Default is ` ` re . compile ( br " \ d " ) ` ` .
 _ START _ VOCAB : list of str
 The pad , go , eos and unk token , default is ` ` [ b " _ PAD " , b " _ GO " , b " _ EOS " , b " _ UNK " ] ` ` .
 References
 - Code from ` ` / tensorflow / models / rnn / translation / data _ utils . py ` `"""

implementation:
if _START_VOCAB is None :
    _START_VOCAB = [ b"_PAD" , b"_GO" , b"_EOS" , b"_UNK" ]
if not gfile . Exists ( vocabulary_path ) :
    tl . logging . info ( "Creating vocabulary %s from data %s" % ( vocabulary_path , data_path ) )
    vocab = { }
    with gfile . GFile ( data_path , mode = "rb" ) as f :
        counter = 0
        for line in f :
            counter += 1
            if counter % 100000 == 0 :
                tl . logging . info ( "  processing line %d" % counter )
            tokens = tokenizer ( line ) if tokenizer else basic_tokenizer ( line )
            for w in tokens :
                word = re . sub ( _DIGIT_RE , b"0" , w ) if normalize_digits else w
                if word in vocab :
                    vocab [ word ] += 1
                else :
                    vocab [ word ] = 1
        vocab_list = _START_VOCAB + sorted ( vocab , key = vocab . get , reverse = True )
        if len ( vocab_list ) > max_vocabulary_size :
            vocab_list = vocab_list [ : max_vocabulary_size ]
        with gfile . GFile ( vocabulary_path , mode = "wb" ) as vocab_file :
            for w in vocab_list :
                vocab_file . write ( w + b"\n" )
else :
    tl . logging . info ( "Vocabulary %s from data %s exists" % ( vocabulary_path , data_path ) )


signature:
def initialize_vocabulary ( vocabulary_path ) :
    """Initialize vocabulary from file , return the ` word _ to _ id ` ( dictionary )
 and ` id _ to _ word ` ( list ) .
 We assume the vocabulary is stored one - item - per - line , so a file will result in a vocabulary { " dog " : 0 , " cat " : 1 } , and this function will also return the reversed - vocabulary [ " dog " , " cat " ] .
 Parameters
 vocabulary _ path : str
 Path to the file containing the vocabulary .
 Returns
 vocab : dictionary
 a dictionary that maps word to ID .
 rev _ vocab : list of int
 a list that maps ID to word .
 Examples
 > > > Assume ' test ' contains
 dog
 cat
 bird
 > > > vocab , rev _ vocab = tl . nlp . initialize _ vocabulary ( " test " )
 > > > print ( vocab )
 > > > { b ' cat ' : 1 , b ' dog ' : 0 , b ' bird ' : 2}
 > > > print ( rev _ vocab )
 > > > [ b ' dog ' , b ' cat ' , b ' bird ' ]
 Raises
 ValueError : if the provided vocabulary _ path does not exist ."""

implementation:
if gfile . Exists ( vocabulary_path ) :
    rev_vocab = [ ]
    with gfile . GFile ( vocabulary_path , mode = "rb" ) as f :
        rev_vocab . extend ( f . readlines ( ) )
    rev_vocab = [ tf . compat . as_bytes ( line . strip ( ) ) for line in rev_vocab ]
    vocab = dict ( [ ( x , y ) for ( y , x ) in enumerate ( rev_vocab ) ] )
    return vocab , rev_vocab
else :
    raise ValueError ( "Vocabulary file %s not found." , vocabulary_path )


signature:
def sentence_to_token_ids ( sentence , vocabulary , tokenizer = None , normalize_digits = True , UNK_ID = 3 , _DIGIT_RE = re . compile ( br"\d" ) ) :
    """Convert a string to list of integers representing token - ids .
 For example , a sentence " I have a dog " may become tokenized into
 [ " I " , " have " , " a " , " dog " ] and with vocabulary { " I " : 1 , " have " : 2,
 " a " : 4 , " dog " : 7 " } this function will return [ 1 , 2 , 4 , 7 ] .
 Parameters
 sentence : tensorflow . python . platform . gfile . GFile Object
 The sentence in bytes format to convert to token - ids , see ` ` basic _ tokenizer ( ) ` ` and ` ` data _ to _ token _ ids ( ) ` ` .
 vocabulary : dictionary
 Mmapping tokens to integers .
 tokenizer : function
 A function to use to tokenize each sentence . If None , ` ` basic _ tokenizer ` ` will be used .
 normalize _ digits : boolean
 If true , all digits are replaced by 0.
 Returns
 list of int
 The token - ids for the sentence ."""

implementation:
if tokenizer :
    words = tokenizer ( sentence )
else :
    words = basic_tokenizer ( sentence )
if not normalize_digits :
    return [ vocabulary . get ( w , UNK_ID ) for w in words ]
# Normalize digits by 0 before looking words up in the vocabulary .
return [ vocabulary . get ( re . sub ( _DIGIT_RE , b"0" , w ) , UNK_ID ) for w in words ]


signature:
def data_to_token_ids ( data_path , target_path , vocabulary_path , tokenizer = None , normalize_digits = True , UNK_ID = 3 , _DIGIT_RE = re . compile ( br"\d" ) ) :
    """Tokenize data file and turn into token - ids using given vocabulary file .
 This function loads data line - by - line from data _ path , calls the above
 sentence _ to _ token _ ids , and saves the result to target _ path . See comment
 for sentence _ to _ token _ ids on the details of token - ids format .
 Parameters
 data _ path : str
 Path to the data file in one - sentence - per - line format .
 target _ path : str
 Path where the file with token - ids will be created .
 vocabulary _ path : str
 Path to the vocabulary file .
 tokenizer : function
 A function to use to tokenize each sentence . If None , ` ` basic _ tokenizer ` ` will be used .
 normalize _ digits : boolean
 If true , all digits are replaced by 0.
 References
 - Code from ` ` / tensorflow / models / rnn / translation / data _ utils . py ` `"""

implementation:
if not gfile . Exists ( target_path ) :
    tl . logging . info ( "Tokenizing data in %s" % data_path )
    vocab , _ = initialize_vocabulary ( vocabulary_path )
    with gfile . GFile ( data_path , mode = "rb" ) as data_file :
        with gfile . GFile ( target_path , mode = "w" ) as tokens_file :
            counter = 0
            for line in data_file :
                counter += 1
                if counter % 100000 == 0 :
                    tl . logging . info ( "  tokenizing line %d" % counter )
                token_ids = sentence_to_token_ids ( line , vocab , tokenizer , normalize_digits , UNK_ID = UNK_ID , _DIGIT_RE = _DIGIT_RE )
                tokens_file . write ( " " . join ( [ str ( tok ) for tok in token_ids ] ) + "\n" )
else :
    tl . logging . info ( "Target path %s exists" % target_path )


signature:
def moses_multi_bleu ( hypotheses , references , lowercase = False ) :
    """Calculate the bleu score for hypotheses and references
 using the MOSES ulti - bleu . perl script .
 Parameters
 hypotheses : numpy . array . string
 A numpy array of strings where each string is a single example .
 references : numpy . array . string
 A numpy array of strings where each string is a single example .
 lowercase : boolean
 If True , pass the " - lc " flag to the multi - bleu script
 Examples
 > > > hypotheses = [ " a bird is flying on the sky " ]
 > > > references = [ " two birds are flying on the sky " , " a bird is on the top of the tree " , " an airplane is on the sky " , ]
 > > > score = tl . nlp . moses _ multi _ bleu ( hypotheses , references )
 Returns
 float
 The BLEU score
 References
 - ` Google / seq2seq / metric / bleu < https : / / github . com / google / seq2seq > ` _ _"""

implementation:
if np . size ( hypotheses ) == 0 :
    return np . float32 ( 0.0 )
# Get MOSES multi - bleu script
try :
    multi_bleu_path , _ = urllib . request . urlretrieve ( "https://raw.githubusercontent.com/moses-smt/mosesdecoder/" "master/scripts/generic/multi-bleu.perl" )
    os . chmod ( multi_bleu_path , 0o755 )
except Exception : # pylint : disable = W0702
    tl . logging . info ( "Unable to fetch multi-bleu.perl script, using local." )
    metrics_dir = os . path . dirname ( os . path . realpath ( __file__ ) )
    bin_dir = os . path . abspath ( os . path . join ( metrics_dir , ".." , ".." , "bin" ) )
    multi_bleu_path = os . path . join ( bin_dir , "tools/multi-bleu.perl" )
# Dump hypotheses and references to tempfiles
hypothesis_file = tempfile . NamedTemporaryFile ( )
hypothesis_file . write ( "\n" . join ( hypotheses ) . encode ( "utf-8" ) )
hypothesis_file . write ( b"\n" )
hypothesis_file . flush ( )
reference_file = tempfile . NamedTemporaryFile ( )
reference_file . write ( "\n" . join ( references ) . encode ( "utf-8" ) )
reference_file . write ( b"\n" )
reference_file . flush ( )
# Calculate BLEU using multi - bleu script
with open ( hypothesis_file . name , "r" ) as read_pred :
    bleu_cmd = [ multi_bleu_path ]
    if lowercase :
        bleu_cmd += [ "-lc" ]
    bleu_cmd += [ reference_file . name ]
    try :
        bleu_out = subprocess . check_output ( bleu_cmd , stdin = read_pred , stderr = subprocess . STDOUT )
        bleu_out = bleu_out . decode ( "utf-8" )
        bleu_score = re . search ( r"BLEU = (.+?)," , bleu_out ) . group ( 1 )
        bleu_score = float ( bleu_score )
    except subprocess . CalledProcessError as error :
        if error . output is not None :
            tl . logging . warning ( "multi-bleu.perl script returned non-zero exit code" )
            tl . logging . warning ( error . output )
        bleu_score = np . float32 ( 0.0 )
# Close temp files
hypothesis_file . close ( )
reference_file . close ( )
return np . float32 ( bleu_score )


signature:
def word_to_id ( self , word ) :
    """Returns the integer id of a word string ."""

implementation:
if word in self . _vocab :
    return self . _vocab [ word ]
else :
    return self . _unk_id


signature:
def word_to_id ( self , word ) :
    """Returns the integer word id of a word string ."""

implementation:
if word in self . vocab :
    return self . vocab [ word ]
else :
    return self . unk_id


signature:
def id_to_word ( self , word_id ) :
    """Returns the word string of an integer word id ."""

implementation:
if word_id >= len ( self . reverse_vocab ) :
    return self . reverse_vocab [ self . unk_id ]
else :
    return self . reverse_vocab [ word_id ]


signature:
def basic_clean_str ( string ) :
    """Tokenization / string cleaning for a datasets ."""

implementation:
string = re . sub ( r"\n" , " " , string )
string = re . sub ( r"\'s" , " \'s" , string )
# it ' s - - > it ' s
string = re . sub ( r"\’s" , " \'s" , string )
string = re . sub ( r"\'ve" , " have" , string )
# they ' ve - - > they have
string = re . sub ( r"\’ve" , " have" , string )
string = re . sub ( r"\'t" , " not" , string )
# can ' t - - > can not
string = re . sub ( r"\’t" , " not" , string )
string = re . sub ( r"\'re" , " are" , string )
# they ' re - - > they are
string = re . sub ( r"\’re" , " are" , string )
string = re . sub ( r"\'d" , "" , string )
# I ' d ( I had , I would ) - - > I
string = re . sub ( r"\’d" , "" , string )
string = re . sub ( r"\'ll" , " will" , string )
# I ' ll - - > I will
string = re . sub ( r"\’ll" , " will" , string )
string = re . sub ( r"\“" , "  " , string )
# “ a ” - - > “ a ”
string = re . sub ( r"\”" , "  " , string )
string = re . sub ( r"\"" , "  " , string )
# " a " - - > " a "
string = re . sub ( r"\'" , "  " , string )
# they ' - - > they '
string = re . sub ( r"\’" , "  " , string )
# they ’ - - > they ’
string = re . sub ( r"\." , " . " , string )
# they . - - > they .
string = re . sub ( r"\," , " , " , string )
# they , - - > they ,
string = re . sub ( r"\!" , " ! " , string )
string = re . sub ( r"\-" , "  " , string )
# " low - cost " - - > lost cost
string = re . sub ( r"\(" , "  " , string )
# ( they ) - - > ( they )
string = re . sub ( r"\)" , "  " , string )
# ( they ) - - > ( they )
string = re . sub ( r"\]" , "  " , string )
# they ] - - > they ]
string = re . sub ( r"\[" , "  " , string )
# they [ - - > they [
string = re . sub ( r"\?" , "  " , string )
# they ? - - > they ?
string = re . sub ( r"\>" , "  " , string )
# they > - - > they >
string = re . sub ( r"\<" , "  " , string )
# they < - - > they <
string = re . sub ( r"\=" , "  " , string )
# easier = - - > easier =
string = re . sub ( r"\;" , "  " , string )
# easier ; - - > easier ;
string = re . sub ( r"\;" , "  " , string )
string = re . sub ( r"\:" , "  " , string )
# easier : - - > easier :
string = re . sub ( r"\"" , "  " , string )
# easier " - - > easier "
string = re . sub ( r"\$" , "  " , string )
# $ 380 - - > $ 380
string = re . sub ( r"\_" , "  " , string )
# _100 - - > _ 100
string = re . sub ( r"\s{2,}" , " " , string )
# Akara is handsome - - > Akara is handsome
return string . strip ( ) . lower ( )


signature:
def main_restore_embedding_layer ( ) :
    """How to use Embedding layer , and how to convert IDs to vector ,
 IDs to words , etc ."""

implementation:
# Step 1 : Build the embedding matrix and load the existing embedding matrix .
vocabulary_size = 50000
embedding_size = 128
model_file_name = "model_word2vec_50k_128"
batch_size = None
print ( "Load existing embedding matrix and dictionaries" )
all_var = tl . files . load_npy_to_any ( name = model_file_name + '.npy' )
data = all_var [ 'data' ]
count = all_var [ 'count' ]
dictionary = all_var [ 'dictionary' ]
reverse_dictionary = all_var [ 'reverse_dictionary' ]
tl . nlp . save_vocab ( count , name = 'vocab_' + model_file_name + '.txt' )
del all_var , data , count
load_params = tl . files . load_npz ( name = model_file_name + '.npz' )
x = tf . placeholder ( tf . int32 , shape = [ batch_size ] )
emb_net = tl . layers . EmbeddingInputlayer ( x , vocabulary_size , embedding_size , name = 'emb' )
# sess . run ( tf . global _ variables _ initializer ( ) )
sess . run ( tf . global_variables_initializer ( ) )
tl . files . assign_params ( sess , [ load_params [ 0 ] ] , emb_net )
emb_net . print_params ( )
emb_net . print_layers ( )
# Step 2 : Input word ( s ) , output the word vector ( s ) .
word = b'hello'
word_id = dictionary [ word ]
print ( 'word_id:' , word_id )
words = [ b'i' , b'am' , b'tensor' , b'layer' ]
word_ids = tl . nlp . words_to_word_ids ( words , dictionary , _UNK )
context = tl . nlp . word_ids_to_words ( word_ids , reverse_dictionary )
print ( 'word_ids:' , word_ids )
print ( 'context:' , context )
vector = sess . run ( emb_net . outputs , feed_dict = { x : [ word_id ] } )
print ( 'vector:' , vector . shape )
vectors = sess . run ( emb_net . outputs , feed_dict = { x : word_ids } )
print ( 'vectors:' , vectors . shape )


signature:
def createAndStartSwarm ( client , clientInfo = "" , clientKey = "" , params = "" , minimumWorkers = None , maximumWorkers = None , alreadyRunning = False ) :
    """Create and start a swarm job .
 Args :
 client - A string identifying the calling client . There is a small limit
 for the length of the value . See ClientJobsDAO . CLIENT _ MAX _ LEN .
 clientInfo - JSON encoded dict of client specific information .
 clientKey - Foreign key . Limited in length , see ClientJobsDAO . _ initTables .
 params - JSON encoded dict of the parameters for the job . This can be
 fetched out of the database by the worker processes based on the jobID .
 minimumWorkers - The minimum workers to allocate to the swarm . Set to None
 to use the default .
 maximumWorkers - The maximum workers to allocate to the swarm . Set to None
 to use the swarm default . Set to 0 to use the maximum scheduler value .
 alreadyRunning - Insert a job record for an already running process . Used
 for testing ."""

implementation:
if minimumWorkers is None :
    minimumWorkers = Configuration . getInt ( "nupic.hypersearch.minWorkersPerSwarm" )
if maximumWorkers is None :
    maximumWorkers = Configuration . getInt ( "nupic.hypersearch.maxWorkersPerSwarm" )
return ClientJobsDAO . get ( ) . jobInsert ( client = client , cmdLine = "$HYPERSEARCH" , clientInfo = clientInfo , clientKey = clientKey , alreadyRunning = alreadyRunning , params = params , minimumWorkers = minimumWorkers , maximumWorkers = maximumWorkers , jobType = ClientJobsDAO . JOB_TYPE_HS )


signature:
def getSwarmModelParams ( modelID ) :
    """Retrieve the Engine - level model params from a Swarm model
 Args :
 modelID - Engine - level model ID of the Swarm model
 Returns :
 JSON - encoded string containing Model Params"""

implementation:
# TODO : the use of nupic . frameworks . opf . helpers . loadExperimentDescriptionScriptFromDir when
# retrieving module params results in a leakage of pf _ base _ descriptionNN and
# pf _ descriptionNN module imports for every call to getSwarmModelParams , so
# the leakage is unlimited when getSwarmModelParams is called by a
# long - running process . An alternate solution is to execute the guts of
# this function ' s logic in a seprate process ( via multiprocessing module ) .
cjDAO = ClientJobsDAO . get ( )
( jobID , description ) = cjDAO . modelsGetFields ( modelID , [ "jobId" , "genDescription" ] )
( baseDescription , ) = cjDAO . jobGetFields ( jobID , [ "genBaseDescription" ] )
# Construct a directory with base . py and description . py for loading model
# params , and use nupic . frameworks . opf . helpers to extract model params from
# those files
descriptionDirectory = tempfile . mkdtemp ( )
try :
    baseDescriptionFilePath = os . path . join ( descriptionDirectory , "base.py" )
    with open ( baseDescriptionFilePath , mode = "wb" ) as f :
        f . write ( baseDescription )
    descriptionFilePath = os . path . join ( descriptionDirectory , "description.py" )
    with open ( descriptionFilePath , mode = "wb" ) as f :
        f . write ( description )
    expIface = helpers . getExperimentDescriptionInterfaceFromModule ( helpers . loadExperimentDescriptionScriptFromDir ( descriptionDirectory ) )
    return json . dumps ( dict ( modelConfig = expIface . getModelDescription ( ) , inferenceArgs = expIface . getModelControl ( ) . get ( "inferenceArgs" , None ) ) )
finally :
    shutil . rmtree ( descriptionDirectory , ignore_errors = True )


signature:
def enableConcurrencyChecks ( maxConcurrency , raiseException = True ) :
    """Enable the diagnostic feature for debugging unexpected concurrency in
 acquiring ConnectionWrapper instances .
 NOTE : This MUST be done early in your application ' s execution , BEFORE any
 accesses to ConnectionFactory or connection policies from your application
 ( including imports and sub - imports of your app ) .
 Parameters :
 maxConcurrency : A non - negative integer that represents the maximum expected
 number of outstanding connections . When this value is
 exceeded , useful information will be logged and , depending
 on the value of the raiseException arg ,
 ConcurrencyExceededError may be raised .
 raiseException : If true , ConcurrencyExceededError will be raised when
 maxConcurrency is exceeded ."""

implementation:
global g_max_concurrency , g_max_concurrency_raise_exception
assert maxConcurrency >= 0
g_max_concurrency = maxConcurrency
g_max_concurrency_raise_exception = raiseException
return


signature:
def _getCommonSteadyDBArgsDict ( ) :
    """Returns a dictionary of arguments for DBUtils . SteadyDB . SteadyDBConnection
 constructor ."""

implementation: return dict ( creator = pymysql , host = Configuration . get ( 'nupic.cluster.database.host' ) , port = int ( Configuration . get ( 'nupic.cluster.database.port' ) ) , user = Configuration . get ( 'nupic.cluster.database.user' ) , passwd = Configuration . get ( 'nupic.cluster.database.passwd' ) , charset = 'utf8' , use_unicode = True , setsession = [ 'SET AUTOCOMMIT = 1' ] )


signature:
def _getLogger ( cls , logLevel = None ) :
    """Gets a logger for the given class in this module"""

implementation:
logger = logging . getLogger ( "." . join ( [ 'com.numenta' , _MODULE_NAME , cls . __name__ ] ) )
if logLevel is not None :
    logger . setLevel ( logLevel )
return logger


signature:
def get ( cls ) :
    """Acquire a ConnectionWrapper instance that represents a connection
 to the SQL server per nupic . cluster . database . * configuration settings .
 NOTE : caller is responsible for calling the ConnectionWrapper instance ' s
 release ( ) method after using the connection in order to release resources .
 Better yet , use the returned ConnectionWrapper instance in a Context Manager
 statement for automatic invocation of release ( ) :
 Example :
 # If using Jython 2.5 . x , first import with _ statement at the very top of
 your script ( don ' t need this import for Jython / Python 2.6 . x and later ) :
 from _ _ future _ _ import with _ statement
 # Then :
 from nupic . database . Connection import ConnectionFactory
 # Then use it like this
 with ConnectionFactory . get ( ) as conn :
 conn . cursor . execute ( " SELECT . . . " )
 conn . cursor . fetchall ( )
 conn . cursor . execute ( " INSERT . . . " )
 WARNING : DO NOT close the underlying connection or cursor as it may be
 shared by other modules in your process . ConnectionWrapper ' s release ( )
 method will do the right thing .
 Parameters :
 retval : A ConnectionWrapper instance . NOTE : Caller is responsible
 for releasing resources as described above ."""

implementation:
if cls . _connectionPolicy is None :
    logger = _getLogger ( cls )
    logger . info ( "Creating db connection policy via provider %r" , cls . _connectionPolicyInstanceProvider )
    cls . _connectionPolicy = cls . _connectionPolicyInstanceProvider ( )
    logger . debug ( "Created connection policy: %r" , cls . _connectionPolicy )
return cls . _connectionPolicy . acquireConnection ( )


signature:
def _createDefaultPolicy ( cls ) :
    """[ private ] Create the default database connection policy instance
 Parameters :
 retval : The default database connection policy instance"""

implementation:
logger = _getLogger ( cls )
logger . debug ( "Creating database connection policy: platform=%r; pymysql.VERSION=%r" , platform . system ( ) , pymysql . VERSION )
if platform . system ( ) == "Java" : # NOTE : PooledDB doesn ' t seem to work under Jython
# NOTE : not appropriate for multi - threaded applications .
# TODO : this was fixed in Webware DBUtils r8228 , so once
# we pick up a realease with this fix , we should use
# PooledConnectionPolicy for both Jython and Python .
    policy = SingleSharedConnectionPolicy ( )
else :
    policy = PooledConnectionPolicy ( )
return policy


signature:
def release ( self ) :
    """Release the database connection and cursor
 The receiver of the Connection instance MUST call this method in order
 to reclaim resources"""

implementation:
self . _logger . debug ( "Releasing: %r" , self )
# Discard self from set of outstanding instances
if self . _addedToInstanceSet :
    try :
        self . _clsOutstandingInstances . remove ( self )
    except :
        self . _logger . exception ( "Failed to remove self from _clsOutstandingInstances: %r;" , self )
        raise
self . _releaser ( dbConn = self . dbConn , cursor = self . cursor )
self . __class__ . _clsNumOutstanding -= 1
assert self . _clsNumOutstanding >= 0 , "_clsNumOutstanding=%r" % ( self . _clsNumOutstanding , )
self . _releaser = None
self . cursor = None
self . dbConn = None
self . _creationTracebackString = None
self . _addedToInstanceSet = False
self . _logger = None
return


signature:
def _trackInstanceAndCheckForConcurrencyViolation ( self ) :
    """Check for concurrency violation and add self to
 _ clsOutstandingInstances .
 ASSUMPTION : Called from constructor BEFORE _ clsNumOutstanding is
 incremented"""

implementation:
global g_max_concurrency , g_max_concurrency_raise_exception
assert g_max_concurrency is not None
assert self not in self . _clsOutstandingInstances , repr ( self )
# Populate diagnostic info
self . _creationTracebackString = traceback . format_stack ( )
# Check for concurrency violation
if self . _clsNumOutstanding >= g_max_concurrency : # NOTE : It ' s possible for _ clsNumOutstanding to be greater than
# len ( _ clsOutstandingInstances ) if concurrency check was enabled after
# unrelease allocations .
    errorMsg = ( "With numOutstanding=%r, exceeded concurrency limit=%r " "when requesting %r. OTHER TRACKED UNRELEASED " "INSTANCES (%s): %r" ) % ( self . _clsNumOutstanding , g_max_concurrency , self , len ( self . _clsOutstandingInstances ) , self . _clsOutstandingInstances , )
    self . _logger . error ( errorMsg )
    if g_max_concurrency_raise_exception :
        raise ConcurrencyExceededError ( errorMsg )
# Add self to tracked instance set
self . _clsOutstandingInstances . add ( self )
self . _addedToInstanceSet = True
return


signature:
def close ( self ) :
    """Close the policy instance and its shared database connection ."""

implementation:
self . _logger . info ( "Closing" )
if self . _conn is not None :
    self . _conn . close ( )
    self . _conn = None
else :
    self . _logger . warning ( "close() called, but connection policy was alredy closed" )
return


signature:
def acquireConnection ( self ) :
    """Get a Connection instance .
 Parameters :
 retval : A ConnectionWrapper instance . NOTE : Caller
 is responsible for calling the ConnectionWrapper
 instance ' s release ( ) method or use it in a context manager
 expression ( with . . . as : ) to release resources ."""

implementation:
self . _logger . debug ( "Acquiring connection" )
# Check connection and attempt to re - establish it if it died ( this is
# what PooledDB does )
self . _conn . _ping_check ( )
connWrap = ConnectionWrapper ( dbConn = self . _conn , cursor = self . _conn . cursor ( ) , releaser = self . _releaseConnection , logger = self . _logger )
return connWrap


signature:
def close ( self ) :
    """Close the policy instance and its database connection pool ."""

implementation:
self . _logger . info ( "Closing" )
if self . _pool is not None :
    self . _pool . close ( )
    self . _pool = None
else :
    self . _logger . warning ( "close() called, but connection policy was alredy closed" )
return


signature:
def acquireConnection ( self ) :
    """Get a connection from the pool .
 Parameters :
 retval : A ConnectionWrapper instance . NOTE : Caller
 is responsible for calling the ConnectionWrapper
 instance ' s release ( ) method or use it in a context manager
 expression ( with . . . as : ) to release resources ."""

implementation:
self . _logger . debug ( "Acquiring connection" )
dbConn = self . _pool . connection ( shareable = False )
connWrap = ConnectionWrapper ( dbConn = dbConn , cursor = dbConn . cursor ( ) , releaser = self . _releaseConnection , logger = self . _logger )
return connWrap


signature:
def close ( self ) :
    """Close the policy instance ."""

implementation:
self . _logger . info ( "Closing" )
if self . _opened :
    self . _opened = False
else :
    self . _logger . warning ( "close() called, but connection policy was alredy closed" )
return


signature:
def acquireConnection ( self ) :
    """Create a Connection instance .
 Parameters :
 retval : A ConnectionWrapper instance . NOTE : Caller
 is responsible for calling the ConnectionWrapper
 instance ' s release ( ) method or use it in a context manager
 expression ( with . . . as : ) to release resources ."""

implementation:
self . _logger . debug ( "Acquiring connection" )
dbConn = SteadyDB . connect ( ** _getCommonSteadyDBArgsDict ( ) )
connWrap = ConnectionWrapper ( dbConn = dbConn , cursor = dbConn . cursor ( ) , releaser = self . _releaseConnection , logger = self . _logger )
return connWrap


signature:
def _releaseConnection ( self , dbConn , cursor ) :
    """Release database connection and cursor ; passed as a callback to
 ConnectionWrapper"""

implementation:
self . _logger . debug ( "Releasing connection" )
# Close the cursor
cursor . close ( )
# . . . then close the database connection
dbConn . close ( )
return


signature:
def getSpec ( cls ) :
    """Overrides : meth : ` nupic . bindings . regions . PyRegion . PyRegion . getSpec ` ."""

implementation:
ns = dict ( description = KNNAnomalyClassifierRegion . __doc__ , singleNodeOnly = True , inputs = dict ( spBottomUpOut = dict ( description = """The output signal generated from the bottom-up inputs
                            from lower levels.""" , dataType = 'Real32' , count = 0 , required = True , regionLevel = False , isDefaultInput = True , requireSplitterMap = False ) , tpTopDownOut = dict ( description = """The top-down inputsignal, generated from
                          feedback from upper levels""" , dataType = 'Real32' , count = 0 , required = True , regionLevel = False , isDefaultInput = True , requireSplitterMap = False ) , tpLrnActiveStateT = dict ( description = """Active cells in the learn state at time T from TM.
                        This is used to classify on.""" , dataType = 'Real32' , count = 0 , required = True , regionLevel = False , isDefaultInput = True , requireSplitterMap = False ) , sequenceIdIn = dict ( description = "Sequence ID" , dataType = 'UInt64' , count = 1 , required = False , regionLevel = True , isDefaultInput = False , requireSplitterMap = False ) , ) , outputs = dict ( ) , parameters = dict ( trainRecords = dict ( description = 'Number of records to wait for training' , dataType = 'UInt32' , count = 1 , constraints = '' , defaultValue = 0 , accessMode = 'Create' ) , anomalyThreshold = dict ( description = 'Threshold used to classify anomalies.' , dataType = 'Real32' , count = 1 , constraints = '' , defaultValue = 0 , accessMode = 'Create' ) , cacheSize = dict ( description = 'Number of records to store in cache.' , dataType = 'UInt32' , count = 1 , constraints = '' , defaultValue = 0 , accessMode = 'Create' ) , classificationVectorType = dict ( description = """Vector type to use when classifying.
              1 - Vector Column with Difference (TM and SP)
            """ , dataType = 'UInt32' , count = 1 , constraints = '' , defaultValue = 1 , accessMode = 'ReadWrite' ) , activeColumnCount = dict ( description = """Number of active columns in a given step. Typically
            equivalent to SP.numActiveColumnsPerInhArea""" , dataType = 'UInt32' , count = 1 , constraints = '' , defaultValue = 40 , accessMode = 'ReadWrite' ) , classificationMaxDist = dict ( description = """Maximum distance a sample can be from an anomaly
            in the classifier to be labeled as an anomaly.

            Ex: With rawOverlap distance, a value of 0.65 means that the points
            must be at most a distance 0.65 apart from each other. This
            translates to they must be at least 35% similar.""" , dataType = 'Real32' , count = 1 , constraints = '' , defaultValue = 0.65 , accessMode = 'Create' ) ) , commands = dict ( getLabels = dict ( description = "Returns a list of label dicts with properties ROWID and labels." "ROWID corresponds to the records id and labels is a list of " "strings representing the records labels.  Takes additional " "integer properties start and end representing the range that " "will be returned." ) , addLabel = dict ( description = "Takes parameters start, end and labelName. Adds the label " "labelName to the records from start to end. This will recalculate " "labels from end to the most recent record." ) , removeLabels = dict ( description = "Takes additional parameters start, end, labelFilter.  Start and " "end correspond to range to remove the label. Remove labels from " "each record with record ROWID in range from start to end, " "noninclusive of end. Removes all records if labelFilter is None, " "otherwise only removes the labels eqaul to labelFilter." ) ) )
ns [ 'parameters' ] . update ( KNNClassifierRegion . getSpec ( ) [ 'parameters' ] )
return ns


signature:
def getParameter ( self , name , index = - 1 ) :
    """Overrides : meth : ` nupic . bindings . regions . PyRegion . PyRegion . getParameter ` ."""

implementation:
if name == "trainRecords" :
    return self . trainRecords
elif name == "anomalyThreshold" :
    return self . anomalyThreshold
elif name == "activeColumnCount" :
    return self . _activeColumnCount
elif name == "classificationMaxDist" :
    return self . _classificationMaxDist
else : # If any spec parameter name is the same as an attribute , this call
# will get it automatically , e . g . self . learningMode
    return PyRegion . getParameter ( self , name , index )


signature:
def setParameter ( self , name , index , value ) :
    """Overrides : meth : ` nupic . bindings . regions . PyRegion . PyRegion . setParameter ` ."""

implementation:
if name == "trainRecords" : # Ensure that the trainRecords can only be set to minimum of the ROWID in
# the saved states
    if not ( isinstance ( value , float ) or isinstance ( value , int ) ) :
        raise HTMPredictionModelInvalidArgument ( "Invalid argument type \'%s\'. threshold " "must be a number." % ( type ( value ) ) )
    if len ( self . _recordsCache ) > 0 and value < self . _recordsCache [ 0 ] . ROWID :
        raise HTMPredictionModelInvalidArgument ( "Invalid value. autoDetectWaitRecord " "value must be valid record within output stream. Current minimum " " ROWID in output stream is %d." % ( self . _recordsCache [ 0 ] . ROWID ) )
    self . trainRecords = value
    # Remove any labels before the first cached record ( wont be used anymore )
    self . _deleteRangeFromKNN ( 0 , self . _recordsCache [ 0 ] . ROWID )
    # Reclassify all states
    self . _classifyStates ( )
elif name == "anomalyThreshold" :
    if not ( isinstance ( value , float ) or isinstance ( value , int ) ) :
        raise HTMPredictionModelInvalidArgument ( "Invalid argument type \'%s\'. threshold " "must be a number." % ( type ( value ) ) )
    self . anomalyThreshold = value
    self . _classifyStates ( )
elif name == "classificationMaxDist" :
    if not ( isinstance ( value , float ) or isinstance ( value , int ) ) :
        raise HTMPredictionModelInvalidArgument ( "Invalid argument type \'%s\'. " "classificationMaxDist must be a number." % ( type ( value ) ) )
    self . _classificationMaxDist = value
    self . _classifyStates ( )
elif name == "activeColumnCount" :
    self . _activeColumnCount = value
else :
    return PyRegion . setParameter ( self , name , index , value )


signature:
def compute ( self , inputs , outputs ) :
    """Process one input sample .
 This method is called by the runtime engine ."""

implementation:
record = self . _constructClassificationRecord ( inputs )
# Classify this point after waiting the classification delay
if record . ROWID >= self . getParameter ( 'trainRecords' ) :
    self . _classifyState ( record )
# Save new classification record and keep history as moving window
self . _recordsCache . append ( record )
while len ( self . _recordsCache ) > self . cacheSize :
    self . _recordsCache . pop ( 0 )
self . labelResults = record . anomalyLabel
self . _iteration += 1


signature:
def _classifyState ( self , state ) :
    """Reclassifies given state ."""

implementation:
# Record is before wait period do not classifiy
if state . ROWID < self . getParameter ( 'trainRecords' ) :
    if not state . setByUser :
        state . anomalyLabel = [ ]
        self . _deleteRecordsFromKNN ( [ state ] )
    return
label = KNNAnomalyClassifierRegion . AUTO_THRESHOLD_CLASSIFIED_LABEL
autoLabel = label + KNNAnomalyClassifierRegion . AUTO_TAG
# Update the label based on classifications
newCategory = self . _recomputeRecordFromKNN ( state )
labelList = self . _categoryToLabelList ( newCategory )
if state . setByUser :
    if label in state . anomalyLabel :
        state . anomalyLabel . remove ( label )
    if autoLabel in state . anomalyLabel :
        state . anomalyLabel . remove ( autoLabel )
    labelList . extend ( state . anomalyLabel )
# Add threshold classification label if above threshold , else if
# classified to add the auto threshold classification .
if state . anomalyScore >= self . getParameter ( 'anomalyThreshold' ) :
    labelList . append ( label )
elif label in labelList :
    ind = labelList . index ( label )
    labelList [ ind ] = autoLabel
# Make all entries unique
labelList = list ( set ( labelList ) )
# If both above threshold and auto classified above - remove auto label
if label in labelList and autoLabel in labelList :
    labelList . remove ( autoLabel )
if state . anomalyLabel == labelList :
    return
# Update state ' s labeling
state . anomalyLabel = labelList
# Update KNN Classifier with new labeling
if state . anomalyLabel == [ ] :
    self . _deleteRecordsFromKNN ( [ state ] )
else :
    self . _addRecordToKNN ( state )


signature:
def _constructClassificationRecord ( self , inputs ) :
    """Construct a _ HTMClassificationRecord based on the state of the model
 passed in through the inputs .
 Types for self . classificationVectorType :
 1 - TM active cells in learn state
 2 - SP columns concatenated with error from TM column predictions and SP"""

implementation:
# Count the number of unpredicted columns
allSPColumns = inputs [ "spBottomUpOut" ]
activeSPColumns = allSPColumns . nonzero ( ) [ 0 ]
score = anomaly . computeRawAnomalyScore ( activeSPColumns , self . _prevPredictedColumns )
spSize = len ( allSPColumns )
allTPCells = inputs [ 'tpTopDownOut' ]
tpSize = len ( inputs [ 'tpLrnActiveStateT' ] )
classificationVector = numpy . array ( [ ] )
if self . classificationVectorType == 1 : # Classification Vector : [ - - - TM Cells - - - ]
    classificationVector = numpy . zeros ( tpSize )
    activeCellMatrix = inputs [ "tpLrnActiveStateT" ] . reshape ( tpSize , 1 )
    activeCellIdx = numpy . where ( activeCellMatrix > 0 ) [ 0 ]
    if activeCellIdx . shape [ 0 ] > 0 :
        classificationVector [ numpy . array ( activeCellIdx , dtype = numpy . uint16 ) ] = 1
elif self . classificationVectorType == 2 : # Classification Vecotr : [ - - - SP - - - | - - - ( TM - SP ) - - - - ]
    classificationVector = numpy . zeros ( spSize + spSize )
    if activeSPColumns . shape [ 0 ] > 0 :
        classificationVector [ activeSPColumns ] = 1.0
    errorColumns = numpy . setdiff1d ( self . _prevPredictedColumns , activeSPColumns )
    if errorColumns . shape [ 0 ] > 0 :
        errorColumnIndexes = ( numpy . array ( errorColumns , dtype = numpy . uint16 ) + spSize )
        classificationVector [ errorColumnIndexes ] = 1.0
else :
    raise TypeError ( "Classification vector type must be either 'tpc' or" " 'sp_tpe', current value is %s" % ( self . classificationVectorType ) )
# Store the state for next time step
numPredictedCols = len ( self . _prevPredictedColumns )
predictedColumns = allTPCells . nonzero ( ) [ 0 ]
self . _prevPredictedColumns = copy . deepcopy ( predictedColumns )
if self . _anomalyVectorLength is None :
    self . _anomalyVectorLength = len ( classificationVector )
result = _CLAClassificationRecord ( ROWID = self . _iteration , # _ _ numRunCalls called
# at beginning of model . run
anomalyScore = score , anomalyVector = classificationVector . nonzero ( ) [ 0 ] . tolist ( ) , anomalyLabel = [ ] )
return result


signature:
def _addRecordToKNN ( self , record ) :
    """Adds the record to the KNN classifier ."""

implementation:
knn = self . _knnclassifier . _knn
prototype_idx = self . _knnclassifier . getParameter ( 'categoryRecencyList' )
category = self . _labelListToCategoryNumber ( record . anomalyLabel )
# If record is already in the classifier , overwrite its labeling
if record . ROWID in prototype_idx :
    knn . prototypeSetCategory ( record . ROWID , category )
    return
# Learn this pattern in the knn
pattern = self . _getStateAnomalyVector ( record )
rowID = record . ROWID
knn . learn ( pattern , category , rowID = rowID )


signature:
def _deleteRecordsFromKNN ( self , recordsToDelete ) :
    """Removes the given records from the classifier .
 parameters
 recordsToDelete - list of records to delete from the classififier"""

implementation:
prototype_idx = self . _knnclassifier . getParameter ( 'categoryRecencyList' )
idsToDelete = ( [ r . ROWID for r in recordsToDelete if not r . setByUser and r . ROWID in prototype_idx ] )
nProtos = self . _knnclassifier . _knn . _numPatterns
self . _knnclassifier . _knn . removeIds ( idsToDelete )
assert self . _knnclassifier . _knn . _numPatterns == nProtos - len ( idsToDelete )


signature:
def _deleteRangeFromKNN ( self , start = 0 , end = None ) :
    """Removes any stored records within the range from start to
 end . Noninclusive of end .
 parameters
 start - integer representing the ROWID of the start of the deletion range ,
 end - integer representing the ROWID of the end of the deletion range ,
 if None , it will default to end ."""

implementation:
prototype_idx = numpy . array ( self . _knnclassifier . getParameter ( 'categoryRecencyList' ) )
if end is None :
    end = prototype_idx . max ( ) + 1
idsIdxToDelete = numpy . logical_and ( prototype_idx >= start , prototype_idx < end )
idsToDelete = prototype_idx [ idsIdxToDelete ]
nProtos = self . _knnclassifier . _knn . _numPatterns
self . _knnclassifier . _knn . removeIds ( idsToDelete . tolist ( ) )
assert self . _knnclassifier . _knn . _numPatterns == nProtos - len ( idsToDelete )


signature:
def _recomputeRecordFromKNN ( self , record ) :
    """returns the classified labeling of record"""

implementation:
inputs = { "categoryIn" : [ None ] , "bottomUpIn" : self . _getStateAnomalyVector ( record ) , }
outputs = { "categoriesOut" : numpy . zeros ( ( 1 , ) ) , "bestPrototypeIndices" : numpy . zeros ( ( 1 , ) ) , "categoryProbabilitiesOut" : numpy . zeros ( ( 1 , ) ) }
# Only use points before record to classify and after the wait period .
classifier_indexes = numpy . array ( self . _knnclassifier . getParameter ( 'categoryRecencyList' ) )
valid_idx = numpy . where ( ( classifier_indexes >= self . getParameter ( 'trainRecords' ) ) & ( classifier_indexes < record . ROWID ) ) [ 0 ] . tolist ( )
if len ( valid_idx ) == 0 :
    return None
self . _knnclassifier . setParameter ( 'inferenceMode' , None , True )
self . _knnclassifier . setParameter ( 'learningMode' , None , False )
self . _knnclassifier . compute ( inputs , outputs )
self . _knnclassifier . setParameter ( 'learningMode' , None , True )
classifier_distances = self . _knnclassifier . getLatestDistances ( )
valid_distances = classifier_distances [ valid_idx ]
if valid_distances . min ( ) <= self . _classificationMaxDist :
    classifier_indexes_prev = classifier_indexes [ valid_idx ]
    rowID = classifier_indexes_prev [ valid_distances . argmin ( ) ]
    indexID = numpy . where ( classifier_indexes == rowID ) [ 0 ] [ 0 ]
    category = self . _knnclassifier . getCategoryList ( ) [ indexID ]
    return category
return None


signature:
def _labelToCategoryNumber ( self , label ) :
    """Since the KNN Classifier stores categories as numbers , we must store each
 label as a number . This method converts from a label to a unique number .
 Each label is assigned a unique bit so multiple labels may be assigned to
 a single record ."""

implementation:
if label not in self . saved_categories :
    self . saved_categories . append ( label )
return pow ( 2 , self . saved_categories . index ( label ) )


signature:
def _labelListToCategoryNumber ( self , labelList ) :
    """This method takes a list of labels and returns a unique category number .
 This enables this class to store a list of categories for each point since
 the KNN classifier only stores a single number category for each record ."""

implementation:
categoryNumber = 0
for label in labelList :
    categoryNumber += self . _labelToCategoryNumber ( label )
return categoryNumber


signature:
def _categoryToLabelList ( self , category ) :
    """Converts a category number into a list of labels"""

implementation:
if category is None :
    return [ ]
labelList = [ ]
labelNum = 0
while category > 0 :
    if category % 2 == 1 :
        labelList . append ( self . saved_categories [ labelNum ] )
    labelNum += 1
    category = category >> 1
return labelList


signature:
def _getStateAnomalyVector ( self , state ) :
    """Returns a state ' s anomaly vertor converting it from spare to dense"""

implementation:
vector = numpy . zeros ( self . _anomalyVectorLength )
vector [ state . anomalyVector ] = 1
return vector


signature:
def getLabels ( self , start = None , end = None ) :
    """Get the labels on classified points within range start to end . Not inclusive
 of end .
 : returns : ( dict ) with format :
 ' isProcessing ' : boolean ,
 ' recordLabels ' : list of results
 ` ` isProcessing ` ` - currently always false as recalculation blocks ; used if
 reprocessing of records is still being performed ;
 Each item in ` ` recordLabels ` ` is of format :
 ' ROWID ' : id of the row ,
 ' labels ' : list of strings"""

implementation:
if len ( self . _recordsCache ) == 0 :
    return { 'isProcessing' : False , 'recordLabels' : [ ] }
try :
    start = int ( start )
except Exception :
    start = 0
try :
    end = int ( end )
except Exception :
    end = self . _recordsCache [ - 1 ] . ROWID
if end <= start :
    raise HTMPredictionModelInvalidRangeError ( "Invalid supplied range for 'getLabels'." , debugInfo = { 'requestRange' : { 'startRecordID' : start , 'endRecordID' : end } , 'numRecordsStored' : len ( self . _recordsCache ) } )
results = { 'isProcessing' : False , 'recordLabels' : [ ] }
ROWIDX = numpy . array ( self . _knnclassifier . getParameter ( 'categoryRecencyList' ) )
validIdx = numpy . where ( ( ROWIDX >= start ) & ( ROWIDX < end ) ) [ 0 ] . tolist ( )
categories = self . _knnclassifier . getCategoryList ( )
for idx in validIdx :
    row = dict ( ROWID = int ( ROWIDX [ idx ] ) , labels = self . _categoryToLabelList ( categories [ idx ] ) )
    results [ 'recordLabels' ] . append ( row )
return results


signature:
def addLabel ( self , start , end , labelName ) :
    """Add the label labelName to each record with record ROWID in range from
 ` ` start ` ` to ` ` end ` ` , noninclusive of end .
 This will recalculate all points from end to the last record stored in the
 internal cache of this classifier .
 : param start : ( int ) start index
 : param end : ( int ) end index ( noninclusive )
 : param labelName : ( string ) label name"""

implementation:
if len ( self . _recordsCache ) == 0 :
    raise HTMPredictionModelInvalidRangeError ( "Invalid supplied range for 'addLabel'. " "Model has no saved records." )
try :
    start = int ( start )
except Exception :
    start = 0
try :
    end = int ( end )
except Exception :
    end = int ( self . _recordsCache [ - 1 ] . ROWID )
startID = self . _recordsCache [ 0 ] . ROWID
clippedStart = max ( 0 , start - startID )
clippedEnd = max ( 0 , min ( len ( self . _recordsCache ) , end - startID ) )
if clippedEnd <= clippedStart :
    raise HTMPredictionModelInvalidRangeError ( "Invalid supplied range for 'addLabel'." , debugInfo = { 'requestRange' : { 'startRecordID' : start , 'endRecordID' : end } , 'clippedRequestRange' : { 'startRecordID' : clippedStart , 'endRecordID' : clippedEnd } , 'validRange' : { 'startRecordID' : startID , 'endRecordID' : self . _recordsCache [ len ( self . _recordsCache ) - 1 ] . ROWID } , 'numRecordsStored' : len ( self . _recordsCache ) } )
# Add label to range [ clippedStart , clippedEnd )
for state in self . _recordsCache [ clippedStart : clippedEnd ] :
    if labelName not in state . anomalyLabel :
        state . anomalyLabel . append ( labelName )
        state . setByUser = True
        self . _addRecordToKNN ( state )
assert len ( self . saved_categories ) > 0
# Recompute [ end , . . . )
for state in self . _recordsCache [ clippedEnd : ] :
    self . _classifyState ( state )


signature:
def removeLabels ( self , start = None , end = None , labelFilter = None ) :
    """Remove labels from each record with record ROWID in range from
 ` ` start ` ` to ` ` end ` ` , noninclusive of end . Removes all records if
 ` ` labelFilter ` ` is None , otherwise only removes the labels equal to
 ` ` labelFilter ` ` .
 This will recalculate all points from end to the last record stored in the
 internal cache of this classifier .
 : param start : ( int ) start index
 : param end : ( int ) end index ( noninclusive )
 : param labelFilter : ( string ) label filter"""

implementation:
if len ( self . _recordsCache ) == 0 :
    raise HTMPredictionModelInvalidRangeError ( "Invalid supplied range for " "'removeLabels'. Model has no saved records." )
try :
    start = int ( start )
except Exception :
    start = 0
try :
    end = int ( end )
except Exception :
    end = self . _recordsCache [ - 1 ] . ROWID
startID = self . _recordsCache [ 0 ] . ROWID
clippedStart = 0 if start is None else max ( 0 , start - startID )
clippedEnd = len ( self . _recordsCache ) if end is None else max ( 0 , min ( len ( self . _recordsCache ) , end - startID ) )
if clippedEnd <= clippedStart :
    raise HTMPredictionModelInvalidRangeError ( "Invalid supplied range for " "'removeLabels'." , debugInfo = { 'requestRange' : { 'startRecordID' : start , 'endRecordID' : end } , 'clippedRequestRange' : { 'startRecordID' : clippedStart , 'endRecordID' : clippedEnd } , 'validRange' : { 'startRecordID' : startID , 'endRecordID' : self . _recordsCache [ len ( self . _recordsCache ) - 1 ] . ROWID } , 'numRecordsStored' : len ( self . _recordsCache ) } )
# Remove records within the cache
recordsToDelete = [ ]
for state in self . _recordsCache [ clippedStart : clippedEnd ] :
    if labelFilter is not None :
        if labelFilter in state . anomalyLabel :
            state . anomalyLabel . remove ( labelFilter )
    else :
        state . anomalyLabel = [ ]
    state . setByUser = False
    recordsToDelete . append ( state )
self . _deleteRecordsFromKNN ( recordsToDelete )
# Remove records not in cache
self . _deleteRangeFromKNN ( start , end )
# Recompute [ clippedEnd , . . . )
for state in self . _recordsCache [ clippedEnd : ] :
    self . _classifyState ( state )


signature:
def replace ( self , columnIndex , bitmap ) :
    """Wraps replaceSparseRow ( )"""

implementation: return super ( _SparseMatrixCorticalColumnAdapter , self ) . replaceSparseRow ( columnIndex , bitmap )


signature:
def update ( self , columnIndex , vector ) :
    """Wraps setRowFromDense ( )"""

implementation: return super ( _SparseMatrixCorticalColumnAdapter , self ) . setRowFromDense ( columnIndex , vector )


signature:
def setLocalAreaDensity ( self , localAreaDensity ) :
    """Sets the local area density . Invalidates the ' numActiveColumnsPerInhArea '
 parameter
 : param localAreaDensity : ( float ) value to set"""

implementation:
assert ( localAreaDensity > 0 and localAreaDensity <= 1 )
self . _localAreaDensity = localAreaDensity
self . _numActiveColumnsPerInhArea = 0


signature:
def getPotential ( self , columnIndex , potential ) :
    """: param columnIndex : ( int ) column index to get potential for .
 : param potential : ( list ) will be overwritten with column potentials . Must
 match the number of inputs ."""

implementation:
assert ( columnIndex < self . _numColumns )
potential [ : ] = self . _potentialPools [ columnIndex ]


signature:
def setPotential ( self , columnIndex , potential ) :
    """Sets the potential mapping for a given column . ` ` potential ` ` size must match
 the number of inputs , and must be greater than ` ` stimulusThreshold ` ` .
 : param columnIndex : ( int ) column index to set potential for .
 : param potential : ( list ) value to set ."""

implementation:
assert ( columnIndex < self . _numColumns )
potentialSparse = numpy . where ( potential > 0 ) [ 0 ]
if len ( potentialSparse ) < self . _stimulusThreshold :
    raise Exception ( "This is likely due to a " + "value of stimulusThreshold that is too large relative " + "to the input size." )
self . _potentialPools . replace ( columnIndex , potentialSparse )


signature:
def getPermanence ( self , columnIndex , permanence ) :
    """Returns the permanence values for a given column . ` ` permanence ` ` size
 must match the number of inputs .
 : param columnIndex : ( int ) column index to get permanence for .
 : param permanence : ( list ) will be overwritten with permanences ."""

implementation:
assert ( columnIndex < self . _numColumns )
permanence [ : ] = self . _permanences [ columnIndex ]


signature:
def setPermanence ( self , columnIndex , permanence ) :
    """Sets the permanence values for a given column . ` ` permanence ` ` size must
 match the number of inputs .
 : param columnIndex : ( int ) column index to set permanence for .
 : param permanence : ( list ) value to set ."""

implementation:
assert ( columnIndex < self . _numColumns )
self . _updatePermanencesForColumn ( permanence , columnIndex , raisePerm = False )


signature:
def getConnectedSynapses ( self , columnIndex , connectedSynapses ) :
    """: param connectedSynapses : ( list ) will be overwritten
 : returns : ( iter ) the connected synapses for a given column .
 ` ` connectedSynapses ` ` size must match the number of inputs"""

implementation:
assert ( columnIndex < self . _numColumns )
connectedSynapses [ : ] = self . _connectedSynapses [ columnIndex ]


signature:
def stripUnlearnedColumns ( self , activeArray ) :
    """Removes the set of columns who have never been active from the set of
 active columns selected in the inhibition round . Such columns cannot
 represent learned pattern and are therefore meaningless if only inference
 is required . This should not be done when using a random , unlearned SP
 since you would end up with no active columns .
 : param activeArray : An array whose size is equal to the number of columns .
 Any columns marked as active with an activeDutyCycle of 0 have
 never been activated before and therefore are not active due to
 learning . Any of these ( unlearned ) columns will be disabled ( set to 0 ) ."""

implementation:
neverLearned = numpy . where ( self . _activeDutyCycles == 0 ) [ 0 ]
activeArray [ neverLearned ] = 0


signature:
def _updateMinDutyCycles ( self ) :
    """Updates the minimum duty cycles defining normal activity for a column . A
 column with activity duty cycle below this minimum threshold is boosted ."""

implementation:
if self . _globalInhibition or self . _inhibitionRadius > self . _numInputs :
    self . _updateMinDutyCyclesGlobal ( )
else :
    self . _updateMinDutyCyclesLocal ( )


signature:
def _updateMinDutyCyclesGlobal ( self ) :
    """Updates the minimum duty cycles in a global fashion . Sets the minimum duty
 cycles for the overlap all columns to be a percent of the maximum in the
 region , specified by minPctOverlapDutyCycle . Functionality it is equivalent
 to _ updateMinDutyCyclesLocal , but this function exploits the globality of
 the computation to perform it in a straightforward , and efficient manner ."""

implementation: self . _minOverlapDutyCycles . fill ( self . _minPctOverlapDutyCycles * self . _overlapDutyCycles . max ( ) )


signature:
def _updateMinDutyCyclesLocal ( self ) :
    """Updates the minimum duty cycles . The minimum duty cycles are determined
 locally . Each column ' s minimum duty cycles are set to be a percent of the
 maximum duty cycles in the column ' s neighborhood . Unlike
 _ updateMinDutyCyclesGlobal , here the values can be quite different for
 different columns ."""

implementation:
for column in xrange ( self . _numColumns ) :
    neighborhood = self . _getColumnNeighborhood ( column )
    maxActiveDuty = self . _activeDutyCycles [ neighborhood ] . max ( )
    maxOverlapDuty = self . _overlapDutyCycles [ neighborhood ] . max ( )
    self . _minOverlapDutyCycles [ column ] = ( maxOverlapDuty * self . _minPctOverlapDutyCycles )


signature:
def _updateDutyCycles ( self , overlaps , activeColumns ) :
    """Updates the duty cycles for each column . The OVERLAP duty cycle is a moving
 average of the number of inputs which overlapped with the each column . The
 ACTIVITY duty cycles is a moving average of the frequency of activation for
 each column .
 Parameters :
 : param overlaps :
 An array containing the overlap score for each column .
 The overlap score for a column is defined as the number
 of synapses in a " connected state " ( connected synapses )
 that are connected to input bits which are turned on .
 : param activeColumns :
 An array containing the indices of the active columns ,
 the sparse set of columns which survived inhibition"""

implementation:
overlapArray = numpy . zeros ( self . _numColumns , dtype = realDType )
activeArray = numpy . zeros ( self . _numColumns , dtype = realDType )
overlapArray [ overlaps > 0 ] = 1
activeArray [ activeColumns ] = 1
period = self . _dutyCyclePeriod
if ( period > self . _iterationNum ) :
    period = self . _iterationNum
self . _overlapDutyCycles = self . _updateDutyCyclesHelper ( self . _overlapDutyCycles , overlapArray , period )
self . _activeDutyCycles = self . _updateDutyCyclesHelper ( self . _activeDutyCycles , activeArray , period )


signature:
def _updateInhibitionRadius ( self ) :
    """Update the inhibition radius . The inhibition radius is a measure of the
 square ( or hypersquare ) of columns that each a column is " connected to "
 on average . Since columns are are not connected to each other directly , we
 determine this quantity by first figuring out how many * inputs * a column is
 connected to , and then multiplying it by the total number of columns that
 exist for each input . For multiple dimension the aforementioned
 calculations are averaged over all dimensions of inputs and columns . This
 value is meaningless if global inhibition is enabled ."""

implementation:
if self . _globalInhibition :
    self . _inhibitionRadius = int ( self . _columnDimensions . max ( ) )
    return
avgConnectedSpan = numpy . average ( [ self . _avgConnectedSpanForColumnND ( i ) for i in xrange ( self . _numColumns ) ] )
columnsPerInput = self . _avgColumnsPerInput ( )
diameter = avgConnectedSpan * columnsPerInput
radius = ( diameter - 1 ) / 2.0
radius = max ( 1.0 , radius )
self . _inhibitionRadius = int ( radius + 0.5 )


signature:
def _avgColumnsPerInput ( self ) :
    """The average number of columns per input , taking into account the topology
 of the inputs and columns . This value is used to calculate the inhibition
 radius . This function supports an arbitrary number of dimensions . If the
 number of column dimensions does not match the number of input dimensions ,
 we treat the missing , or phantom dimensions as ' ones ' ."""

implementation:
# TODO : extend to support different number of dimensions for inputs and
# columns
numDim = max ( self . _columnDimensions . size , self . _inputDimensions . size )
colDim = numpy . ones ( numDim )
colDim [ : self . _columnDimensions . size ] = self . _columnDimensions
inputDim = numpy . ones ( numDim )
inputDim [ : self . _inputDimensions . size ] = self . _inputDimensions
columnsPerInput = colDim . astype ( realDType ) / inputDim
return numpy . average ( columnsPerInput )


signature:
def _avgConnectedSpanForColumn1D ( self , columnIndex ) :
    """The range of connected synapses for column . This is used to
 calculate the inhibition radius . This variation of the function only
 supports a 1 dimensional column topology .
 Parameters :
 : param columnIndex : The index identifying a column in the permanence ,
 potential and connectivity matrices"""

implementation:
assert ( self . _inputDimensions . size == 1 )
connected = self . _connectedSynapses [ columnIndex ] . nonzero ( ) [ 0 ]
if connected . size == 0 :
    return 0
else :
    return max ( connected ) - min ( connected ) + 1


signature:
def _avgConnectedSpanForColumn2D ( self , columnIndex ) :
    """The range of connectedSynapses per column , averaged for each dimension .
 This value is used to calculate the inhibition radius . This variation of
 the function only supports a 2 dimensional column topology .
 Parameters :
 : param columnIndex : The index identifying a column in the permanence ,
 potential and connectivity matrices"""

implementation:
assert ( self . _inputDimensions . size == 2 )
connected = self . _connectedSynapses [ columnIndex ]
( rows , cols ) = connected . reshape ( self . _inputDimensions ) . nonzero ( )
if rows . size == 0 and cols . size == 0 :
    return 0
rowSpan = rows . max ( ) - rows . min ( ) + 1
colSpan = cols . max ( ) - cols . min ( ) + 1
return numpy . average ( [ rowSpan , colSpan ] )


signature:
def _bumpUpWeakColumns ( self ) :
    """This method increases the permanence values of synapses of columns whose
 activity level has been too low . Such columns are identified by having an
 overlap duty cycle that drops too much below those of their peers . The
 permanence values for such columns are increased ."""

implementation:
weakColumns = numpy . where ( self . _overlapDutyCycles < self . _minOverlapDutyCycles ) [ 0 ]
for columnIndex in weakColumns :
    perm = self . _permanences [ columnIndex ] . astype ( realDType )
    maskPotential = numpy . where ( self . _potentialPools [ columnIndex ] > 0 ) [ 0 ]
    perm [ maskPotential ] += self . _synPermBelowStimulusInc
    self . _updatePermanencesForColumn ( perm , columnIndex , raisePerm = False )


signature:
def _raisePermanenceToThreshold ( self , perm , mask ) :
    """This method ensures that each column has enough connections to input bits
 to allow it to become active . Since a column must have at least
 ' self . _ stimulusThreshold ' overlaps in order to be considered during the
 inhibition phase , columns without such minimal number of connections , even
 if all the input bits they are connected to turn on , have no chance of
 obtaining the minimum threshold . For such columns , the permanence values
 are increased until the minimum number of connections are formed .
 Parameters :
 : param perm : An array of permanence values for a column . The array is
 " dense " , i . e . it contains an entry for each input bit , even
 if the permanence value is 0.
 : param mask : the indices of the columns whose permanences need to be
 raised ."""

implementation:
if len ( mask ) < self . _stimulusThreshold :
    raise Exception ( "This is likely due to a " + "value of stimulusThreshold that is too large relative " + "to the input size. [len(mask) < self._stimulusThreshold]" )
numpy . clip ( perm , self . _synPermMin , self . _synPermMax , out = perm )
while True :
    numConnected = numpy . nonzero ( perm > self . _synPermConnected - PERMANENCE_EPSILON ) [ 0 ] . size
    if numConnected >= self . _stimulusThreshold :
        return
    perm [ mask ] += self . _synPermBelowStimulusInc


signature:
def _updatePermanencesForColumn ( self , perm , columnIndex , raisePerm = True ) :
    """This method updates the permanence matrix with a column ' s new permanence
 values . The column is identified by its index , which reflects the row in
 the matrix , and the permanence is given in ' dense ' form , i . e . a full
 array containing all the zeros as well as the non - zero values . It is in
 charge of implementing ' clipping ' - ensuring that the permanence values are
 always between 0 and 1 - and ' trimming ' - enforcing sparsity by zeroing out
 all permanence values below ' _ synPermTrimThreshold ' . It also maintains
 the consistency between ' self . _ permanences ' ( the matrix storing the
 permanence values ) , ' self . _ connectedSynapses ' , ( the matrix storing the bits
 each column is connected to ) , and ' self . _ connectedCounts ' ( an array storing
 the number of input bits each column is connected to ) . Every method wishing
 to modify the permanence matrix should do so through this method .
 Parameters :
 : param perm : An array of permanence values for a column . The array is
 " dense " , i . e . it contains an entry for each input bit , even
 if the permanence value is 0.
 : param index : The index identifying a column in the permanence , potential
 and connectivity matrices
 : param raisePerm : A boolean value indicating whether the permanence values
 should be raised until a minimum number are synapses are in
 a connected state . Should be set to ' false ' when a direct
 assignment is required ."""

implementation:
maskPotential = numpy . where ( self . _potentialPools [ columnIndex ] > 0 ) [ 0 ]
if raisePerm :
    self . _raisePermanenceToThreshold ( perm , maskPotential )
perm [ perm < self . _synPermTrimThreshold ] = 0
numpy . clip ( perm , self . _synPermMin , self . _synPermMax , out = perm )
newConnected = numpy . where ( perm >= self . _synPermConnected - PERMANENCE_EPSILON ) [ 0 ]
self . _permanences . update ( columnIndex , perm )
self . _connectedSynapses . replace ( columnIndex , newConnected )
self . _connectedCounts [ columnIndex ] = newConnected . size


signature:
def _initPermConnected ( self ) :
    """Returns a randomly generated permanence value for a synapses that is
 initialized in a connected state . The basic idea here is to initialize
 permanence values very close to synPermConnected so that a small number of
 learning steps could make it disconnected or connected .
 Note : experimentation was done a long time ago on the best way to initialize
 permanence values , but the history for this particular scheme has been lost ."""

implementation:
p = self . _synPermConnected + ( self . _synPermMax - self . _synPermConnected ) * self . _random . getReal64 ( )
# Ensure we don ' t have too much unnecessary precision . A full 64 bits of
# precision causes numerical stability issues across platforms and across
# implementations
p = int ( p * 100000 ) / 100000.0
return p


signature:
def _initPermNonConnected ( self ) :
    """Returns a randomly generated permanence value for a synapses that is to be
 initialized in a non - connected state ."""

implementation:
p = self . _synPermConnected * self . _random . getReal64 ( )
# Ensure we don ' t have too much unnecessary precision . A full 64 bits of
# precision causes numerical stability issues across platforms and across
# implementations
p = int ( p * 100000 ) / 100000.0
return p


signature:
def _initPermanence ( self , potential , connectedPct ) :
    """Initializes the permanences of a column . The method
 returns a 1 - D array the size of the input , where each entry in the
 array represents the initial permanence value between the input bit
 at the particular index in the array , and the column represented by
 the ' index ' parameter .
 Parameters :
 : param potential : A numpy array specifying the potential pool of the column .
 Permanence values will only be generated for input bits
 corresponding to indices for which the mask value is 1.
 : param connectedPct : A value between 0 or 1 governing the chance , for each
 permanence , that the initial permanence value will
 be a value that is considered connected ."""

implementation:
# Determine which inputs bits will start out as connected
# to the inputs . Initially a subset of the input bits in a
# column ' s potential pool will be connected . This number is
# given by the parameter " connectedPct "
perm = numpy . zeros ( self . _numInputs , dtype = realDType )
for i in xrange ( self . _numInputs ) :
    if ( potential [ i ] < 1 ) :
        continue
    if ( self . _random . getReal64 ( ) <= connectedPct ) :
        perm [ i ] = self . _initPermConnected ( )
    else :
        perm [ i ] = self . _initPermNonConnected ( )
# Clip off low values . Since we use a sparse representation
# to store the permanence values this helps reduce memory
# requirements .
perm [ perm < self . _synPermTrimThreshold ] = 0
return perm


signature:
def _mapColumn ( self , index ) :
    """Maps a column to its respective input index , keeping to the topology of
 the region . It takes the index of the column as an argument and determines
 what is the index of the flattened input vector that is to be the center of
 the column ' s potential pool . It distributes the columns over the inputs
 uniformly . The return value is an integer representing the index of the
 input bit . Examples of the expected output of this method :
 * If the topology is one dimensional , and the column index is 0 , this
 method will return the input index 0 . If the column index is 1 , and there
 are 3 columns over 7 inputs , this method will return the input index 3.
 * If the topology is two dimensional , with column dimensions [ 3 , 5 ] and
 input dimensions [ 7 , 11 ] , and the column index is 3 , the method
 returns input index 8.
 Parameters :
 : param index : The index identifying a column in the permanence , potential
 and connectivity matrices .
 : param wrapAround : A boolean value indicating that boundaries should be
 ignored ."""

implementation:
columnCoords = numpy . unravel_index ( index , self . _columnDimensions )
columnCoords = numpy . array ( columnCoords , dtype = realDType )
ratios = columnCoords / self . _columnDimensions
inputCoords = self . _inputDimensions * ratios
inputCoords += 0.5 * self . _inputDimensions / self . _columnDimensions
inputCoords = inputCoords . astype ( int )
inputIndex = numpy . ravel_multi_index ( inputCoords , self . _inputDimensions )
return inputIndex


signature:
def _mapPotential ( self , index ) :
    """Maps a column to its input bits . This method encapsulates the topology of
 the region . It takes the index of the column as an argument and determines
 what are the indices of the input vector that are located within the
 column ' s potential pool . The return value is a list containing the indices
 of the input bits . The current implementation of the base class only
 supports a 1 dimensional topology of columns with a 1 dimensional topology
 of inputs . To extend this class to support 2 - D topology you will need to
 override this method . Examples of the expected output of this method :
 * If the potentialRadius is greater than or equal to the largest input
 dimension then each column connects to all of the inputs .
 * If the topology is one dimensional , the input space is divided up evenly
 among the columns and each column is centered over its share of the
 inputs . If the potentialRadius is 5 , then each column connects to the
 input it is centered above as well as the 5 inputs to the left of that
 input and the five inputs to the right of that input , wrapping around if
 wrapAround = True .
 * If the topology is two dimensional , the input space is again divided up
 evenly among the columns and each column is centered above its share of
 the inputs . If the potentialRadius is 5 , the column connects to a square
 that has 11 inputs on a side and is centered on the input that the column
 is centered above .
 Parameters :
 : param index : The index identifying a column in the permanence , potential
 and connectivity matrices ."""

implementation:
centerInput = self . _mapColumn ( index )
columnInputs = self . _getInputNeighborhood ( centerInput ) . astype ( uintType )
# Select a subset of the receptive field to serve as the
# the potential pool
numPotential = int ( columnInputs . size * self . _potentialPct + 0.5 )
selectedInputs = numpy . empty ( numPotential , dtype = uintType )
self . _random . sample ( columnInputs , selectedInputs )
potential = numpy . zeros ( self . _numInputs , dtype = uintType )
potential [ selectedInputs ] = 1
return potential


signature:
def _updateBoostFactorsGlobal ( self ) :
    """Update boost factors when global inhibition is used"""

implementation:
# When global inhibition is enabled , the target activation level is
# the sparsity of the spatial pooler
if ( self . _localAreaDensity > 0 ) :
    targetDensity = self . _localAreaDensity
else :
    inhibitionArea = ( ( 2 * self . _inhibitionRadius + 1 ) ** self . _columnDimensions . size )
    inhibitionArea = min ( self . _numColumns , inhibitionArea )
    targetDensity = float ( self . _numActiveColumnsPerInhArea ) / inhibitionArea
    targetDensity = min ( targetDensity , 0.5 )
self . _boostFactors = numpy . exp ( ( targetDensity - self . _activeDutyCycles ) * self . _boostStrength )


signature:
def _updateBoostFactorsLocal ( self ) :
    """Update boost factors when local inhibition is used"""

implementation:
# Determine the target activation level for each column
# The targetDensity is the average activeDutyCycles of the neighboring
# columns of each column .
targetDensity = numpy . zeros ( self . _numColumns , dtype = realDType )
for i in xrange ( self . _numColumns ) :
    maskNeighbors = self . _getColumnNeighborhood ( i )
    targetDensity [ i ] = numpy . mean ( self . _activeDutyCycles [ maskNeighbors ] )
self . _boostFactors = numpy . exp ( ( targetDensity - self . _activeDutyCycles ) * self . _boostStrength )


signature:
def _calculateOverlap ( self , inputVector ) :
    """This function determines each column ' s overlap with the current input
 vector . The overlap of a column is the number of synapses for that column
 that are connected ( permanence value is greater than ' _ synPermConnected ' )
 to input bits which are turned on . The implementation takes advantage of
 the SparseBinaryMatrix class to perform this calculation efficiently .
 Parameters :
 : param inputVector : a numpy array of 0 ' s and 1 ' s that comprises the input to
 the spatial pooler ."""

implementation:
overlaps = numpy . zeros ( self . _numColumns , dtype = realDType )
self . _connectedSynapses . rightVecSumAtNZ_fast ( inputVector . astype ( realDType ) , overlaps )
return overlaps


signature:
def _inhibitColumns ( self , overlaps ) :
    """Performs inhibition . This method calculates the necessary values needed to
 actually perform inhibition and then delegates the task of picking the
 active columns to helper functions .
 Parameters :
 : param overlaps : an array containing the overlap score for each column .
 The overlap score for a column is defined as the number
 of synapses in a " connected state " ( connected synapses )
 that are connected to input bits which are turned on ."""

implementation:
# determine how many columns should be selected in the inhibition phase .
# This can be specified by either setting the ' numActiveColumnsPerInhArea '
# parameter or the ' localAreaDensity ' parameter when initializing the class
if ( self . _localAreaDensity > 0 ) :
    density = self . _localAreaDensity
else :
    inhibitionArea = ( ( 2 * self . _inhibitionRadius + 1 ) ** self . _columnDimensions . size )
    inhibitionArea = min ( self . _numColumns , inhibitionArea )
    density = float ( self . _numActiveColumnsPerInhArea ) / inhibitionArea
    density = min ( density , 0.5 )
if self . _globalInhibition or self . _inhibitionRadius > max ( self . _columnDimensions ) :
    return self . _inhibitColumnsGlobal ( overlaps , density )
else :
    return self . _inhibitColumnsLocal ( overlaps , density )


signature:
def _inhibitColumnsGlobal ( self , overlaps , density ) :
    """Perform global inhibition . Performing global inhibition entails picking the
 top ' numActive ' columns with the highest overlap score in the entire
 region . At most half of the columns in a local neighborhood are allowed to
 be active . Columns with an overlap score below the ' stimulusThreshold ' are
 always inhibited .
 : param overlaps : an array containing the overlap score for each column .
 The overlap score for a column is defined as the number
 of synapses in a " connected state " ( connected synapses )
 that are connected to input bits which are turned on .
 : param density : The fraction of columns to survive inhibition .
 @ return list with indices of the winning columns"""

implementation:
# calculate num active per inhibition area
numActive = int ( density * self . _numColumns )
# Calculate winners using stable sort algorithm ( mergesort )
# for compatibility with C + +
sortedWinnerIndices = numpy . argsort ( overlaps , kind = 'mergesort' )
# Enforce the stimulus threshold
start = len ( sortedWinnerIndices ) - numActive
while start < len ( sortedWinnerIndices ) :
    i = sortedWinnerIndices [ start ]
    if overlaps [ i ] >= self . _stimulusThreshold :
        break
    else :
        start += 1
return sortedWinnerIndices [ start : ] [ : : - 1 ]