signature:
def create_query ( section ) :
    """Creates a search query based on the section of the config file ."""

implementation:
query = { }
if 'ports' in section :
    query [ 'ports' ] = [ section [ 'ports' ] ]
if 'up' in section :
    query [ 'up' ] = bool ( section [ 'up' ] )
if 'search' in section :
    query [ 'search' ] = [ section [ 'search' ] ]
if 'tags' in section :
    query [ 'tags' ] = [ section [ 'tags' ] ]
if 'groups' in section :
    query [ 'groups' ] = [ section [ 'groups' ] ]
return query


signature:
def create_pipe_workers ( configfile , directory ) :
    """Creates the workers based on the given configfile to provide named pipes in the directory ."""

implementation:
type_map = { 'service' : ServiceSearch , 'host' : HostSearch , 'range' : RangeSearch , 'user' : UserSearch }
config = configparser . ConfigParser ( )
config . read ( configfile )
if not len ( config . sections ( ) ) :
    print_error ( "No named pipes configured" )
    return
print_notification ( "Starting {} pipes in directory {}" . format ( len ( config . sections ( ) ) , directory ) )
workers = [ ]
for name in config . sections ( ) :
    section = config [ name ]
    query = create_query ( section )
    object_type = type_map [ section [ 'type' ] ]
    args = ( name , os . path . join ( directory , name ) , object_type , query , section [ 'format' ] , bool ( section . get ( 'unique' , 0 ) ) )
    workers . append ( multiprocessing . Process ( target = pipe_worker , args = args ) )
return workers


signature:
def main ( ) :
    """Loads the config and handles the workers ."""

implementation:
config = Config ( )
pipes_dir = config . get ( 'pipes' , 'directory' )
pipes_config = config . get ( 'pipes' , 'config_file' )
pipes_config_path = os . path . join ( config . config_dir , pipes_config )
if not os . path . exists ( pipes_config_path ) :
    print_error ( "Please configure the named pipes first" )
    return
workers = create_pipe_workers ( pipes_config_path , pipes_dir )
if workers :
    for worker in workers :
        worker . start ( )
    try :
        for worker in workers :
            worker . join ( )
    except KeyboardInterrupt :
        print_notification ( "Shutting down" )
        for worker in workers :
            worker . terminate ( )
            worker . join ( )


signature:
def f_i18n_iso ( isocode , lang = "eng" ) :
    """Replace isocode by its language equivalent
 : param isocode : Three character long language code
 : param lang : Lang in which to return the language name
 : return : Full Text Language Name"""

implementation:
if lang not in flask_nemo . _data . AVAILABLE_TRANSLATIONS :
    lang = "eng"
try :
    return flask_nemo . _data . ISOCODES [ isocode ] [ lang ]
except KeyError :
    return "Unknown"


signature:
def f_hierarchical_passages ( reffs , citation ) :
    """A function to construct a hierarchical dictionary representing the different citation layers of a text
 : param reffs : passage references with human - readable equivalent
 : type reffs : [ ( str , str ) ]
 : param citation : Main Citation
 : type citation : Citation
 : return : nested dictionary representing where keys represent the names of the levels and the final values represent the passage reference
 : rtype : OrderedDict"""

implementation:
d = OrderedDict ( )
levels = [ x for x in citation ]
for cit , name in reffs :
    ref = cit . split ( '-' ) [ 0 ]
    levs = [ '%{}|{}%' . format ( levels [ i ] . name , v ) for i , v in enumerate ( ref . split ( '.' ) ) ]
    getFromDict ( d , levs [ : - 1 ] ) [ name ] = cit
return d


signature:
def f_i18n_citation_type ( string , lang = "eng" ) :
    """Take a string of form % citation _ type | passage % and format it for human
 : param string : String of formation % citation _ type | passage %
 : param lang : Language to translate to
 : return : Human Readable string
 . . note : : To Do : Use i18n tools and provide real i18n"""

implementation:
s = " " . join ( string . strip ( "%" ) . split ( "|" ) )
return s . capitalize ( )


signature:
def f_annotation_filter ( annotations , type_uri , number ) :
    """Annotation filtering filter
 : param annotations : List of annotations
 : type annotations : [ AnnotationResource ]
 : param type _ uri : URI Type on which to filter
 : type type _ uri : str
 : param number : Number of the annotation to return
 : type number : int
 : return : Annotation ( s ) matching the request
 : rtype : [ AnnotationResource ] or AnnotationResource"""

implementation:
filtered = [ annotation for annotation in annotations if annotation . type_uri == type_uri ]
number = min ( [ len ( filtered ) , number ] )
if number == 0 :
    return None
else :
    return filtered [ number - 1 ]


signature:
def scan ( self , store ) :
    '''Scans the local files for changes ( either additions , modifications or
 deletions ) and reports them to the ` store ` object , which is expected to
 implement the : class : ` pysyncml . Store ` interface .'''

implementation:
# steps :
# 1 ) generate a table of all store files , with filename ,
# inode , checksum
# 2 ) generate a table of all current files , with filename ,
# inode , checksum
# 3 ) iterate over all stored values and find matches , delete
# them from the " not - matched - yet " table , and record the
# change
# TODO : if this engine is running as the client , i think the best
# strategy is to delete all pending changes before starting
# the scan process . that way , any left - over gunk from a
# previous sync that did not terminate well is cleaned up . . .
# TODO : this algorithm , although better than the last , has the
# inconvenient downside of being very memory - hungry : it
# assumes that the entire list of notes ( with sha256
# checksums - not the entire body ) fits in memory . although
# it is not a ridiculous assumption ( these are " notes " after
# all . . . ) , it would be nice if it did not rely on that .
# todo : by tracking inode ' s , this * could * also potentially reduce
# some " del / add " operations with a single " mod "
# todo : should this make use of lastmod timestamps ? . . . that may
# help to reduce the number of checksums calculated and the
# number of entries loaded into memory . . .
if self . ignoreRoot is None :
    self . ignoreRoot = re . compile ( '^(%s)$' % ( re . escape ( self . engine . syncSubdir ) , ) )
dbnotes = list ( self . engine . model . NoteItem . q ( ) )
dbnames = dict ( ( e . name , e ) for e in dbnotes )
fsnotes = list ( self . _scandir ( '.' ) )
fsnames = dict ( ( e . name , e ) for e in fsnotes )
# first pass : eliminate all entries with matching filenames & checksum
for fsent in fsnames . values ( ) :
    if fsent . name in dbnames and dbnames [ fsent . name ] . sha256 == fsent . sha256 :
        log . debug ( 'entry "%s" not modified' , fsent . name )
        # todo : update db inode and lastmod if needed . . .
        del dbnames [ fsent . name ]
        del fsnames [ fsent . name ]
# second pass : find entries that were moved to override another entry
dbskip = [ ]
for dbent in dbnames . values ( ) :
    if dbent . id in dbskip or dbent . name in fsnames :
        continue
    for fsent in fsnames . values ( ) :
        if fsent . sha256 != dbent . sha256 or fsent . name not in dbnames :
            continue
        log . debug ( 'entry "%s" deleted and replaced by "%s"' , fsent . name , dbent . name )
        dbother = dbnames [ fsent . name ]
        del dbnames [ dbent . name ]
        del dbnames [ fsent . name ]
        del fsnames [ fsent . name ]
        dbskip . append ( dbother . id )
        store . registerChange ( dbent . id , pysyncml . ITEM_DELETED )
        for key , val in fsent . items ( ) :
            setattr ( dbother , key , val )
        # the digest didn ' t change , so this is just a filename change . . .
        if self . engine . options . syncFilename :
            store . registerChange ( dbother . id , pysyncml . ITEM_MODIFIED )
        break
# third pass : find entries that were renamed
dbskip = [ ]
for dbent in dbnames . values ( ) :
    if dbent . id in dbskip :
        continue
    for fsent in fsnames . values ( ) :
        if fsent . sha256 != dbent . sha256 :
            continue
        log . debug ( 'entry "%s" renamed to "%s"' , dbent . name , fsent . name )
        del dbnames [ dbent . name ]
        del fsnames [ fsent . name ]
        for key , val in fsent . items ( ) :
            setattr ( dbent , key , val )
        # the digest didn ' t change , so this is just a filename change . . .
        if self . engine . options . syncFilename :
            store . registerChange ( dbent . id , pysyncml . ITEM_MODIFIED )
        break
# fourth pass : find new and modified entries
for fsent in fsnames . values ( ) :
    if fsent . name in dbnames :
        log . debug ( 'entry "%s" modified' , fsent . name )
        dbent = dbnames [ fsent . name ]
        del dbnames [ fsent . name ]
        store . registerChange ( dbent . id , pysyncml . ITEM_MODIFIED )
    else :
        log . debug ( 'entry "%s" added' , fsent . name )
        dbent = self . engine . model . NoteItem ( )
        self . engine . dbsession . add ( dbent )
        store . registerChange ( dbent . id , pysyncml . ITEM_ADDED )
    for key , val in fsent . items ( ) :
        setattr ( dbent , key , val )
    del fsnames [ fsent . name ]
# fifth pass : find deleted entries
for dbent in dbnames . values ( ) :
    store . registerChange ( dbent . id , pysyncml . ITEM_DELETED )
    self . engine . dbsession . add ( dbent )


signature:
def check_service ( service ) :
    """Connect to a service to see if it is a http or https server ."""

implementation:
# Try HTTP
service . add_tag ( 'header_scan' )
http = False
try :
    result = requests . head ( 'http://{}:{}' . format ( service . address , service . port ) , timeout = 1 )
    print_success ( "Found http service on {}:{}" . format ( service . address , service . port ) )
    service . add_tag ( 'http' )
    http = True
    try :
        service . banner = result . headers [ 'Server' ]
    except KeyError :
        pass
except ( ConnectionError , ConnectTimeout , ReadTimeout , Error ) :
    pass
if not http : # Try HTTPS
    try :
        result = requests . head ( 'https://{}:{}' . format ( service . address , service . port ) , verify = False , timeout = 3 )
        service . add_tag ( 'https' )
        print_success ( "Found https service on {}:{}" . format ( service . address , service . port ) )
        try :
            service . banner = result . headers [ 'Server' ]
        except KeyError :
            pass
    except ( ConnectionError , ConnectTimeout , ReadTimeout , Error ) :
        pass
service . save ( )


signature:
def main ( ) :
    """Retrieves services starts check _ service in a gevent pool of 100."""

implementation:
search = ServiceSearch ( )
services = search . get_services ( up = True , tags = [ '!header_scan' ] )
print_notification ( "Scanning {} services" . format ( len ( services ) ) )
# Disable the insecure request warning
urllib3 . disable_warnings ( urllib3 . exceptions . InsecureRequestWarning )
pool = Pool ( 100 )
count = 0
for service in services :
    count += 1
    if count % 50 == 0 :
        print_notification ( "Checking {}/{} services" . format ( count , len ( services ) ) )
    pool . spawn ( check_service , service )
pool . join ( )
print_notification ( "Completed, 'http' tag added to services that respond to http, 'https' tag added to services that respond to https." )


signature:
def import_nmap ( result , tag , check_function = all_hosts , import_services = False ) :
    """Imports the given nmap result ."""

implementation:
host_search = HostSearch ( arguments = False )
service_search = ServiceSearch ( )
parser = NmapParser ( )
report = parser . parse_fromstring ( result )
imported_hosts = 0
imported_services = 0
for nmap_host in report . hosts :
    if check_function ( nmap_host ) :
        imported_hosts += 1
        host = host_search . id_to_object ( nmap_host . address )
        host . status = nmap_host . status
        host . add_tag ( tag )
        if nmap_host . os_fingerprinted :
            host . os = nmap_host . os_fingerprint
        if nmap_host . hostnames :
            host . hostname . extend ( nmap_host . hostnames )
        if import_services :
            for service in nmap_host . services :
                imported_services += 1
                serv = Service ( ** service . get_dict ( ) )
                serv . address = nmap_host . address
                service_id = service_search . object_to_id ( serv )
                if service_id : # Existing object , save the banner and script results .
                    serv_old = Service . get ( service_id )
                    if service . banner :
                        serv_old . banner = service . banner
                    # TODO implement
                    # if service . script _ results :
                    # serv _ old . script _ results . extend ( service . script _ results )
                    serv_old . save ( )
                else : # New object
                    serv . address = nmap_host . address
                    serv . save ( )
                if service . state == 'open' :
                    host . open_ports . append ( service . port )
                if service . state == 'closed' :
                    host . closed_ports . append ( service . port )
                if service . state == 'filtered' :
                    host . filtered_ports . append ( service . port )
        host . save ( )
if imported_hosts :
    print_success ( "Imported {} hosts, with tag {}" . format ( imported_hosts , tag ) )
else :
    print_error ( "No hosts found" )
return { 'hosts' : imported_hosts , 'services' : imported_services }


signature:
def nmap ( nmap_args , ips ) :
    """Start an nmap process with the given args on the given ips ."""

implementation:
config = Config ( )
arguments = [ 'nmap' , '-Pn' ]
arguments . extend ( ips )
arguments . extend ( nmap_args )
output_file = ''
now = datetime . datetime . now ( )
if not '-oA' in nmap_args :
    output_name = 'nmap_jackal_{}' . format ( now . strftime ( "%Y-%m-%d %H:%M" ) )
    path_name = os . path . join ( config . get ( 'nmap' , 'directory' ) , output_name )
    print_notification ( "Writing output of nmap to {}" . format ( path_name ) )
    if not os . path . exists ( config . get ( 'nmap' , 'directory' ) ) :
        os . makedirs ( config . get ( 'nmap' , 'directory' ) )
    output_file = path_name + '.xml'
    arguments . extend ( [ '-oA' , path_name ] )
else :
    output_file = nmap_args [ nmap_args . index ( '-oA' ) + 1 ] + '.xml'
print_notification ( "Starting nmap" )
subprocess . call ( arguments )
with open ( output_file , 'r' ) as f :
    return f . read ( )


signature:
def nmap_discover ( ) :
    """This function retrieves ranges from jackal
 Uses two functions of nmap to find hosts :
 ping : icmp / arp pinging of targets
 lookup : reverse dns lookup"""

implementation:
rs = RangeSearch ( )
rs_parser = rs . argparser
arg = argparse . ArgumentParser ( parents = [ rs_parser ] , conflict_handler = 'resolve' )
arg . add_argument ( 'type' , metavar = 'type' , help = 'The type of nmap scan to do, choose from ping or lookup' , type = str , choices = [ 'ping' , 'lookup' ] )
arguments , nmap_args = arg . parse_known_args ( )
tag = None
if arguments . type == 'ping' :
    tag = 'nmap_ping'
    nmap_args . append ( '-sn' )
    nmap_args . append ( '-n' )
    check_function = include_up_hosts
elif arguments . type == 'lookup' :
    tag = 'nmap_lookup'
    nmap_args . append ( '-sL' )
    check_function = include_hostnames
ranges = rs . get_ranges ( tags = [ '!{}' . format ( tag ) ] )
ranges = [ r for r in ranges ]
ips = [ ]
for r in ranges :
    ips . append ( r . range )
print_notification ( "Running nmap with args: {} on {} range(s)" . format ( nmap_args , len ( ips ) ) )
result = nmap ( nmap_args , ips )
stats = import_nmap ( result , tag , check_function )
stats [ 'scanned_ranges' ] = len ( ips )
Logger ( ) . log ( 'nmap_discover' , "Nmap discover with args: {} on {} range(s)" . format ( nmap_args , len ( ips ) ) , stats )
for r in ranges :
    r . add_tag ( tag )
    r . save ( )


signature:
def nmap_scan ( ) :
    """Scans the given hosts with nmap ."""

implementation:
# Create the search and config objects
hs = HostSearch ( )
config = Config ( )
# Static options to be able to figure out what options to use depending on the input the user gives .
nmap_types = [ 'top10' , 'top100' , 'custom' , 'top1000' , 'all' ]
options = { 'top10' : '--top-ports 10' , 'top100' : '--top-ports 100' , 'custom' : config . get ( 'nmap' , 'options' ) , 'top1000' : '--top-ports 1000' , 'all' : '-p-' }
# Create an argument parser
hs_parser = hs . argparser
argparser = argparse . ArgumentParser ( parents = [ hs_parser ] , conflict_handler = 'resolve' , description = "Scans hosts from the database using nmap, any arguments that are not in the help are passed to nmap" )
argparser . add_argument ( 'type' , metavar = 'type' , help = 'The number of ports to scan: top10, top100, custom, top1000 (default) or all' , type = str , choices = nmap_types , default = 'top1000' , const = 'top1000' , nargs = '?' )
arguments , extra_nmap_args = argparser . parse_known_args ( )
# Fix the tags for the search
tags = nmap_types [ nmap_types . index ( arguments . type ) : ]
tags = [ "!nmap_" + tag for tag in tags ]
hosts = hs . get_hosts ( tags = tags )
hosts = [ host for host in hosts ]
# Create the nmap arguments
nmap_args = [ ]
nmap_args . extend ( extra_nmap_args )
nmap_args . extend ( options [ arguments . type ] . split ( ' ' ) )
# Run nmap
print_notification ( "Running nmap with args: {} on {} hosts(s)" . format ( nmap_args , len ( hosts ) ) )
if len ( hosts ) :
    result = nmap ( nmap_args , [ str ( h . address ) for h in hosts ] )
    # Import the nmap result
    for host in hosts :
        host . add_tag ( "nmap_{}" . format ( arguments . type ) )
        host . save ( )
    print_notification ( "Nmap done, importing results" )
    stats = import_nmap ( result , "nmap_{}" . format ( arguments . type ) , check_function = all_hosts , import_services = True )
    stats [ 'scanned_hosts' ] = len ( hosts )
    stats [ 'type' ] = arguments . type
    Logger ( ) . log ( 'nmap_scan' , "Performed nmap {} scan on {} hosts" . format ( arguments . type , len ( hosts ) ) , stats )
else :
    print_notification ( "No hosts found" )


signature:
def nmap_smb_vulnscan ( ) :
    """Scans available smb services in the database for smb signing and ms17-010."""

implementation:
service_search = ServiceSearch ( )
services = service_search . get_services ( ports = [ '445' ] , tags = [ '!smb_vulnscan' ] , up = True )
services = [ service for service in services ]
service_dict = { }
for service in services :
    service . add_tag ( 'smb_vulnscan' )
    service_dict [ str ( service . address ) ] = service
nmap_args = "-Pn -n --disable-arp-ping --script smb-security-mode.nse,smb-vuln-ms17-010.nse -p 445" . split ( " " )
if services :
    result = nmap ( nmap_args , [ str ( s . address ) for s in services ] )
    parser = NmapParser ( )
    report = parser . parse_fromstring ( result )
    smb_signing = 0
    ms17 = 0
    for nmap_host in report . hosts :
        for script_result in nmap_host . scripts_results :
            script_result = script_result . get ( 'elements' , { } )
            service = service_dict [ str ( nmap_host . address ) ]
            if script_result . get ( 'message_signing' , '' ) == 'disabled' :
                print_success ( "({}) SMB Signing disabled" . format ( nmap_host . address ) )
                service . add_tag ( 'smb_signing_disabled' )
                smb_signing += 1
            if script_result . get ( 'CVE-2017-0143' , { } ) . get ( 'state' , '' ) == 'VULNERABLE' :
                print_success ( "({}) Vulnerable for MS17-010" . format ( nmap_host . address ) )
                service . add_tag ( 'MS17-010' )
                ms17 += 1
            service . update ( tags = service . tags )
    print_notification ( "Completed, 'smb_signing_disabled' tag added to systems with smb signing disabled, 'MS17-010' tag added to systems that did not apply MS17-010." )
    stats = { 'smb_signing' : smb_signing , 'MS17_010' : ms17 , 'scanned_services' : len ( services ) }
    Logger ( ) . log ( 'smb_vulnscan' , 'Scanned {} smb services for vulnerabilities' . format ( len ( services ) ) , stats )
else :
    print_notification ( "No services found to scan." )


signature:
def os_discovery ( ) :
    """Performs os ( and domain ) discovery of smb hosts ."""

implementation:
hs = HostSearch ( )
hosts = hs . get_hosts ( ports = [ 445 ] , tags = [ '!nmap_os' ] )
# TODO fix filter for emtpy fields .
hosts = [ host for host in hosts if not host . os ]
host_dict = { }
for host in hosts :
    host_dict [ str ( host . address ) ] = host
arguments = "--script smb-os-discovery.nse -p 445 -Pn -n --disable-arp-ping" . split ( ' ' )
if len ( hosts ) :
    count = 0
    print_notification ( "Checking OS of {} systems" . format ( len ( hosts ) ) )
    result = nmap ( arguments , [ str ( h . address ) for h in hosts ] )
    parser = NmapParser ( )
    report = parser . parse_fromstring ( result )
    for nmap_host in report . hosts :
        for script_result in nmap_host . scripts_results :
            script_result = script_result . get ( 'elements' , { } )
            host = host_dict [ str ( nmap_host . address ) ]
            if 'fqdn' in script_result :
                host . hostname . append ( script_result [ 'fqdn' ] )
            if 'os' in script_result :
                count += 1
                host . os = script_result [ 'os' ]
            host_dict [ str ( nmap_host . address ) ] = host
    for host in hosts :
        host . add_tag ( 'nmap_os' )
        host . save ( )
    print_notification ( "Done, found the os of {} systems" . format ( count ) )
else :
    print_notification ( "No systems found to be checked." )


signature:
def overview ( ) :
    """Function to create an overview of the services .
 Will print a list of ports found an the number of times the port was seen ."""

implementation:
search = Service . search ( )
search = search . filter ( "term" , state = 'open' )
search . aggs . bucket ( 'port_count' , 'terms' , field = 'port' , order = { '_count' : 'desc' } , size = 100 ) . metric ( 'unique_count' , 'cardinality' , field = 'address' )
response = search . execute ( )
print_line ( "Port     Count" )
print_line ( "---------------" )
for entry in response . aggregations . port_count . buckets :
    print_line ( "{0:<7}  {1}" . format ( entry . key , entry . unique_count . value ) )


signature:
def _plugin_endpoint_rename ( fn_name , instance ) :
    """Rename endpoint function name to avoid conflict when namespacing is set to true
 : param fn _ name : Name of the route function
 : param instance : Instance bound to the function
 : return : Name of the new namespaced function name"""

implementation:
if instance and instance . namespaced :
    fn_name = "r_{0}_{1}" . format ( instance . name , fn_name [ 2 : ] )
return fn_name


signature:
def get_locale ( self ) :
    """Retrieve the best matching locale using request headers
 . . note : : Probably one of the thing to enhance quickly .
 : rtype : str"""

implementation:
best_match = request . accept_languages . best_match ( [ 'de' , 'fr' , 'en' , 'la' ] )
if best_match is None :
    if len ( request . accept_languages ) > 0 :
        best_match = request . accept_languages [ 0 ] [ 0 ] [ : 2 ]
    else :
        return self . __default_lang__
lang = self . __default_lang__
if best_match == "de" :
    lang = "ger"
elif best_match == "fr" :
    lang = "fre"
elif best_match == "en" :
    lang = "eng"
elif best_match == "la" :
    lang = "lat"
return lang


signature:
def transform ( self , work , xml , objectId , subreference = None ) :
    """Transform input according to potentially registered XSLT
 . . note : : Since 1.0.0 , transform takes an objectId parameter which represent the passage which is called
 . . note : : Due to XSLT not being able to be used twice , we rexsltise the xml at every call of xslt
 . . warning : : Until a C libxslt error is fixed ( https : / / bugzilla . gnome . org / show _ bug . cgi ? id = 620102 ) , it is not possible to use strip tags in the xslt given to this application
 : param work : Work object containing metadata about the xml
 : type work : MyCapytains . resources . inventory . Text
 : param xml : XML to transform
 : type xml : etree . _ Element
 : param objectId : Object Identifier
 : type objectId : str
 : param subreference : Subreference
 : type subreference : str
 : return : String representation of transformed resource
 : rtype : str"""

implementation:
# We check first that we don ' t have
if str ( objectId ) in self . _transform :
    func = self . _transform [ str ( objectId ) ]
else :
    func = self . _transform [ "default" ]
# If we have a string , it means we get a XSL filepath
if isinstance ( func , str ) :
    with open ( func ) as f :
        xslt = etree . XSLT ( etree . parse ( f ) )
    return etree . tostring ( xslt ( xml ) , encoding = str , method = "html" , xml_declaration = None , pretty_print = False , with_tail = True , standalone = None )
# If we have a function , it means we return the result of the function
elif isinstance ( func , Callable ) :
    return func ( work , xml , objectId , subreference )
# If we have None , it means we just give back the xml
elif func is None :
    return etree . tostring ( xml , encoding = str )


signature:
def get_inventory ( self ) :
    """Request the api endpoint to retrieve information about the inventory
 : return : Main Collection
 : rtype : Collection"""

implementation:
if self . _inventory is not None :
    return self . _inventory
self . _inventory = self . resolver . getMetadata ( )
return self . _inventory


signature:
def get_reffs ( self , objectId , subreference = None , collection = None , export_collection = False ) :
    """Retrieve and transform a list of references .
 Returns the inventory collection object with its metadata and a callback function taking a level parameter and returning a list of strings .
 : param objectId : Collection Identifier
 : type objectId : str
 : param subreference : Subreference from which to retrieve children
 : type subreference : str
 : param collection : Collection object bearing metadata
 : type collection : Collection
 : param export _ collection : Return collection metadata
 : type export _ collection : bool
 : return : Returns either the list of references , or the text collection object with its references as tuple
 : rtype : ( Collection , [ str ] ) or [ str ]"""

implementation:
if collection is not None :
    text = collection
else :
    text = self . get_collection ( objectId )
reffs = self . chunk ( text , lambda level : self . resolver . getReffs ( objectId , level = level , subreference = subreference ) )
if export_collection is True :
    return text , reffs
return reffs


signature:
def get_passage ( self , objectId , subreference ) :
    """Retrieve the passage identified by the parameters
 : param objectId : Collection Identifier
 : type objectId : str
 : param subreference : Subreference of the passage
 : type subreference : str
 : return : An object bearing metadata and its text
 : rtype : InteractiveTextualNode"""

implementation:
passage = self . resolver . getTextualNode ( textId = objectId , subreference = subreference , metadata = True )
return passage


signature:
def get_siblings ( self , objectId , subreference , passage ) :
    """Get siblings of a browsed subreference
 . . note : : Since 1.0.0c , there is no more prevnext dict . Nemo uses the list of original chunked references to retrieve next and previous , or simply relies on the resolver to get siblings when the subreference is not found in given original chunks .
 : param objectId : Id of the object
 : param subreference : Subreference of the object
 : param passage : Current Passage
 : return : Previous and next references
 : rtype : ( str , str )"""

implementation:
reffs = [ reff for reff , _ in self . get_reffs ( objectId ) ]
if subreference in reffs :
    index = reffs . index ( subreference )
    # Not the first item and not the last one
    if 0 < index < len ( reffs ) - 1 :
        return reffs [ index - 1 ] , reffs [ index + 1 ]
    elif index == 0 and index < len ( reffs ) - 1 :
        return None , reffs [ 1 ]
    elif index > 0 and index == len ( reffs ) - 1 :
        return reffs [ index - 1 ] , None
    else :
        return None , None
else :
    return passage . siblingsId


signature:
def semantic ( self , collection , parent = None ) :
    """Generates a SEO friendly string for given collection
 : param collection : Collection object to generate string for
 : param parent : Current collection parent
 : return : SEO / URL Friendly string"""

implementation:
if parent is not None :
    collections = parent . parents [ : : - 1 ] + [ parent , collection ]
else :
    collections = collection . parents [ : : - 1 ] + [ collection ]
return filters . slugify ( "--" . join ( [ item . get_label ( ) for item in collections if item . get_label ( ) ] ) )


signature:
def make_coins ( self , collection , text , subreference = "" , lang = None ) :
    """Creates a CoINS Title string from information
 : param collection : Collection to create coins from
 : param text : Text / Passage object
 : param subreference : Subreference
 : param lang : Locale information
 : return : Coins HTML title value"""

implementation:
if lang is None :
    lang = self . __default_lang__
return "url_ver=Z39.88-2004" "&ctx_ver=Z39.88-2004" "&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook" "&rft_id={cid}" "&rft.genre=bookitem" "&rft.btitle={title}" "&rft.edition={edition}" "&rft.au={author}" "&rft.atitle={pages}" "&rft.language={language}" "&rft.pages={pages}" . format ( title = quote ( str ( text . get_title ( lang ) ) ) , author = quote ( str ( text . get_creator ( lang ) ) ) , cid = url_for ( ".r_collection" , objectId = collection . id , _external = True ) , language = collection . lang , pages = quote ( subreference ) , edition = quote ( str ( text . get_description ( lang ) ) ) )


signature:
def expose_ancestors_or_children ( self , member , collection , lang = None ) :
    """Build an ancestor or descendant dict view based on selected information
 : param member : Current Member to build for
 : param collection : Collection from which we retrieved it
 : param lang : Language to express data in
 : return :"""

implementation:
x = { "id" : member . id , "label" : str ( member . get_label ( lang ) ) , "model" : str ( member . model ) , "type" : str ( member . type ) , "size" : member . size , "semantic" : self . semantic ( member , parent = collection ) }
if isinstance ( member , ResourceCollection ) :
    x [ "lang" ] = str ( member . lang )
return x


signature:
def make_members ( self , collection , lang = None ) :
    """Build member list for given collection
 : param collection : Collection to build dict view of for its members
 : param lang : Language to express data in
 : return : List of basic objects"""

implementation:
objects = sorted ( [ self . expose_ancestors_or_children ( member , collection , lang = lang ) for member in collection . members if member . get_label ( ) ] , key = itemgetter ( "label" ) )
return objects


signature:
def make_parents ( self , collection , lang = None ) :
    """Build parents list for given collection
 : param collection : Collection to build dict view of for its members
 : param lang : Language to express data in
 : return : List of basic objects"""

implementation: return [ { "id" : member . id , "label" : str ( member . get_label ( lang ) ) , "model" : str ( member . model ) , "type" : str ( member . type ) , "size" : member . size } for member in collection . parents if member . get_label ( ) ]


signature:
def r_collections ( self , lang = None ) :
    """Retrieve the top collections of the inventory
 : param lang : Lang in which to express main data
 : type lang : str
 : return : Collections information and template
 : rtype : { str : Any }"""

implementation:
collection = self . resolver . getMetadata ( )
return { "template" : "main::collection.html" , "current_label" : collection . get_label ( lang ) , "collections" : { "members" : self . make_members ( collection , lang = lang ) } }


signature:
def r_collection ( self , objectId , lang = None ) :
    """Collection content browsing route function
 : param objectId : Collection identifier
 : type objectId : str
 : param lang : Lang in which to express main data
 : type lang : str
 : return : Template and collections contained in given collection
 : rtype : { str : Any }"""

implementation:
collection = self . resolver . getMetadata ( objectId )
return { "template" : "main::collection.html" , "collections" : { "current" : { "label" : str ( collection . get_label ( lang ) ) , "id" : collection . id , "model" : str ( collection . model ) , "type" : str ( collection . type ) , } , "members" : self . make_members ( collection , lang = lang ) , "parents" : self . make_parents ( collection , lang = lang ) } , }


signature:
def r_references ( self , objectId , lang = None ) :
    """Text exemplar references browsing route function
 : param objectId : Collection identifier
 : type objectId : str
 : param lang : Lang in which to express main data
 : type lang : str
 : return : Template and required information about text with its references"""

implementation:
collection , reffs = self . get_reffs ( objectId = objectId , export_collection = True )
return { "template" : "main::references.html" , "objectId" : objectId , "citation" : collection . citation , "collections" : { "current" : { "label" : collection . get_label ( lang ) , "id" : collection . id , "model" : str ( collection . model ) , "type" : str ( collection . type ) , } , "parents" : self . make_parents ( collection , lang = lang ) } , "reffs" : reffs }


signature:
def r_first_passage ( self , objectId ) :
    """Provides a redirect to the first passage of given objectId
 : param objectId : Collection identifier
 : type objectId : str
 : return : Redirection to the first passage of given text"""

implementation:
collection , reffs = self . get_reffs ( objectId = objectId , export_collection = True )
first , _ = reffs [ 0 ]
return redirect ( url_for ( ".r_passage_semantic" , objectId = objectId , subreference = first , semantic = self . semantic ( collection ) ) )


signature:
def r_passage ( self , objectId , subreference , lang = None ) :
    """Retrieve the text of the passage
 : param objectId : Collection identifier
 : type objectId : str
 : param lang : Lang in which to express main data
 : type lang : str
 : param subreference : Reference identifier
 : type subreference : str
 : return : Template , collections metadata and Markup object representing the text
 : rtype : { str : Any }"""

implementation:
collection = self . get_collection ( objectId )
if isinstance ( collection , CtsWorkMetadata ) :
    editions = [ t for t in collection . children . values ( ) if isinstance ( t , CtsEditionMetadata ) ]
    if len ( editions ) == 0 :
        raise UnknownCollection ( "This work has no default edition" )
    return redirect ( url_for ( ".r_passage" , objectId = str ( editions [ 0 ] . id ) , subreference = subreference ) )
text = self . get_passage ( objectId = objectId , subreference = subreference )
passage = self . transform ( text , text . export ( Mimetypes . PYTHON . ETREE ) , objectId )
prev , next = self . get_siblings ( objectId , subreference , text )
return { "template" : "main::text.html" , "objectId" : objectId , "subreference" : subreference , "collections" : { "current" : { "label" : collection . get_label ( lang ) , "id" : collection . id , "model" : str ( collection . model ) , "type" : str ( collection . type ) , "author" : text . get_creator ( lang ) , "title" : text . get_title ( lang ) , "description" : text . get_description ( lang ) , "citation" : collection . citation , "coins" : self . make_coins ( collection , text , subreference , lang = lang ) } , "parents" : self . make_parents ( collection , lang = lang ) } , "text_passage" : Markup ( passage ) , "prev" : prev , "next" : next }


signature:
def r_assets ( self , filetype , asset ) :
    """Route for specific assets .
 : param filetype : Asset Type
 : param asset : Filename of an asset
 : return : Response"""

implementation:
if filetype in self . assets and asset in self . assets [ filetype ] and self . assets [ filetype ] [ asset ] :
    return send_from_directory ( directory = self . assets [ filetype ] [ asset ] , filename = asset )
abort ( 404 )


signature:
def register_assets ( self ) :
    """Merge and register assets , both as routes and dictionary
 : return : None"""

implementation:
self . blueprint . add_url_rule ( # Register another path to ensure assets compatibility
"{0}.secondary/<filetype>/<asset>" . format ( self . static_url_path ) , view_func = self . r_assets , endpoint = "secondary_assets" , methods = [ "GET" ] )


signature:
def create_blueprint ( self ) :
    """Create blueprint and register rules
 : return : Blueprint of the current nemo app
 : rtype : flask . Blueprint"""

implementation:
self . register_plugins ( )
self . blueprint = Blueprint ( self . name , "nemo" , url_prefix = self . prefix , template_folder = self . template_folder , static_folder = self . static_folder , static_url_path = self . static_url_path )
for url , name , methods , instance in self . _urls :
    self . blueprint . add_url_rule ( url , view_func = self . view_maker ( name , instance ) , endpoint = _plugin_endpoint_rename ( name , instance ) , methods = methods )
for url , name , methods , instance in self . _semantic_url :
    self . blueprint . add_url_rule ( url , view_func = self . view_maker ( name , instance ) , endpoint = _plugin_endpoint_rename ( name , instance ) + "_semantic" , methods = methods )
self . register_assets ( )
self . register_filters ( )
# We extend the loading list by the instance value
self . __templates_namespaces__ . extend ( self . __instance_templates__ )
# We generate a template loader
for namespace , directory in self . __templates_namespaces__ [ : : - 1 ] :
    if namespace not in self . __template_loader__ :
        self . __template_loader__ [ namespace ] = [ ]
    self . __template_loader__ [ namespace ] . append ( jinja2 . FileSystemLoader ( op . abspath ( directory ) ) )
self . blueprint . jinja_loader = jinja2 . PrefixLoader ( { namespace : jinja2 . ChoiceLoader ( paths ) for namespace , paths in self . __template_loader__ . items ( ) } , "::" )
if self . cache is not None :
    for func , instance in self . cached :
        setattr ( instance , func . __name__ , self . cache . memoize ( ) ( func ) )
return self . blueprint


signature:
def view_maker ( self , name , instance = None ) :
    """Create a view
 : param name : Name of the route function to use for the view .
 : type name : str
 : return : Route function which makes use of Nemo context ( such as menu informations )
 : rtype : function"""

implementation:
if instance is None :
    instance = self
sig = "lang" in [ parameter . name for parameter in inspect . signature ( getattr ( instance , name ) ) . parameters . values ( ) ]
def route ( ** kwargs ) :
    if sig and "lang" not in kwargs :
        kwargs [ "lang" ] = self . get_locale ( )
    if "semantic" in kwargs :
        del kwargs [ "semantic" ]
    return self . route ( getattr ( instance , name ) , ** kwargs )
return route


signature:
def main_collections ( self , lang = None ) :
    """Retrieve main parent collections of a repository
 : param lang : Language to retrieve information in
 : return : Sorted collections representations"""

implementation: return sorted ( [ { "id" : member . id , "label" : str ( member . get_label ( lang = lang ) ) , "model" : str ( member . model ) , "type" : str ( member . type ) , "size" : member . size } for member in self . resolver . getMetadata ( ) . members ] , key = itemgetter ( "label" ) )


signature:
def make_cache_keys ( self , endpoint , kwargs ) :
    """This function is built to provide cache keys for templates
 : param endpoint : Current endpoint
 : param kwargs : Keyword Arguments
 : return : tuple of i18n dependant cache key and i18n ignoring cache key
 : rtype : tuple ( str )"""

implementation:
keys = sorted ( kwargs . keys ( ) )
i18n_cache_key = endpoint + "|" + "|" . join ( [ kwargs [ k ] for k in keys ] )
if "lang" in keys :
    cache_key = endpoint + "|" + "|" . join ( [ kwargs [ k ] for k in keys if k != "lang" ] )
else :
    cache_key = i18n_cache_key
return i18n_cache_key , cache_key


signature:
def render ( self , template , ** kwargs ) :
    """Render a route template and adds information to this route .
 : param template : Template name .
 : type template : str
 : param kwargs : dictionary of named arguments used to be passed to the template
 : type kwargs : dict
 : return : Http Response with rendered template
 : rtype : flask . Response"""

implementation:
kwargs [ "cache_key" ] = "%s" % kwargs [ "url" ] . values ( )
kwargs [ "lang" ] = self . get_locale ( )
kwargs [ "assets" ] = self . assets
kwargs [ "main_collections" ] = self . main_collections ( kwargs [ "lang" ] )
kwargs [ "cache_active" ] = self . cache is not None
kwargs [ "cache_time" ] = 0
kwargs [ "cache_key" ] , kwargs [ "cache_key_i18n" ] = self . make_cache_keys ( request . endpoint , kwargs [ "url" ] )
kwargs [ "template" ] = template
for plugin in self . __plugins_render_views__ :
    kwargs . update ( plugin . render ( ** kwargs ) )
return render_template ( kwargs [ "template" ] , ** kwargs )


signature:
def route ( self , fn , ** kwargs ) :
    """Route helper : apply fn function but keep the calling object , * ie * kwargs , for other functions
 : param fn : Function to run the route with
 : type fn : function
 : param kwargs : Parsed url arguments
 : type kwargs : dict
 : return : HTTP Response with rendered template
 : rtype : flask . Response"""

implementation:
new_kwargs = fn ( ** kwargs )
# If there is no templates , we assume that the response is finalized :
if not isinstance ( new_kwargs , dict ) :
    return new_kwargs
new_kwargs [ "url" ] = kwargs
return self . render ( ** new_kwargs )


signature:
def register ( self ) :
    """Register the app using Blueprint
 : return : Nemo blueprint
 : rtype : flask . Blueprint"""

implementation:
if self . app is not None :
    if not self . blueprint :
        self . blueprint = self . create_blueprint ( )
    self . app . register_blueprint ( self . blueprint )
    if self . cache is None : # We register a fake cache extension .
        setattr ( self . app . jinja_env , "_fake_cache_extension" , self )
        self . app . jinja_env . add_extension ( FakeCacheExtension )
    return self . blueprint
return None


signature:
def register_filters ( self ) :
    """Register filters for Jinja to use
 . . note : : Extends the dictionary filters of jinja _ env using self . _ filters list"""

implementation:
for _filter , instance in self . _filters :
    if not instance :
        self . app . jinja_env . filters [ _filter . replace ( "f_" , "" ) ] = getattr ( flask_nemo . filters , _filter )
    else :
        self . app . jinja_env . filters [ _filter . replace ( "f_" , "" ) ] = getattr ( instance , _filter . replace ( "_{}" . format ( instance . name ) , "" ) )


signature:
def register_plugins ( self ) :
    """Register plugins in Nemo instance
 - Clear routes first if asked by one plugin
 - Clear assets if asked by one plugin and replace by the last plugin registered static _ folder
 - Register each plugin
 - Append plugin routes to registered routes
 - Append plugin filters to registered filters
 - Append templates directory to given namespaces
 - Append assets ( CSS , JS , statics ) to given resources
 - Append render view ( if exists ) to Nemo . render stack"""

implementation:
if len ( [ plugin for plugin in self . __plugins__ . values ( ) if plugin . clear_routes ] ) > 0 : # Clear current routes
    self . _urls = list ( )
    self . cached = list ( )
clear_assets = [ plugin for plugin in self . __plugins__ . values ( ) if plugin . clear_assets ]
if len ( clear_assets ) > 0 and not self . prevent_plugin_clearing_assets : # Clear current Assets
    self . __assets__ = copy ( type ( self ) . ASSETS )
    static_path = [ plugin . static_folder for plugin in clear_assets if plugin . static_folder ]
    if len ( static_path ) > 0 :
        self . static_folder = static_path [ - 1 ]
for plugin in self . __plugins__ . values ( ) :
    self . _urls . extend ( [ ( url , function , methods , plugin ) for url , function , methods in plugin . routes ] )
    self . _filters . extend ( [ ( filt , plugin ) for filt in plugin . filters ] )
    self . __templates_namespaces__ . extend ( [ ( namespace , directory ) for namespace , directory in plugin . templates . items ( ) ] )
    for asset_type in self . __assets__ :
        for key , value in plugin . assets [ asset_type ] . items ( ) :
            self . __assets__ [ asset_type ] [ key ] = value
    if plugin . augment :
        self . __plugins_render_views__ . append ( plugin )
    if hasattr ( plugin , "CACHED" ) :
        for func in plugin . CACHED :
            self . cached . append ( ( getattr ( plugin , func ) , plugin ) )
    plugin . register_nemo ( self )


signature:
def chunk ( self , text , reffs ) :
    """Handle a list of references depending on the text identifier using the chunker dictionary .
 : param text : Text object from which comes the references
 : type text : MyCapytains . resources . texts . api . Text
 : param reffs : List of references to transform
 : type reffs : References
 : return : Transformed list of references
 : rtype : [ str ]"""

implementation:
if str ( text . id ) in self . chunker :
    return self . chunker [ str ( text . id ) ] ( text , reffs )
return self . chunker [ "default" ] ( text , reffs )


signature:
def update ( self , fieldname , localValue , remoteValue ) :
    '''Returns the appropriate current value , based on the changes
 recorded by this ChangeTracker , the value stored by the server
 ( ` localValue ` ) , and the value stored by the synchronizing client
 ( ` remoteValue ` ) . If ` remoteValue ` conflicts with changes stored
 locally , then a ` pysyncml . ConflictError ` is raised .
 If a change needs to be applied because ` remoteValue ` has been
 updated , then the new value will be returned , and this
 ChangeTracker will be updated such that a call to
 : meth : ` getChangeSpec ` will incorporate the change .
 : param fieldname :
 The name of the fieldname being evaluated .
 : param localValue :
 The value of the field as stored by the server , usually the one that
 also stored the current change - spec . If ` localValue ` is ` ` None ` ` ,
 then it is assumed that the field was potentially added ( this will
 first be verified against the stored change - spec ) .
 : param remoteValue :
 The new value being presented that may or may not be a source of
 conflict . If ` remoteValue ` is ` ` None ` ` , then it is assumed that
 the field was potentially deleted ( this will first be verified
 against the stored change - spec ) .'''

implementation:
if localValue == remoteValue :
    return localValue
ct = constants . ITEM_DELETED if remoteValue is None else constants . ITEM_MODIFIED
if localValue is None :
    ct = constants . ITEM_ADDED
# todo : i should probably trap irep errors . for example , if this
# cspec has a field " x " marked as deleted , then ` localValue `
# must be None . . . etc .
# TODO : i think this kind of handling would break in ListChangeTracker ! . . .
changed = self . isChange ( fieldname , ct , remoteValue )
if changed is None :
    return localValue
self . append ( changed , ct , initialValue = localValue , isMd5 = False )
return remoteValue


signature:
def isChange ( self , fieldname , changeType , newValue = None , isMd5 = False ) :
    '''Implements as specified in : meth : ` . ChangeTracker . isChange ` where
 the ` changeObject ` is simply the fieldname that needs to be
 updated with the ` newValue ` . Currently , this is always equal to
 ` fieldname ` .'''

implementation:
# todo : this seems inefficient . . .
changes = self . _collapseChanges ( self . baseline , self . current )
if fieldname not in changes :
    return fieldname
cur = changes [ fieldname ]
if changeType == constants . ITEM_DELETED :
    if cur . op == constants . ITEM_ADDED or cur . op == constants . ITEM_DELETED : # the field is deleted because it hasn ' t been added yet
    # ( the check for cur . op = = constants . ITEM _ DELETED should
    # never be true , so just here for paranoia . . . )
        return None
    # we are requiring that the current / new values are different ,
    # thus there is a collision between the added values
    raise ConflictError ( 'conflicting deletion of field "%s"' % ( fieldname , ) )
# the ` newValue ` is different than the current value ( otherwise
# this method should not have been called ) - - either it was added
# or modified .
# if it appears to be " added " , then it may be because it was
# deleted in this tracker .
# if it appears to be " modified " , then it may be because it
# was modified in this tracker .
# in either case , check to see if it is equal to the initial
# value , and if it was , then there was actually no change .
if isMd5Equal ( newValue , isMd5 , cur . ival , cur . md5 ) : # the new value is equal to the initial value , so this
# field was not changed ( but has local changes )
    return None
# the new value is not equal to the initial value , which means
# that they were both changed and / or added .
raise ConflictError ( 'conflicting addition or modification of field "%s"' % ( fieldname , ) )


signature:
def append ( self , listIndex , changeType , initialValue = None , isMd5 = False ) :
    '''Adds a change spec to the current list of changes . The ` listIndex `
 represents the line number ( in multi - line mode ) or word number ( in
 single - line mode ) , and must be * * INCLUSIVE * * of both additions and
 deletions .'''

implementation:
if not isMd5 and initialValue is not None and len ( initialValue ) > 32 :
    initialValue = hashlib . md5 ( initialValue ) . hexdigest ( )
    isMd5 = True
cur = adict ( index = int ( listIndex ) , op = changeType , ival = initialValue , md5 = isMd5 )
for idx , val in enumerate ( self . current ) :
    if val . index < cur . index :
        continue
    if val . index > cur . index :
        self . current . insert ( idx , cur )
        break
    # todo : this should never happen . . . ( there should not be a change
    # reported for the same line without a ` pushChangeSpec ( ) ` between )
    # todo : perhaps attempt a merging ? . . .
    raise InvalidChangeSpec ( 'conflicting changes for index %d' % ( cur . index , ) )
else :
    self . current . append ( cur )


signature:
def isChange ( self , listIndex , changeType , newValue = None , isMd5 = False , token = None ) :
    '''Implements as specified in : meth : ` . ChangeTracker . isChange ` where
 the ` changeObject ` is a two - element tuple . The first element is
 the index at which the change should be applied , and the second
 element is an abstract token that should be passed back into this
 method at every iteration .
 IMPORTANT : unlike the AttributeChangeTracker , the
 ListChangeTracker ' s ` isChange ( ) ` method is sensitive to order
 ( which is why it uses the ` changeObject ` and ` token `
 mechanisms . Therefore , it is important to call ` isChange ( ) `
 sequentially with all changes in the order that they occur in the
 change list .'''

implementation:
# THE INDEX PASSED TO ListChangeTracker . isChange ( ) DOES NOT INCLUDE :
# - local deletions
# - remote additions
adjust = 0
# tracks local deletes
token = token
# tracks consecutive addition adjustments
index = int ( listIndex )
ret = index
# todo : this should reduce complexity later on , but something
# went wrong . . .
# if changeType ! = constants . ITEM _ ADDED :
# token = None
# else :
# if token is None or token [ 0 ] ! = index :
# token = ( ret , 0)
# token = ( ret , token [ 1 ] + 1)
# todo : this seems inefficient . . .
changes = self . _collapseChanges ( self . baseline , self . current )
for cur in changes :
    if cur . index > index :
        if changeType != constants . ITEM_ADDED :
            return ( ret , None )
        if token is None or token [ 0 ] != index - adjust :
            token = ( ret , 0 )
        token = ( ret , token [ 1 ] + 1 )
        return ( ret , token )
    if cur . index != index :
        if cur . op == constants . ITEM_DELETED :
            index += 1
            adjust += 1
        continue
    if token is not None and token [ 0 ] == index - adjust :
        index += token [ 1 ]
        continue
    if changeType == constants . ITEM_DELETED :
        if cur . op == constants . ITEM_ADDED : # the field is deleted because it hasn ' t been added yet
            return ( None , None )
        # we are requiring that the current / new values are different ,
        # thus there is a collision between the added values
        raise ConflictError ( 'conflicting deletion of list index %r' % ( index , ) )
    if changeType == constants . ITEM_ADDED :
        if token is None :
            token = ( ret , 0 )
        token = ( ret , token [ 1 ] + 1 )
        if cur . op == constants . ITEM_DELETED :
            if isMd5Equal ( newValue , isMd5 , cur . ival , cur . md5 ) :
                return ( None , token )
            # todo : this * could * be a del - mod * conflict * . . . but not
            # * NECESSARILY * so , since it could be a
            # del - adjacent - add , which is not a problem . in the
            # conflict case , the resolution will cause the
            # modified line to silently win .
            # TODO : perhaps i should err on the side of safety and
            # issue a ConflictError ? . . .
        return ( ret , token )
    if cur . op == constants . ITEM_DELETED :
        index += 1
        adjust += 1
        continue
    # changeType = mod , op = add / mod
    if cur . op == constants . ITEM_ADDED : # todo : i ' m not sure if this case is even possible . . .
        raise ConflictError ( 'conflicting addition of list index %r' % ( index , ) )
    # mod / mod - check initvalue
    if isMd5Equal ( newValue , isMd5 , cur . ival , cur . md5 ) : # the new value is equal to the initial value , so this
    # line was not changed ( but has local changes )
        return ( None , None )
    # the new value is not equal to the initial value , which means
    # that they were both changed and / or added .
    raise ConflictError ( 'conflicting modification of list index %r' % ( index , ) )
if changeType != constants . ITEM_ADDED :
    return ( ret , None )
if token is None or token [ 0 ] != index - adjust :
    token = ( ret , 0 )
token = ( ret , token [ 1 ] + 1 )
return ( ret , token )


signature:
def add_tag ( ) :
    """Obtains the data from the pipe and appends the given tag ."""

implementation:
if len ( sys . argv ) > 1 :
    tag = sys . argv [ 1 ]
    doc_mapper = DocMapper ( )
    if doc_mapper . is_pipe :
        count = 0
        for obj in doc_mapper . get_pipe ( ) :
            obj . add_tag ( tag )
            obj . update ( tags = obj . tags )
            count += 1
        print_success ( "Added tag '{}' to {} object(s)" . format ( tag , count ) )
    else :
        print_error ( "Please use this script with pipes" )
else :
    print_error ( "Usage: jk-add-tag <tag>" )
    sys . exit ( )


signature:
def manual_configure ( ) :
    """Function to manually configure jackal ."""

implementation:
print ( "Manual configuring jackal" )
mapping = { '1' : 'y' , '0' : 'n' }
config = Config ( )
# Host
host = input_with_default ( "What is the Elasticsearch host?" , config . get ( 'jackal' , 'host' ) )
config . set ( 'jackal' , 'host' , host )
# SSL
if input_with_default ( "Use SSL?" , mapping [ config . get ( 'jackal' , 'use_ssl' ) ] ) == 'y' :
    config . set ( 'jackal' , 'use_ssl' , '1' )
    if input_with_default ( "Setup custom server cert?" , 'y' ) == 'y' :
        ca_certs = input_with_default ( "Server certificate location?" , config . get ( 'jackal' , 'ca_certs' ) )
        config . set ( 'jackal' , 'ca_certs' , ca_certs )
    else :
        config . set ( 'jackal' , 'ca_certs' , '' )
else :
    config . set ( 'jackal' , 'use_ssl' , '0' )
if input_with_default ( "Setup client certificates?" , mapping [ config . get ( 'jackal' , 'client_certs' ) ] ) == 'y' :
    config . set ( 'jackal' , 'client_certs' , '1' )
    client_cert = input_with_default ( "Client cert location?" , config . get ( 'jackal' , 'client_cert' ) )
    config . set ( 'jackal' , 'client_cert' , client_cert )
    client_key = input_with_default ( "Client key location?" , config . get ( 'jackal' , 'client_key' ) )
    config . set ( 'jackal' , 'client_key' , client_key )
else :
    config . set ( 'jackal' , 'client_certs' , '0' )
# Index
index = input_with_default ( "What index prefix should jackal use?" , config . get ( 'jackal' , 'index' ) )
config . set ( 'jackal' , 'index' , index )
initialize_indices = ( input_with_default ( "Do you want to initialize the indices?" , 'y' ) . lower ( ) == 'y' )
# Nmap
nmap_dir = input_with_default ( "What directory do you want to place the nmap results in?" , config . get ( 'nmap' , 'directory' ) )
if not os . path . exists ( nmap_dir ) :
    os . makedirs ( nmap_dir )
config . set ( 'nmap' , 'directory' , nmap_dir )
nmap_options = input_with_default ( "What nmap options do you want to set for 'custom' (for example '-p 22,445')?" , config . get ( 'nmap' , 'options' ) )
config . set ( 'nmap' , 'options' , nmap_options )
# Nessus
configure_nessus = ( input_with_default ( "Do you want to setup nessus?" , 'n' ) . lower ( ) == 'y' )
if configure_nessus :
    nessus_host = input_with_default ( "What is the nessus host?" , config . get ( 'nessus' , 'host' ) )
    nessus_template = input_with_default ( "What template should jackal use?" , config . get ( 'nessus' , 'template_name' ) )
    nessus_access = input_with_default ( "What api access key should jackal use?" , config . get ( 'nessus' , 'access_key' ) )
    nessus_secret = input_with_default ( "What api secret key should jackal use?" , config . get ( 'nessus' , 'secret_key' ) )
    config . set ( 'nessus' , 'host' , nessus_host )
    config . set ( 'nessus' , 'template_name' , nessus_template )
    config . set ( 'nessus' , 'access_key' , nessus_access )
    config . set ( 'nessus' , 'secret_key' , nessus_secret )
# Named pipes
configure_pipes = ( input_with_default ( "Do you want to setup named pipes?" , 'n' ) . lower ( ) == 'y' )
if configure_pipes :
    directory = input_with_default ( "What directory do you want to place the named pipes in?" , config . get ( 'pipes' , 'directory' ) )
    config . set ( 'pipes' , 'directory' , directory )
    config_file = input_with_default ( "What is the name of the named pipe config?" , config . get ( 'pipes' , 'config_file' ) )
    config . set ( 'pipes' , 'config_file' , config_file )
    if not os . path . exists ( directory ) :
        create = ( input_with_default ( "Do you want to create the directory?" , 'n' ) . lower ( ) == 'y' )
        if create :
            os . makedirs ( directory )
    if not os . path . exists ( os . path . join ( config . config_dir , config_file ) ) :
        f = open ( os . path . join ( config . config_dir , config_file ) , 'a' )
        f . close ( )
config . write_config ( initialize_indices )


signature:
def set ( self , section , key , value ) :
    """Creates the section value if it does not exists and sets the value .
 Use write _ config to actually set the value ."""

implementation:
if not section in self . config :
    self . config . add_section ( section )
self . config . set ( section , key , value )


signature:
def get ( self , section , key ) :
    """This function tries to retrieve the value from the configfile
 otherwise will return a default ."""

implementation:
try :
    return self . config . get ( section , key )
except configparser . NoSectionError :
    pass
except configparser . NoOptionError :
    pass
return self . defaults [ section ] [ key ]


signature:
def config_dir ( self ) :
    """Returns the configuration directory"""

implementation:
home = expanduser ( '~' )
config_dir = os . path . join ( home , '.jackal' )
return config_dir


signature:
def write_config ( self , initialize_indices = False ) :
    """Write the current config to disk to store them ."""

implementation:
if not os . path . exists ( self . config_dir ) :
    os . mkdir ( self . config_dir )
with open ( self . config_file , 'w' ) as configfile :
    self . config . write ( configfile )
if initialize_indices :
    index = self . get ( 'jackal' , 'index' )
    from jackal import Host , Range , Service , User , Credential , Log
    from jackal . core import create_connection
    create_connection ( self )
    Host . init ( index = "{}-hosts" . format ( index ) )
    Range . init ( index = "{}-ranges" . format ( index ) )
    Service . init ( index = "{}-services" . format ( index ) )
    User . init ( index = "{}-users" . format ( index ) )
    Credential . init ( index = "{}-creds" . format ( index ) )
    Log . init ( index = "{}-log" . format ( index ) )


signature:
def ensure_remote_branch_is_tracked ( branch ) :
    """Track the specified remote branch if it is not already tracked ."""

implementation:
if branch == MASTER_BRANCH : # We don ' t need to explicitly track the master branch , so we ' re done .
    return
# Ensure the specified branch is in the local branch list .
output = subprocess . check_output ( [ 'git' , 'branch' , '--list' ] )
for line in output . split ( '\n' ) :
    if line . strip ( ) == branch : # We are already tracking the remote branch
        break
else : # We are not tracking the remote branch , so track it .
    try :
        sys . stdout . write ( subprocess . check_output ( [ 'git' , 'checkout' , '--track' , 'origin/%s' % branch ] ) )
    except subprocess . CalledProcessError : # Bail gracefully .
        raise SystemExit ( 1 )


signature:
def main ( branch ) :
    """Checkout , update and branch from the specified branch ."""

implementation:
try : # Ensure that we ' re in a git repository . This command is silent unless
# you ' re not actually in a git repository , in which case , you receive a
# " Not a git repository " error message .
    output = subprocess . check_output ( [ 'git' , 'rev-parse' ] ) . decode ( 'utf-8' )
    sys . stdout . write ( output )
except subprocess . CalledProcessError : # Bail if we ' re not in a git repository .
    return
# This behavior ensures a better user experience for those that aren ' t
# intimately familiar with git .
ensure_remote_branch_is_tracked ( branch )
# Switch to the specified branch and update it .
subprocess . check_call ( [ 'git' , 'checkout' , '--quiet' , branch ] )
# Pulling is always safe here , because we never commit to this branch .
subprocess . check_call ( [ 'git' , 'pull' , '--quiet' ] )
# Checkout the top commit in the branch , effectively going " untracked . "
subprocess . check_call ( [ 'git' , 'checkout' , '--quiet' , '%s~0' % branch ] )
# Clean up the repository of Python cruft . Because we ' ve just switched
# branches and compiled Python files should not be version controlled ,
# there are likely leftover compiled Python files sitting on disk which may
# confuse some tools , such as sqlalchemy - migrate .
subprocess . check_call ( [ 'find' , '.' , '-name' , '"*.pyc"' , '-delete' ] )
# For the sake of user experience , give some familiar output .
print ( 'Your branch is up to date with branch \'origin/%s\'.' % branch )


signature:
def get_interface_name ( ) :
    """Returns the interface name of the first not link _ local and not loopback interface ."""

implementation:
interface_name = ''
interfaces = psutil . net_if_addrs ( )
for name , details in interfaces . items ( ) :
    for detail in details :
        if detail . family == socket . AF_INET :
            ip_address = ipaddress . ip_address ( detail . address )
            if not ( ip_address . is_link_local or ip_address . is_loopback ) :
                interface_name = name
                break
return interface_name


signature:
def load_targets ( self ) :
    """load _ targets will load the services with smb signing disabled and if ldap is enabled the services with the ldap port open ."""

implementation:
ldap_services = [ ]
if self . ldap :
    ldap_services = self . search . get_services ( ports = [ 389 ] , up = True )
self . ldap_strings = [ "ldap://{}" . format ( service . address ) for service in ldap_services ]
self . services = self . search . get_services ( tags = [ 'smb_signing_disabled' ] )
self . ips = [ str ( service . address ) for service in self . services ]


signature:
def write_targets ( self ) :
    """write _ targets will write the contents of ips and ldap _ strings to the targets _ file ."""

implementation:
if len ( self . ldap_strings ) == 0 and len ( self . ips ) == 0 :
    print_notification ( "No targets left" )
    if self . auto_exit :
        if self . notifier :
            self . notifier . stop ( )
        self . terminate_processes ( )
with open ( self . targets_file , 'w' ) as f :
    f . write ( '\n' . join ( self . ldap_strings + self . ips ) )


signature:
def start_processes ( self ) :
    """Starts the ntlmrelayx . py and responder processes .
 Assumes you have these programs in your path ."""

implementation:
self . relay = subprocess . Popen ( [ 'ntlmrelayx.py' , '-6' , '-tf' , self . targets_file , '-w' , '-l' , self . directory , '-of' , self . output_file ] , cwd = self . directory )
self . responder = subprocess . Popen ( [ 'responder' , '-I' , self . interface_name ] )


signature:
def callback ( self , event ) :
    """Function that gets called on each event from pyinotify ."""

implementation:
# IN _ CLOSE _ WRITE - > 0x000008
if event . mask == 0x00000008 :
    if event . name . endswith ( '.json' ) :
        print_success ( "Ldapdomaindump file found" )
        if event . name in [ 'domain_groups.json' , 'domain_users.json' ] :
            if event . name == 'domain_groups.json' :
                self . domain_groups_file = event . pathname
            if event . name == 'domain_users.json' :
                self . domain_users_file = event . pathname
            if self . domain_groups_file and self . domain_users_file :
                print_success ( "Importing users" )
                subprocess . Popen ( [ 'jk-import-domaindump' , self . domain_groups_file , self . domain_users_file ] )
        elif event . name == 'domain_computers.json' :
            print_success ( "Importing computers" )
            subprocess . Popen ( [ 'jk-import-domaindump' , event . pathname ] )
        # Ldap has been dumped , so remove the ldap targets .
        self . ldap_strings = [ ]
        self . write_targets ( )
    if event . name . endswith ( '_samhashes.sam' ) :
        host = event . name . replace ( '_samhashes.sam' , '' )
        # TODO import file .
        print_success ( "Secretsdump file, host ip: {}" . format ( host ) )
        subprocess . Popen ( [ 'jk-import-secretsdump' , event . pathname ] )
        # Remove this system from this ip list .
        self . ips . remove ( host )
        self . write_targets ( )


signature:
def watch ( self ) :
    """Watches directory for changes"""

implementation:
wm = pyinotify . WatchManager ( )
self . notifier = pyinotify . Notifier ( wm , default_proc_fun = self . callback )
wm . add_watch ( self . directory , pyinotify . ALL_EVENTS )
try :
    self . notifier . loop ( )
except ( KeyboardInterrupt , AttributeError ) :
    print_notification ( "Stopping" )
finally :
    self . notifier . stop ( )
    self . terminate_processes ( )


signature:
def terminate_processes ( self ) :
    """Terminate the processes ."""

implementation:
if self . relay :
    self . relay . terminate ( )
if self . responder :
    self . responder . terminate ( )


signature:
def wait ( self ) :
    """This function waits for the relay and responding processes to exit .
 Captures KeyboardInterrupt to shutdown these processes ."""

implementation:
try :
    self . relay . wait ( )
    self . responder . wait ( )
except KeyboardInterrupt :
    print_notification ( "Stopping" )
finally :
    self . terminate_processes ( )


signature:
def getAnnotations ( self , targets , wildcard = "." , include = None , exclude = None , limit = None , start = 1 , expand = False , ** kwargs ) :
    """Retrieve annotations from the query provider
 : param targets : The CTS URN ( s ) to query as the target of annotations
 : type targets : [ MyCapytain . common . reference . URN ] , URN or None
 : param wildcard : Wildcard specifier for how to match the URN
 : type wildcard : str
 : param include : URI ( s ) of Annotation types to include in the results
 : type include : list ( str )
 : param exclude : URI ( s ) of Annotation types to include in the results
 : type exclude : list ( str )
 : param limit : The max number of results to return ( Default is None for no limit )
 : type limit : int
 : param start : the starting record to return ( Default is 1)
 : type start : int
 : param expand : Flag to state whether Annotations are expanded ( Default is False )
 : type expand : bool
 : return : Tuple representing the query results . The first element
 The first element is the number of total Annotations found
 The second element is the list of Annotations
 : rtype : ( int , list ( Annotation )
 . . note : :
 Wildcard should be one of the following value
 - ' . ' to match exact ,
 - ' . % ' to match exact plus lower in the hierarchy
 - ' % . ' to match exact + higher in the hierarchy
 - ' - ' to match in the range
 - ' % . % ' to match all"""

implementation: return 0 , [ ]


signature:
def render ( self , ** kwargs ) :
    """Make breadcrumbs for a route
 : param kwargs : dictionary of named arguments used to construct the view
 : type kwargs : dict
 : return : List of dict items the view can use to construct the link .
 : rtype : { str : list ( { " link " : str , " title " , str , " args " , dict } ) }"""

implementation:
breadcrumbs = [ ]
# this is the list of items we want to accumulate in the breadcrumb trail .
# item [ 0 ] is the key into the kwargs [ " url " ] object and item [ 1 ] is the name of the route
# setting a route name to None means that it ' s needed to construct the route of the next item in the list
# but shouldn ' t be included in the list itself ( this is currently the case for work - -
# at some point we probably should include work in the navigation )
breadcrumbs = [ ]
if "collections" in kwargs :
    breadcrumbs = [ { "title" : "Text Collections" , "link" : ".r_collections" , "args" : { } } ]
    if "parents" in kwargs [ "collections" ] :
        breadcrumbs += [ { "title" : parent [ "label" ] , "link" : ".r_collection_semantic" , "args" : { "objectId" : parent [ "id" ] , "semantic" : f_slugify ( parent [ "label" ] ) , } , } for parent in kwargs [ "collections" ] [ "parents" ] ] [ : : - 1 ]
    if "current" in kwargs [ "collections" ] :
        breadcrumbs . append ( { "title" : kwargs [ "collections" ] [ "current" ] [ "label" ] , "link" : None , "args" : { } } )
# don ' t link the last item in the trail
if len ( breadcrumbs ) > 0 :
    breadcrumbs [ - 1 ] [ "link" ] = None
return { "breadcrumbs" : breadcrumbs }


signature:
def main ( ) :
    """This function obtains hosts from core and starts a nessus scan on these hosts .
 The nessus tag is appended to the host tags ."""

implementation:
config = Config ( )
core = HostSearch ( )
hosts = core . get_hosts ( tags = [ '!nessus' ] , up = True )
hosts = [ host for host in hosts ]
host_ips = "," . join ( [ str ( host . address ) for host in hosts ] )
url = config . get ( 'nessus' , 'host' )
access = config . get ( 'nessus' , 'access_key' )
secret = config . get ( 'nessus' , 'secret_key' )
template_name = config . get ( 'nessus' , 'template_name' )
nessus = Nessus ( access , secret , url , template_name )
scan_id = nessus . create_scan ( host_ips )
nessus . start_scan ( scan_id )
for host in hosts :
    host . add_tag ( 'nessus' )
    host . save ( )
Logger ( ) . log ( "nessus" , "Nessus scan started on {} hosts" . format ( len ( hosts ) ) , { 'scanned_hosts' : len ( hosts ) } )


signature:
def get_template_uuid ( self ) :
    """Retrieves the uuid of the given template name ."""

implementation:
response = requests . get ( self . url + 'editor/scan/templates' , headers = self . headers , verify = False )
templates = json . loads ( response . text )
for template in templates [ 'templates' ] :
    if template [ 'name' ] == self . template_name :
        return template [ 'uuid' ]


signature:
def create_scan ( self , host_ips ) :
    """Creates a scan with the given host ips
 Returns the scan id of the created object ."""

implementation:
now = datetime . datetime . now ( )
data = { "uuid" : self . get_template_uuid ( ) , "settings" : { "name" : "jackal-" + now . strftime ( "%Y-%m-%d %H:%M" ) , "text_targets" : host_ips } }
response = requests . post ( self . url + 'scans' , data = json . dumps ( data ) , verify = False , headers = self . headers )
if response :
    result = json . loads ( response . text )
    return result [ 'scan' ] [ 'id' ]


signature:
def start_scan ( self , scan_id ) :
    """Starts the scan identified by the scan _ id . s"""

implementation: requests . post ( self . url + 'scans/{}/launch' . format ( scan_id ) , verify = False , headers = self . headers )


implementation:
partners = dict ( ( a , ( rankings [ ( a , 1 ) ] , 1 ) ) for a in A )
is_stable = False
# whether the current pairing ( given by ` partners ` ) is stable
while is_stable == False :
    is_stable = True
    for b in B :
        is_paired = False
        # whether b has a pair which b ranks < = to n
        for n in range ( 1 , len ( B ) + 1 ) :
            a = rankings [ ( b , n ) ]
            a_partner , a_n = partners [ a ]
            if a_partner == b :
                if is_paired :
                    is_stable = False
                    partners [ a ] = ( rankings [ ( a , a_n + 1 ) ] , a_n + 1 )
                else :
                    is_paired = True
return sorted ( ( a , b ) for ( a , ( b , n ) ) in partners . items ( ) )


signature:
def cmpToDataStore_uri ( base , ds1 , ds2 ) :
    '''Bases the comparison of the datastores on URI alone .'''

implementation:
ret = difflib . get_close_matches ( base . uri , [ ds1 . uri , ds2 . uri ] , 1 , cutoff = 0.5 )
if len ( ret ) <= 0 :
    return 0
if ret [ 0 ] == ds1 . uri :
    return - 1
return 1


signature:
def add_tag ( self , tag ) :
    """Adds a tag to the list of tags and makes sure the result list contains only unique results ."""

implementation: self . tags = list ( set ( self . tags or [ ] ) | set ( [ tag ] ) )


signature:
def remove_tag ( self , tag ) :
    """Removes a tag from this object"""

implementation: self . tags = list ( set ( self . tags or [ ] ) - set ( [ tag ] ) )


signature:
def to_dict ( self , include_meta = False ) :
    """Returns the result as a dictionary , provide the include _ meta flag to als show information like index and doctype ."""

implementation:
result = super ( JackalDoc , self ) . to_dict ( include_meta = include_meta )
if include_meta :
    source = result . pop ( '_source' )
    return { ** result , ** source }
else :
    return result


signature:
def r_annotations ( self ) :
    """Route to retrieve annotations by target
 : param target _ urn : The CTS URN for which to retrieve annotations
 : type target _ urn : str
 : return : a JSON string containing count and list of resources
 : rtype : { str : Any }"""

implementation:
target = request . args . get ( "target" , None )
wildcard = request . args . get ( "wildcard" , "." , type = str )
include = request . args . get ( "include" )
exclude = request . args . get ( "exclude" )
limit = request . args . get ( "limit" , None , type = int )
start = request . args . get ( "start" , 1 , type = int )
expand = request . args . get ( "expand" , False , type = bool )
if target :
    try :
        urn = MyCapytain . common . reference . URN ( target )
    except ValueError :
        return "invalid urn" , 400
    count , annotations = self . __queryinterface__ . getAnnotations ( urn , wildcard = wildcard , include = include , exclude = exclude , limit = limit , start = start , expand = expand )
else : # Note that this implementation is not done for too much annotations
# because we do not implement pagination here
    count , annotations = self . __queryinterface__ . getAnnotations ( None , limit = limit , start = start , expand = expand )
mapped = [ ]
response = { "@context" : type ( self ) . JSONLD_CONTEXT , "id" : url_for ( ".r_annotations" , start = start , limit = limit ) , "type" : "AnnotationCollection" , "startIndex" : start , "items" : [ ] , "total" : count }
for a in annotations :
    mapped . append ( { "id" : url_for ( ".r_annotation" , sha = a . sha ) , "body" : url_for ( ".r_annotation_body" , sha = a . sha ) , "type" : "Annotation" , "target" : a . target . to_json ( ) , "dc:type" : a . type_uri , "owl:sameAs" : [ a . uri ] , "nemo:slug" : a . slug } )
response [ "items" ] = mapped
response = jsonify ( response )
return response


signature:
def r_annotation ( self , sha ) :
    """Route to retrieve contents of an annotation resource
 : param uri : The uri of the annotation resource
 : type uri : str
 : return : annotation contents
 : rtype : { str : Any }"""

implementation:
annotation = self . __queryinterface__ . getResource ( sha )
if not annotation :
    return "invalid resource uri" , 404
response = { "@context" : type ( self ) . JSONLD_CONTEXT , "id" : url_for ( ".r_annotation" , sha = annotation . sha ) , "body" : url_for ( ".r_annotation_body" , sha = annotation . sha ) , "type" : "Annotation" , "target" : annotation . target . to_json ( ) , "owl:sameAs" : [ annotation . uri ] , "dc:type" : annotation . type_uri , "nemo:slug" : annotation . slug }
return jsonify ( response )


signature:
def r_annotation_body ( self , sha ) :
    """Route to retrieve contents of an annotation resource
 : param uri : The uri of the annotation resource
 : type uri : str
 : return : annotation contents
 : rtype : { str : Any }"""

implementation:
annotation = self . __queryinterface__ . getResource ( sha )
if not annotation :
    return "invalid resource uri" , 404
# TODO this should inspect the annotation content
# set appropriate Content - Type headers
# and return the actual content
content = annotation . read ( )
if isinstance ( content , Response ) :
    return content
headers = { "Content-Type" : annotation . mimetype }
return Response ( content , headers = headers )


signature:
def describeStats ( stats , stream , title = None , details = True , totals = True , gettext = None ) :
    '''Renders an ASCII - table of the synchronization statistics ` stats ` ,
 example output :
 . . code - block : :
 | TITLE |
 | | | Local | Remote | Conflicts |
 | Source | Mode | Add | Mod | Del | Err | Add | Mod | Del | Err | Col | Mrg |
 | contacts | < = | - | - | - | - | 10,387 | - | - | - | - | - |
 | note | SS | 1,308 | - | 2 | - | - | - | - | - | - | - |
 | 1,310 local changes and 10,387 remote changes . |
 : Parameters :
 stats : dict
 The synchronization stats returned by a call to Adapter . sync ( ) .
 stream : file - like - object
 An output file - like object that has at least a ` write ( ) ` method ,
 e . g . ` ` sys . stdout ` ` can be used .
 title : str , optional , default : null
 A title placed at the top of the table - - if omitted ( the default ) ,
 then no title is rendered .
 details : bool , optional , default : true
 If truthy , a per - datastore listing of changes will be displayed
 ( as in the above example ) .
 totals : bool , optional , default : true
 If truthy , a summary of all changes will be displayed ( as in the
 above example ) .
 gettext : callable , optional , @ DEPRECATED ( 0.2.0 ) , default : null
 A ` gettext . gettext ` compatible callable used for translating
 localized content ( such as number formatting , etc . ) .
 NOTE : this parameter is deprecated , and will be replaced with
 a generalized i18n solution .'''

implementation:
from . import state
modeStringLut = dict ( ( ( constants . SYNCTYPE_TWO_WAY , '<>' ) , ( constants . SYNCTYPE_SLOW_SYNC , 'SS' ) , ( constants . SYNCTYPE_ONE_WAY_FROM_CLIENT , '->' ) , ( constants . SYNCTYPE_REFRESH_FROM_CLIENT , '=>' ) , ( constants . SYNCTYPE_ONE_WAY_FROM_SERVER , '<-' ) , ( constants . SYNCTYPE_REFRESH_FROM_SERVER , '<=' ) , ) )
if gettext is not None :
    _ = gettext
else :
    _ = lambda s : s
# todo : this does not handle the case where the title is wider than the table .
wSrc = len ( _ ( 'Source' ) )
wMode = len ( _ ( 'Mode' ) )
wCon = len ( _ ( 'Conflicts' ) )
wCol = len ( _ ( 'Col' ) )
wMrg = len ( _ ( 'Mrg' ) )
wHereAdd = wPeerAdd = len ( _ ( 'Add' ) )
wHereMod = wPeerMod = len ( _ ( 'Mod' ) )
wHereDel = wPeerDel = len ( _ ( 'Del' ) )
wHereErr = wPeerErr = len ( _ ( 'Err' ) )
totLoc = 0
totRem = 0
totErr = 0
totCol = 0
totMrg = 0
for key in stats . keys ( ) :
    wSrc = max ( wSrc , len ( key ) )
    wMode = max ( wMode , len ( modeStringLut . get ( stats [ key ] . mode ) ) )
    wCol = max ( wCol , len ( num2str ( stats [ key ] . conflicts ) ) )
    wMrg = max ( wMrg , len ( num2str ( stats [ key ] . merged ) ) )
    wHereAdd = max ( wHereAdd , len ( num2str ( stats [ key ] . hereAdd ) ) )
    wPeerAdd = max ( wPeerAdd , len ( num2str ( stats [ key ] . peerAdd ) ) )
    wHereMod = max ( wHereMod , len ( num2str ( stats [ key ] . hereMod ) ) )
    wPeerMod = max ( wPeerMod , len ( num2str ( stats [ key ] . peerMod ) ) )
    wHereDel = max ( wHereDel , len ( num2str ( stats [ key ] . hereDel ) ) )
    wPeerDel = max ( wPeerDel , len ( num2str ( stats [ key ] . peerDel ) ) )
    wHereErr = max ( wHereErr , len ( num2str ( stats [ key ] . hereErr ) ) )
    wPeerErr = max ( wPeerErr , len ( num2str ( stats [ key ] . peerErr ) ) )
    totLoc += stats [ key ] . hereAdd + stats [ key ] . hereMod + stats [ key ] . hereDel
    totRem += stats [ key ] . peerAdd + stats [ key ] . peerMod + stats [ key ] . peerDel
    totErr += stats [ key ] . hereErr + stats [ key ] . peerErr
    totCol += stats [ key ] . conflicts
    totMrg += stats [ key ] . merged
# TODO : i ' m 100 % sure there is a python library that can do this for me . . .
if wCon > wCol + 3 + wMrg :
    diff = wCon - ( wCol + 3 + wMrg )
    wCol += diff / 2
    wMrg = wCon - 3 - wCol
else :
    wCon = wCol + 3 + wMrg
if details :
    tWid = ( wSrc + 3 + wMode + 3 + wHereAdd + wHereMod + wHereDel + wHereErr + 9 + 3 + wPeerAdd + wPeerMod + wPeerDel + wPeerErr + 9 + 3 + wCon )
else :
    if title is None :
        tWid = 0
    else :
        tWid = len ( title )
if totals : # TODO : oh dear . from an i18n POV , this is * horrible * ! . . .
    sumlist = [ ]
    for val , singular , plural in [ ( totLoc , _ ( 'local change' ) , _ ( 'local changes' ) ) , ( totRem , _ ( 'remote change' ) , _ ( 'remote changes' ) ) , ( totErr , _ ( 'error' ) , _ ( 'errors' ) ) , ] :
        if val == 1 :
            sumlist . append ( num2str ( val ) + ' ' + singular )
        elif val > 1 :
            sumlist . append ( num2str ( val ) + ' ' + plural )
    if len ( sumlist ) <= 0 :
        sumlist = _ ( 'No changes' )
    elif len ( sumlist ) == 1 :
        sumlist = sumlist [ 0 ]
    else :
        sumlist = ', ' . join ( sumlist [ : - 1 ] ) + ' ' + _ ( 'and' ) + ' ' + sumlist [ - 1 ]
    if totMrg > 0 or totCol > 0 :
        sumlist += ': '
        if totMrg == 1 :
            sumlist += num2str ( totMrg ) + ' ' + _ ( 'merge' )
        elif totMrg > 1 :
            sumlist += num2str ( totMrg ) + ' ' + _ ( 'merges' )
        if totMrg > 0 and totCol > 0 :
            sumlist += ' ' + _ ( 'and' ) + ' '
        if totCol == 1 :
            sumlist += num2str ( totCol ) + ' ' + _ ( 'conflict' )
        elif totCol > 1 :
            sumlist += num2str ( totCol ) + ' ' + _ ( 'conflicts' )
    sumlist += '.'
    if len ( sumlist ) > tWid :
        wSrc += len ( sumlist ) - tWid
        tWid = len ( sumlist )
if title is not None :
    stream . write ( '+-' + '-' * tWid + '-+\n' )
    stream . write ( '| {0: ^{w}}' . format ( title , w = tWid ) )
    stream . write ( ' |\n' )
hline = '+-' + '-' * wSrc + '-+-' + '-' * wMode + '-+-' + '-' * ( wHereAdd + wHereMod + wHereDel + wHereErr + 9 ) + '-+-' + '-' * ( wPeerAdd + wPeerMod + wPeerDel + wPeerErr + 9 ) + '-+-' + '-' * wCon + '-+\n'
if details :
    stream . write ( hline )
    stream . write ( '| ' + ' ' * wSrc )
    stream . write ( ' | ' + ' ' * wMode )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Local' ) , w = ( wHereAdd + wHereMod + wHereDel + wHereErr + 9 ) ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Remote' ) , w = ( wPeerAdd + wPeerMod + wPeerDel + wPeerErr + 9 ) ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Conflicts' ) , w = wCon ) )
    stream . write ( ' |\n' )
    stream . write ( '| {0: >{w}}' . format ( _ ( 'Source' ) , w = wSrc ) )
    stream . write ( ' | {0: >{w}}' . format ( _ ( 'Mode' ) , w = wMode ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Add' ) , w = wHereAdd ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Mod' ) , w = wHereMod ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Del' ) , w = wHereDel ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Err' ) , w = wHereErr ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Add' ) , w = wPeerAdd ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Mod' ) , w = wPeerMod ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Del' ) , w = wPeerDel ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Err' ) , w = wPeerErr ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Col' ) , w = wCol ) )
    stream . write ( ' | {0: ^{w}}' . format ( _ ( 'Mrg' ) , w = wMrg ) )
    stream . write ( ' |\n' )
    hsline = '+-' + '-' * wSrc + '-+-' + '-' * wMode + '-+-' + '-' * wHereAdd + '-+-' + '-' * wHereMod + '-+-' + '-' * wHereDel + '-+-' + '-' * wHereErr + '-+-' + '-' * wPeerAdd + '-+-' + '-' * wPeerMod + '-+-' + '-' * wPeerDel + '-+-' + '-' * wPeerErr + '-+-' + '-' * wCol + '-+-' + '-' * wMrg + '-+\n'
    stream . write ( hsline )
    def numcol ( val , wid ) :
        if val == 0 :
            return ' | {0: ^{w}}' . format ( '-' , w = wid )
        return ' | {0: >{w}}' . format ( num2str ( val ) , w = wid )
    for key in sorted ( stats . keys ( ) , key = lambda k : str ( k ) . lower ( ) ) :
        stream . write ( '| {0: >{w}}' . format ( key , w = wSrc ) )
        stream . write ( ' | {0: ^{w}}' . format ( modeStringLut . get ( stats [ key ] . mode ) , w = wMode ) )
        stream . write ( numcol ( stats [ key ] . hereAdd , wHereAdd ) )
        stream . write ( numcol ( stats [ key ] . hereMod , wHereMod ) )
        stream . write ( numcol ( stats [ key ] . hereDel , wHereDel ) )
        stream . write ( numcol ( stats [ key ] . hereErr , wHereErr ) )
        stream . write ( numcol ( stats [ key ] . peerAdd , wPeerAdd ) )
        stream . write ( numcol ( stats [ key ] . peerMod , wPeerMod ) )
        stream . write ( numcol ( stats [ key ] . peerDel , wPeerDel ) )
        stream . write ( numcol ( stats [ key ] . peerErr , wPeerErr ) )
        stream . write ( numcol ( stats [ key ] . conflicts , wCol ) )
        stream . write ( numcol ( stats [ key ] . merged , wMrg ) )
        stream . write ( ' |\n' )
    stream . write ( hsline )
if totals :
    if title is None and not details :
        stream . write ( '+-' + '-' * tWid + '-+\n' )
    stream . write ( '| {0: ^{w}}' . format ( sumlist , w = tWid ) )
    stream . write ( ' |\n' )
    stream . write ( '+-' + '-' * tWid + '-+\n' )
return


signature:
def lookup ( cls , key , get = False ) :
    """Returns the label for a given Enum key"""

implementation:
if get :
    item = cls . _item_dict . get ( key )
    return item . name if item else key
return cls . _item_dict [ key ] . name


signature:
def verbose ( cls , key = False , default = '' ) :
    """Returns the verbose name for a given enum value"""

implementation:
if key is False :
    items = cls . _item_dict . values ( )
    return [ ( x . key , x . value ) for x in sorted ( items , key = lambda x : x . sort or x . key ) ]
item = cls . _item_dict . get ( key )
return item . value if item else default


signature:
def get_configured_dns ( ) :
    """Returns the configured DNS servers with the use f nmcli ."""

implementation:
ips = [ ]
try :
    output = subprocess . check_output ( [ 'nmcli' , 'device' , 'show' ] )
    output = output . decode ( 'utf-8' )
    for line in output . split ( '\n' ) :
        if 'DNS' in line :
            pattern = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"
            for hit in re . findall ( pattern , line ) :
                ips . append ( hit )
except FileNotFoundError :
    pass
return ips


signature:
def get_resolv_dns ( ) :
    """Returns the dns servers configured in / etc / resolv . conf"""

implementation:
result = [ ]
try :
    for line in open ( '/etc/resolv.conf' , 'r' ) :
        if line . startswith ( 'search' ) :
            result . append ( line . strip ( ) . split ( ' ' ) [ 1 ] )
except FileNotFoundError :
    pass
return result


signature:
def zone_transfer ( address , dns_name ) :
    """Tries to perform a zone transfer ."""

implementation:
ips = [ ]
try :
    print_notification ( "Attempting dns zone transfer for {} on {}" . format ( dns_name , address ) )
    z = dns . zone . from_xfr ( dns . query . xfr ( address , dns_name ) )
except dns . exception . FormError :
    print_notification ( "Zone transfer not allowed" )
    return ips
names = z . nodes . keys ( )
print_success ( "Zone transfer successfull for {}, found {} entries" . format ( address , len ( names ) ) )
for n in names :
    node = z [ n ]
    data = node . get_rdataset ( dns . rdataclass . IN , dns . rdatatype . A )
    if data : # TODO add hostnames to entries .
    # hostname = n . to _ text ( )
        for item in data . items :
            address = item . address
            ips . append ( address )
return ips


signature:
def resolve_domains ( domains , disable_zone = False ) :
    """Resolves the list of domains and returns the ips ."""

implementation:
dnsresolver = dns . resolver . Resolver ( )
ips = [ ]
for domain in domains :
    print_notification ( "Resolving {}" . format ( domain ) )
    try :
        result = dnsresolver . query ( domain , 'A' )
        for a in result . response . answer [ 0 ] :
            ips . append ( str ( a ) )
            if not disable_zone :
                ips . extend ( zone_transfer ( str ( a ) , domain ) )
    except dns . resolver . NXDOMAIN as e :
        print_error ( e )
return ips


signature:
def parse_ips ( ips , netmask , include_public ) :
    """Parses the list of ips , turns these into ranges based on the netmask given .
 Set include _ public to True to include public IP adresses ."""

implementation:
hs = HostSearch ( )
rs = RangeSearch ( )
ranges = [ ]
ips = list ( set ( ips ) )
included_ips = [ ]
print_success ( "Found {} ips" . format ( len ( ips ) ) )
for ip in ips :
    ip_address = ipaddress . ip_address ( ip )
    if include_public or ip_address . is_private : # To stop the screen filling with ranges .
        if len ( ips ) < 15 :
            print_success ( "Found ip: {}" . format ( ip ) )
        host = hs . id_to_object ( ip )
        host . add_tag ( 'dns_discover' )
        host . save ( )
        r = str ( ipaddress . IPv4Network ( "{}/{}" . format ( ip , netmask ) , strict = False ) )
        ranges . append ( r )
        included_ips . append ( ip )
    else :
        print_notification ( "Excluding ip {}" . format ( ip ) )
ranges = list ( set ( ranges ) )
print_success ( "Found {} ranges" . format ( len ( ranges ) ) )
for rng in ranges : # To stop the screen filling with ranges .
    if len ( ranges ) < 15 :
        print_success ( "Found range: {}" . format ( rng ) )
    r = rs . id_to_object ( rng )
    r . add_tag ( 'dns_discover' )
    r . save ( )
stats = { }
stats [ 'ips' ] = included_ips
stats [ 'ranges' ] = ranges
return stats


signature:
def create_connection ( conf ) :
    """Creates a connection based upon the given configuration object ."""

implementation:
host_config = { }
host_config [ 'hosts' ] = [ conf . get ( 'jackal' , 'host' ) ]
if int ( conf . get ( 'jackal' , 'use_ssl' ) ) :
    host_config [ 'use_ssl' ] = True
if conf . get ( 'jackal' , 'ca_certs' ) :
    host_config [ 'ca_certs' ] = conf . get ( 'jackal' , 'ca_certs' )
if int ( conf . get ( 'jackal' , 'client_certs' ) ) :
    host_config [ 'client_cert' ] = conf . get ( 'jackal' , 'client_cert' )
    host_config [ 'client_key' ] = conf . get ( 'jackal' , 'client_key' )
# Disable hostname checking for now .
host_config [ 'ssl_assert_hostname' ] = False
connections . create_connection ( ** host_config )


signature:
def search ( self , number = None , * args , ** kwargs ) :
    """Searches the elasticsearch instance to retrieve the requested documents ."""

implementation:
search = self . create_search ( * args , ** kwargs )
try :
    if number :
        response = search [ 0 : number ]
    else :
        args , _ = self . core_parser . parse_known_args ( )
        if args . number :
            response = search [ 0 : args . number ]
        else :
            response = search . scan ( )
    return [ hit for hit in response ]
except NotFoundError :
    print_error ( "The index was not found, have you initialized the index?" )
    return [ ]
except ( ConnectionError , TransportError ) :
    print_error ( "Cannot connect to elasticsearch" )
    return [ ]


signature:
def argument_search ( self ) :
    """Uses the command line arguments to fill the search function and call it ."""

implementation:
arguments , _ = self . argparser . parse_known_args ( )
return self . search ( ** vars ( arguments ) )


signature:
def count ( self , * args , ** kwargs ) :
    """Returns the number of results after filtering with the given arguments ."""

implementation:
search = self . create_search ( * args , ** kwargs )
try :
    return search . count ( )
except NotFoundError :
    print_error ( "The index was not found, have you initialized the index?" )
except ( ConnectionError , TransportError ) :
    print_error ( "Cannot connect to elasticsearch" )


signature:
def argument_count ( self ) :
    """Uses the command line arguments to fill the count function and call it ."""

implementation:
arguments , _ = self . argparser . parse_known_args ( )
return self . count ( ** vars ( arguments ) )


signature:
def get_pipe ( self , object_type ) :
    """Returns a generator that maps the input of the pipe to an elasticsearch object .
 Will call id _ to _ object if it cannot serialize the data from json ."""

implementation:
for line in sys . stdin :
    try :
        data = json . loads ( line . strip ( ) )
        obj = object_type ( ** data )
        yield obj
    except ValueError :
        yield self . id_to_object ( line . strip ( ) )


signature:
def id_to_object ( self , line ) :
    """Resolves an ip adres to a range object , creating it if it doesn ' t exists ."""

implementation:
result = Range . get ( line , ignore = 404 )
if not result :
    result = Range ( range = line )
    result . save ( )
return result


signature:
def argparser ( self ) :
    """Argparser option with search functionality specific for ranges ."""

implementation:
core_parser = self . core_parser
core_parser . add_argument ( '-r' , '--range' , type = str , help = "The range to search for use" )
return core_parser


signature:
def object_to_id ( self , obj ) :
    """Searches elasticsearch for objects with the same address , protocol , port and state ."""

implementation:
search = Service . search ( )
search = search . filter ( "term" , address = obj . address )
search = search . filter ( "term" , protocol = obj . protocol )
search = search . filter ( "term" , port = obj . port )
search = search . filter ( "term" , state = obj . state )
if search . count ( ) :
    result = search [ 0 ] . execute ( ) [ 0 ]
    return result . meta . id
else :
    return None


signature:
def id_to_object ( self , line ) :
    """Resolves the given id to a user object , if it doesn ' t exists it will be created ."""

implementation:
user = User . get ( line , ignore = 404 )
if not user :
    user = User ( username = line )
    user . save ( )
return user


signature:
def get_users ( self , * args , ** kwargs ) :
    """Retrieves the users from elastic ."""

implementation:
arguments , _ = self . argparser . parse_known_args ( )
if self . is_pipe and self . use_pipe :
    return self . get_pipe ( self . object_type )
elif arguments . tags or arguments . group or arguments . search or arguments . domain :
    return self . argument_search ( )
else :
    return self . search ( * args , ** kwargs )


signature:
def get_domains ( self ) :
    """Retrieves the domains of the users from elastic ."""

implementation:
search = User . search ( )
search . aggs . bucket ( 'domains' , 'terms' , field = 'domain' , order = { '_count' : 'desc' } , size = 100 )
response = search . execute ( )
return [ entry . key for entry in response . aggregations . domains . buckets ]


signature:
def find_object ( self , username , secret , domain = None , host_ip = None , service_id = None ) :
    """Searches elasticsearch for objects with the same username , password , optional domain , host _ ip and service _ id ."""

implementation:
# Not sure yet if this is advisable . . . Older passwords can be overwritten . . .
search = Credential . search ( )
search = search . filter ( "term" , username = username )
search = search . filter ( "term" , secret = secret )
if domain :
    search = search . filter ( "term" , domain = domain )
else :
    search = search . exclude ( "exists" , field = "domain" )
if host_ip :
    search = search . filter ( "term" , host_ip = host_ip )
else :
    search = search . exclude ( "exists" , field = "host_ip" )
if service_id :
    search = search . filter ( "term" , service_id = service_id )
else :
    search = search . exclude ( "exists" , field = "service_id" )
if search . count ( ) :
    result = search [ 0 ] . execute ( ) [ 0 ]
    return result
else :
    return None