signature:
def object_to_id ( self , obj ) :
    """Searches elasticsearch for objects with the same username , password , optional domain , host _ ip and service _ id ."""

implementation:
# Not sure yet if this is advisable . . . Older passwords can be overwritten . . .
search = Credential . search ( )
search = search . filter ( "term" , username = obj . username )
search = search . filter ( "term" , secret = obj . secret )
if obj . domain :
    search = search . filter ( "term" , domain = obj . domain )
else :
    search = search . exclude ( "exists" , field = "domain" )
if obj . host_ip :
    search = search . filter ( "term" , host_ip = obj . host_ip )
else :
    search = search . exclude ( "exists" , field = "host_ip" )
if obj . service_id :
    search = search . filter ( "term" , service_id = obj . service_id )
else :
    search = search . exclude ( "exists" , field = "service_id" )
if search . count ( ) :
    result = search [ 0 ] . execute ( ) [ 0 ]
    return result . meta . id
else :
    return None


signature:
def get_credentials ( self , * args , ** kwargs ) :
    """Retrieves the users from elastic ."""

implementation:
arguments , _ = self . argparser . parse_known_args ( )
if self . is_pipe and self . use_pipe :
    return self . get_pipe ( self . object_type )
elif arguments . tags or arguments . type or arguments . search or arguments . password or arguments . cracked or arguments . range or arguments . domain :
    return self . argument_search ( )
else :
    return self . search ( * args , ** kwargs )


signature:
def get_pipe ( self ) :
    """Returns a list that maps the input of the pipe to an elasticsearch object .
 Will call id _ to _ object if it cannot serialize the data from json ."""

implementation:
lines = [ ]
for line in sys . stdin :
    try :
        lines . append ( self . line_to_object ( line . strip ( ) ) )
    except ValueError :
        pass
    except KeyError :
        pass
return lines


signature:
def commands2tree ( self , adapter , session , commands ) :
    '''Consumes state . Command commands and converts them to an ET protocol tree'''

implementation:
# todo : trap errors . . .
hdrcmd = commands [ 0 ]
commands = commands [ 1 : ]
if hdrcmd . name != constants . CMD_SYNCHDR :
    raise common . InternalError ( 'unexpected first command "%s" (expected "%s")' % ( hdrcmd . name , constants . CMD_SYNCHDR ) )
if hdrcmd . version != constants . SYNCML_VERSION_1_2 :
    raise common . FeatureNotSupported ( 'unsupported SyncML version "%s"' % ( hdrcmd . version , ) )
xsync = ET . Element ( constants . NODE_SYNCML )
xhdr = ET . SubElement ( xsync , hdrcmd . name )
if hdrcmd . version == constants . SYNCML_VERSION_1_2 :
    ET . SubElement ( xhdr , 'VerDTD' ) . text = constants . SYNCML_DTD_VERSION_1_2
    ET . SubElement ( xhdr , 'VerProto' ) . text = hdrcmd . version
ET . SubElement ( xhdr , 'SessionID' ) . text = hdrcmd . sessionID
ET . SubElement ( xhdr , 'MsgID' ) . text = hdrcmd . msgID
xsrc = ET . SubElement ( xhdr , 'Source' )
ET . SubElement ( xsrc , 'LocURI' ) . text = hdrcmd . source
if hdrcmd . sourceName is not None :
    ET . SubElement ( xsrc , 'LocName' ) . text = hdrcmd . sourceName
xtgt = ET . SubElement ( xhdr , 'Target' )
ET . SubElement ( xtgt , 'LocURI' ) . text = hdrcmd . target
if hdrcmd . targetName is not None :
    ET . SubElement ( xtgt , 'LocName' ) . text = hdrcmd . targetName
if hdrcmd . respUri is not None :
    ET . SubElement ( xhdr , 'RespURI' ) . text = hdrcmd . respUri
if hdrcmd . auth is not None and not session . authAccepted :
    if hdrcmd . auth != constants . NAMESPACE_AUTH_BASIC :
        raise NotImplementedError ( 'auth method "%s"' % ( common . auth2string ( hdrcmd . auth ) , ) )
    if hdrcmd . auth == constants . NAMESPACE_AUTH_BASIC :
        xcred = ET . SubElement ( xhdr , 'Cred' )
        xmeta = ET . SubElement ( xcred , 'Meta' )
        ET . SubElement ( xmeta , 'Format' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = 'b64'
        ET . SubElement ( xmeta , 'Type' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = hdrcmd . auth
        ET . SubElement ( xcred , 'Data' ) . text = base64 . b64encode ( '%s:%s' % ( adapter . peer . username , adapter . peer . password ) )
if hdrcmd . maxMsgSize is not None or hdrcmd . maxObjSize is not None :
    xmeta = ET . SubElement ( xhdr , 'Meta' )
    if hdrcmd . maxMsgSize is not None :
        ET . SubElement ( xmeta , 'MaxMsgSize' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = hdrcmd . maxMsgSize
    if hdrcmd . maxObjSize is not None :
        ET . SubElement ( xmeta , 'MaxObjSize' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = hdrcmd . maxObjSize
xbody = ET . SubElement ( xsync , constants . NODE_SYNCBODY )
for cmdidx , cmd in enumerate ( commands ) :
    xcmd = ET . SubElement ( xbody , cmd . name )
    if cmd . cmdID is not None :
        ET . SubElement ( xcmd , 'CmdID' ) . text = cmd . cmdID
    if cmd . name == constants . CMD_ALERT :
        ET . SubElement ( xcmd , 'Data' ) . text = str ( cmd . data )
        xitem = ET . SubElement ( xcmd , 'Item' )
        ET . SubElement ( ET . SubElement ( xitem , 'Source' ) , 'LocURI' ) . text = cmd . source
        ET . SubElement ( ET . SubElement ( xitem , 'Target' ) , 'LocURI' ) . text = cmd . target
        if cmd . lastAnchor is not None or cmd . nextAnchor is not None or cmd . maxObjSize is not None :
            xmeta = ET . SubElement ( xitem , 'Meta' )
            xanch = ET . SubElement ( xmeta , 'Anchor' , { 'xmlns' : constants . NAMESPACE_METINF } )
            if cmd . lastAnchor is not None :
                ET . SubElement ( xanch , 'Last' ) . text = cmd . lastAnchor
            if cmd . nextAnchor is not None :
                ET . SubElement ( xanch , 'Next' ) . text = cmd . nextAnchor
            if cmd . maxObjSize is not None :
                ET . SubElement ( xmeta , 'MaxObjSize' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = cmd . maxObjSize
        continue
    if cmd . name == constants . CMD_STATUS :
        ET . SubElement ( xcmd , 'MsgRef' ) . text = cmd . msgRef
        ET . SubElement ( xcmd , 'CmdRef' ) . text = cmd . cmdRef
        ET . SubElement ( xcmd , 'Cmd' ) . text = cmd . statusOf
        if cmd . sourceRef is not None :
            ET . SubElement ( xcmd , 'SourceRef' ) . text = cmd . sourceRef
        if cmd . targetRef is not None :
            ET . SubElement ( xcmd , 'TargetRef' ) . text = cmd . targetRef
        ET . SubElement ( xcmd , 'Data' ) . text = cmd . statusCode
        if cmd . nextAnchor is not None or cmd . lastAnchor is not None :
            xdata = ET . SubElement ( ET . SubElement ( xcmd , 'Item' ) , 'Data' )
            xanch = ET . SubElement ( xdata , 'Anchor' , { 'xmlns' : constants . NAMESPACE_METINF } )
            if cmd . lastAnchor is not None :
                ET . SubElement ( xanch , 'Last' ) . text = cmd . lastAnchor
            if cmd . nextAnchor is not None :
                ET . SubElement ( xanch , 'Next' ) . text = cmd . nextAnchor
        # NOTE : this is NOT standard SyncML . . .
        if cmd . errorCode is not None or cmd . errorMsg is not None :
            xerr = ET . SubElement ( xcmd , 'Error' )
            if cmd . errorCode is not None :
                ET . SubElement ( xerr , 'Code' ) . text = cmd . errorCode
            if cmd . errorMsg is not None :
                ET . SubElement ( xerr , 'Message' ) . text = cmd . errorMsg
            if cmd . errorTrace is not None :
                ET . SubElement ( xerr , 'Trace' ) . text = cmd . errorTrace
        continue
    if cmd . name in [ constants . CMD_GET , constants . CMD_PUT ] :
        ET . SubElement ( ET . SubElement ( xcmd , 'Meta' ) , 'Type' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = cmd . type
        if cmd . source is not None or cmd . target is not None or cmd . data :
            xitem = ET . SubElement ( xcmd , 'Item' )
        if cmd . source is not None :
            xsrc = ET . SubElement ( xitem , 'Source' )
            ET . SubElement ( xsrc , 'LocURI' ) . text = cmd . source
            ET . SubElement ( xsrc , 'LocName' ) . text = cmd . source
        if cmd . target is not None :
            xtgt = ET . SubElement ( xitem , 'Target' )
            ET . SubElement ( xtgt , 'LocURI' ) . text = cmd . target
            ET . SubElement ( xtgt , 'LocName' ) . text = cmd . target
        if cmd . data is not None :
            if isinstance ( cmd . data , basestring ) :
                ET . SubElement ( xitem , 'Data' ) . text = cmd . data
            else :
                ET . SubElement ( xitem , 'Data' ) . append ( cmd . data )
        continue
    if cmd . name == constants . CMD_RESULTS :
        ET . SubElement ( xcmd , 'MsgRef' ) . text = cmd . msgRef
        ET . SubElement ( xcmd , 'CmdRef' ) . text = cmd . cmdRef
        ET . SubElement ( ET . SubElement ( xcmd , 'Meta' ) , 'Type' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = cmd . type
        xitem = ET . SubElement ( xcmd , 'Item' )
        xsrc = ET . SubElement ( xitem , 'Source' )
        ET . SubElement ( xsrc , 'LocURI' ) . text = cmd . source
        ET . SubElement ( xsrc , 'LocName' ) . text = cmd . source
        if cmd . data is not None :
            if isinstance ( cmd . data , basestring ) :
                ET . SubElement ( xitem , 'Data' ) . text = cmd . data
            else :
                ET . SubElement ( xitem , 'Data' ) . append ( cmd . data )
        continue
    if cmd . name == constants . CMD_SYNC :
        ET . SubElement ( ET . SubElement ( xcmd , 'Source' ) , 'LocURI' ) . text = cmd . source
        ET . SubElement ( ET . SubElement ( xcmd , 'Target' ) , 'LocURI' ) . text = cmd . target
        if cmd . noc is not None :
            ET . SubElement ( xcmd , 'NumberOfChanges' ) . text = cmd . noc
        if cmd . data is not None :
            for scmd in cmd . data :
                xscmd = ET . SubElement ( xcmd , scmd . name )
                if scmd . cmdID is not None :
                    ET . SubElement ( xscmd , 'CmdID' ) . text = scmd . cmdID
                if scmd . type is not None or ( scmd . format is not None and scmd . format != constants . FORMAT_AUTO ) :
                    xsmeta = ET . SubElement ( xscmd , 'Meta' )
                    # todo : implement auto encoding determination . . .
                    # ( the current implementation just lets XML encoding do it ,
                    # which is for most things good enough , but not so good
                    # for sequences that need a large amount escaping such as
                    # binary data . . . )
                    if scmd . format is not None and scmd . format != constants . FORMAT_AUTO :
                        ET . SubElement ( xsmeta , 'Format' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = scmd . format
                    if scmd . type is not None :
                        ET . SubElement ( xsmeta , 'Type' , { 'xmlns' : constants . NAMESPACE_METINF } ) . text = scmd . type
                xsitem = ET . SubElement ( xscmd , 'Item' )
                if scmd . source is not None :
                    ET . SubElement ( ET . SubElement ( xsitem , 'Source' ) , 'LocURI' ) . text = scmd . source
                if scmd . sourceParent is not None :
                    ET . SubElement ( ET . SubElement ( xsitem , 'SourceParent' ) , 'LocURI' ) . text = scmd . sourceParent
                if scmd . target is not None :
                    ET . SubElement ( ET . SubElement ( xsitem , 'Target' ) , 'LocURI' ) . text = scmd . target
                if scmd . targetParent is not None :
                    ET . SubElement ( ET . SubElement ( xsitem , 'TargetParent' ) , 'LocURI' ) . text = scmd . targetParent
                if scmd . data is not None :
                    if isinstance ( scmd . data , basestring ) :
                        ET . SubElement ( xsitem , 'Data' ) . text = scmd . data
                    else :
                        ET . SubElement ( xsitem , 'Data' ) . append ( scmd . data )
        continue
    if cmd . name == constants . CMD_MAP :
        ET . SubElement ( ET . SubElement ( xcmd , 'Source' ) , 'LocURI' ) . text = cmd . source
        ET . SubElement ( ET . SubElement ( xcmd , 'Target' ) , 'LocURI' ) . text = cmd . target
        if cmd . sourceItem is not None or cmd . targetItem is not None :
            xitem = ET . SubElement ( xcmd , constants . CMD_MAPITEM )
            if cmd . sourceItem is not None :
                ET . SubElement ( ET . SubElement ( xitem , 'Source' ) , 'LocURI' ) . text = cmd . sourceItem
            if cmd . targetItem is not None :
                ET . SubElement ( ET . SubElement ( xitem , 'Target' ) , 'LocURI' ) . text = cmd . targetItem
        continue
    if cmd . name == constants . CMD_FINAL :
        if cmdidx + 1 < len ( commands ) :
            raise common . InternalError ( 'command "%s" not at tail end of commands' % ( cmd . name , ) )
        continue
    raise common . InternalError ( 'unexpected command "%s"' % ( cmd . name , ) )
return xsync


signature:
def tree2commands ( self , adapter , session , lastcmds , xsync ) :
    '''Consumes an ET protocol tree and converts it to state . Command commands'''

implementation:
# do some preliminary sanity checks . . .
# todo : do i really want to be using assert statements ? . . .
assert xsync . tag == constants . NODE_SYNCML
assert len ( xsync ) == 2
assert xsync [ 0 ] . tag == constants . CMD_SYNCHDR
assert xsync [ 1 ] . tag == constants . NODE_SYNCBODY
version = xsync [ 0 ] . findtext ( 'VerProto' )
if version != constants . SYNCML_VERSION_1_2 :
    raise common . FeatureNotSupported ( 'unsupported SyncML version "%s" (expected "%s")' % ( version , constants . SYNCML_VERSION_1_2 ) )
verdtd = xsync [ 0 ] . findtext ( 'VerDTD' )
if verdtd != constants . SYNCML_DTD_VERSION_1_2 :
    raise common . FeatureNotSupported ( 'unsupported SyncML DTD version "%s" (expected "%s")' % ( verdtd , constants . SYNCML_DTD_VERSION_1_2 ) )
ret = self . initialize ( adapter , session , xsync )
hdrcmd = ret [ 0 ]
if session . isServer :
    log . debug ( 'received request SyncML message from "%s" (s%s.m%s)' , hdrcmd . target , hdrcmd . sessionID , hdrcmd . msgID )
else :
    log . debug ( 'received response SyncML message from "%s" (s%s.m%s)' , lastcmds [ 0 ] . target , lastcmds [ 0 ] . sessionID , lastcmds [ 0 ] . msgID )
try :
    return self . _tree2commands ( adapter , session , lastcmds , xsync , ret )
except Exception , e :
    if not session . isServer :
        raise
    # TODO : make this configurable as to whether or not any error
    # is sent back to the peer as a SyncML " standardized " error
    # status . . .
    code = '%s.%s' % ( e . __class__ . __module__ , e . __class__ . __name__ )
    msg = '' . join ( traceback . format_exception_only ( type ( e ) , e ) ) . strip ( )
    log . exception ( 'failed while interpreting command tree: %s' , msg )
    # TODO : for some reason , the active exception is not being logged . . .
    return [ hdrcmd , state . Command ( name = constants . CMD_STATUS , cmdID = '1' , msgRef = session . pendingMsgID , cmdRef = 0 , sourceRef = xsync [ 0 ] . findtext ( 'Source/LocURI' ) , targetRef = xsync [ 0 ] . findtext ( 'Target/LocURI' ) , statusOf = constants . CMD_SYNCHDR , statusCode = constants . STATUS_COMMAND_FAILED , errorCode = code , errorMsg = msg , errorTrace = '' . join ( traceback . format_exception ( type ( e ) , e , sys . exc_info ( ) [ 2 ] ) ) , ) , state . Command ( name = constants . CMD_FINAL ) ]


signature:
def dumps ( self , contentType = None , version = None ) :
    '''[ OPTIONAL ] Identical to : meth : ` dump ` , except the serialized form
 is returned as a string representation . As documented in
 : meth : ` dump ` , the return value can optionally be a three - element
 tuple of ( contentType , version , data ) if the provided content - type
 should be overridden or enhanced . The default implementation just
 wraps : meth : ` dump ` .'''

implementation:
buf = six . StringIO ( )
ret = self . dump ( buf , contentType , version )
if ret is None :
    return buf . getvalue ( )
return ( ret [ 0 ] , ret [ 1 ] , buf . getvalue ( ) )


signature:
def loads ( cls , data , contentType = None , version = None ) :
    '''[ OPTIONAL ] Identical to : meth : ` load ` , except the serialized form
 is provided as a string representation in ` data ` instead of as a
 stream . The default implementation just wraps : meth : ` load ` .'''

implementation:
buf = six . StringIO ( data )
return cls . load ( buf , contentType , version )


signature:
def dumpsItem ( self , item , contentType = None , version = None ) :
    '''[ OPTIONAL ] Identical to : meth : ` dump ` , except the serialized form
 is returned as a string representation . As documented in
 : meth : ` dump ` , the return value can optionally be a three - element
 tuple of ( contentType , version , data ) if the provided content - type
 should be overridden or enhanced . The default implementation just
 wraps : meth : ` dump ` .'''

implementation:
buf = six . StringIO ( )
ret = self . dumpItem ( item , buf , contentType , version )
if ret is None :
    return buf . getvalue ( )
return ( ret [ 0 ] , ret [ 1 ] , buf . getvalue ( ) )


signature:
def loadsItem ( self , data , contentType = None , version = None ) :
    '''[ OPTIONAL ] Identical to : meth : ` loadItem ` , except the serialized
 form is provided as a string representation in ` data ` instead of
 as a stream . The default implementation just wraps
 : meth : ` loadItem ` .'''

implementation:
buf = six . StringIO ( data )
return self . loadItem ( buf , contentType , version )


signature:
def matchItem ( self , item ) :
    '''[ OPTIONAL ] Attempts to find the specified item and returns an item
 that describes the same object although it ' s specific properties
 may be different . For example , a contact whose name is an
 identical match , but whose telephone number has changed would
 return the matched item . ` ` None ` ` should be returned if no match
 is found , otherwise the item that ` item ` matched should be
 returned .
 This is used primarily when a slow - sync is invoked and objects
 that exist in both peers should not be replicated .
 Note that * * NO * * merging of the items ' properties should be done ;
 that will be initiated via a separate call to : meth : ` mergeItems ` .
 This method by default will iterate over all items ( by calling
 : meth : ` getAllItems ` ) and compare them using ` ` cmp ( ) ` ` . This means
 that if the items managed by this agent implement the ` ` _ _ eq _ _ ` `
 or ` ` _ _ cmp _ _ ` ` methods , then matching items will be detected and
 returned . Otherwise , any items that exist in both peers will be
 duplicated on slow - sync .
 Sub - classes * should * implement a more efficient method of finding
 matching items .
 See : doc : ` . . / merging ` for details .'''

implementation:
for match in self . getAllItems ( ) :
    if cmp ( match , item ) == 0 :
        return match
return None


signature:
def initialize_indices ( ) :
    """Initializes the indices"""

implementation:
Host . init ( )
Range . init ( )
Service . init ( )
User . init ( )
Credential . init ( )
Log . init ( )


signature:
def toSyncML ( self , nodeName = None , uniqueVerCt = False ) :
    '''Returns an ElementTree node representing this ContentTypeInfo . If
 ` nodeName ` is not None , then it will be used as the containing
 element node name ( this is useful , for example , to differentiate
 between a standard content - type and a preferred content - type ) . If
 ` uniqueVerCt ` is True , then an array of elements will be returned
 instead of a single element with multiple VerCT elements ( for
 content - types that support multiple versions ) .'''

implementation:
if uniqueVerCt :
    ret = [ ]
    for v in self . versions :
        tmp = ET . Element ( nodeName or 'ContentType' )
        ET . SubElement ( tmp , 'CTType' ) . text = self . ctype
        ET . SubElement ( tmp , 'VerCT' ) . text = v
        ret . append ( tmp )
    return ret
ret = ET . Element ( nodeName or 'ContentType' )
ET . SubElement ( ret , 'CTType' ) . text = self . ctype
for v in self . versions :
    ET . SubElement ( ret , 'VerCT' ) . text = v
return ret


signature:
def parse_single_computer ( entry ) :
    """Parse the entry into a computer object ."""

implementation:
computer = Computer ( dns_hostname = get_field ( entry , 'dNSHostName' ) , description = get_field ( entry , 'description' ) , os = get_field ( entry , 'operatingSystem' ) , group_id = get_field ( entry , 'primaryGroupID' ) )
try :
    ip = str ( ipaddress . ip_address ( get_field ( entry , 'IPv4' ) ) )
except ValueError :
    ip = ''
if ip :
    computer . ip = ip
elif computer . dns_hostname :
    computer . ip = resolve_ip ( computer . dns_hostname )
return computer


signature:
def parse_domain_computers ( filename ) :
    """Parse the file and extract the computers , import the computers that resolve into jackal ."""

implementation:
with open ( filename ) as f :
    data = json . loads ( f . read ( ) )
hs = HostSearch ( )
count = 0
entry_count = 0
print_notification ( "Parsing {} entries" . format ( len ( data ) ) )
for system in data :
    entry_count += 1
    parsed = parse_single_computer ( system )
    if parsed . ip :
        try :
            host = hs . id_to_object ( parsed . ip )
            host . description . append ( parsed . description )
            host . hostname . append ( parsed . dns_hostname )
            if parsed . os :
                host . os = parsed . os
            host . domain_controller = parsed . dc
            host . add_tag ( 'domaindump' )
            host . save ( )
            count += 1
        except ValueError :
            pass
    sys . stdout . write ( '\r' )
    sys . stdout . write ( "[{}/{}] {} resolved" . format ( entry_count , len ( data ) , count ) )
    sys . stdout . flush ( )
sys . stdout . write ( '\r' )
return count


signature:
def parse_user ( entry , domain_groups ) :
    """Parses a single entry from the domaindump"""

implementation:
result = { }
distinguished_name = get_field ( entry , 'distinguishedName' )
result [ 'domain' ] = "." . join ( distinguished_name . split ( ',DC=' ) [ 1 : ] )
result [ 'name' ] = get_field ( entry , 'name' )
result [ 'username' ] = get_field ( entry , 'sAMAccountName' )
result [ 'description' ] = get_field ( entry , 'description' )
result [ 'sid' ] = get_field ( entry , 'objectSid' ) . split ( '-' ) [ - 1 ]
primary_group = get_field ( entry , 'primaryGroupID' )
member_of = entry [ 'attributes' ] . get ( 'memberOf' , [ ] )
groups = [ ]
for member in member_of :
    for e in member . split ( ',' ) :
        if e . startswith ( 'CN=' ) :
            groups . append ( e [ 3 : ] )
groups . append ( domain_groups . get ( primary_group , '' ) )
result [ 'groups' ] = groups
flags = [ ]
try :
    uac = int ( get_field ( entry , 'userAccountControl' ) )
    for flag , value in uac_flags . items ( ) :
        if uac & value :
            flags . append ( flag )
except ValueError :
    pass
result [ 'flags' ] = flags
return result


signature:
def parse_domain_users ( domain_users_file , domain_groups_file ) :
    """Parses the domain users and groups files ."""

implementation:
with open ( domain_users_file ) as f :
    users = json . loads ( f . read ( ) )
domain_groups = { }
if domain_groups_file :
    with open ( domain_groups_file ) as f :
        groups = json . loads ( f . read ( ) )
        for group in groups :
            sid = get_field ( group , 'objectSid' )
            domain_groups [ int ( sid . split ( '-' ) [ - 1 ] ) ] = get_field ( group , 'cn' )
user_search = UserSearch ( )
count = 0
total = len ( users )
print_notification ( "Importing {} users" . format ( total ) )
for entry in users :
    result = parse_user ( entry , domain_groups )
    user = user_search . id_to_object ( result [ 'username' ] )
    user . name = result [ 'name' ]
    user . domain . append ( result [ 'domain' ] )
    user . description = result [ 'description' ]
    user . groups . extend ( result [ 'groups' ] )
    user . flags . extend ( result [ 'flags' ] )
    user . sid = result [ 'sid' ]
    user . add_tag ( "domaindump" )
    user . save ( )
    count += 1
    sys . stdout . write ( '\r' )
    sys . stdout . write ( "[{}/{}]" . format ( count , total ) )
    sys . stdout . flush ( )
sys . stdout . write ( '\r' )
return count


signature:
def import_domaindump ( ) :
    """Parses ldapdomaindump files and stores hosts and users in elasticsearch ."""

implementation:
parser = argparse . ArgumentParser ( description = "Imports users, groups and computers result files from the ldapdomaindump tool, will resolve the names from domain_computers output for IPs" )
parser . add_argument ( "files" , nargs = '+' , help = "The domaindump files to import" )
arguments = parser . parse_args ( )
domain_users_file = ''
domain_groups_file = ''
computer_count = 0
user_count = 0
stats = { }
for filename in arguments . files :
    if filename . endswith ( 'domain_computers.json' ) :
        print_notification ( 'Parsing domain computers' )
        computer_count = parse_domain_computers ( filename )
        if computer_count :
            stats [ 'hosts' ] = computer_count
            print_success ( "{} hosts imported" . format ( computer_count ) )
    elif filename . endswith ( 'domain_users.json' ) :
        domain_users_file = filename
    elif filename . endswith ( 'domain_groups.json' ) :
        domain_groups_file = filename
if domain_users_file :
    print_notification ( "Parsing domain users" )
    user_count = parse_domain_users ( domain_users_file , domain_groups_file )
    if user_count :
        print_success ( "{} users imported" . format ( user_count ) )
        stats [ 'users' ] = user_count
Logger ( ) . log ( "import_domaindump" , 'Imported domaindump, found {} user, {} systems' . format ( user_count , computer_count ) , stats )


signature:
def autocomplete ( query , country = None , hurricanes = False , cities = True , timeout = 5 ) :
    """Make an autocomplete API request
 This can be used to find cities and / or hurricanes by name
 : param string query : city
 : param string country : restrict search to a specific country . Must be a two letter country code
 : param boolean hurricanes : whether to search for hurricanes or not
 : param boolean cities : whether to search for cities or not
 : param integer timeout : timeout of the api request
 : returns : result of the autocomplete API request
 : rtype : dict"""

implementation:
data = { }
data [ 'query' ] = quote ( query )
data [ 'country' ] = country or ''
data [ 'hurricanes' ] = 1 if hurricanes else 0
data [ 'cities' ] = 1 if cities else 0
data [ 'format' ] = 'JSON'
r = requests . get ( AUTOCOMPLETE_URL . format ( ** data ) , timeout = timeout )
results = json . loads ( r . content ) [ 'RESULTS' ]
return results


signature:
def request ( key , features , query , timeout = 5 ) :
    """Make an API request
 : param string key : API key to use
 : param list features : features to request . It must be a subset of : data : ` FEATURES `
 : param string query : query to send
 : param integer timeout : timeout of the request
 : returns : result of the API request
 : rtype : dict"""

implementation:
data = { }
data [ 'key' ] = key
data [ 'features' ] = '/' . join ( [ f for f in features if f in FEATURES ] )
data [ 'query' ] = quote ( query )
data [ 'format' ] = 'json'
r = requests . get ( API_URL . format ( ** data ) , timeout = timeout )
results = json . loads ( _unicode ( r . content ) )
return results


signature:
def _unicode ( string ) :
    """Try to convert a string to unicode using different encodings"""

implementation:
for encoding in [ 'utf-8' , 'latin1' ] :
    try :
        result = unicode ( string , encoding )
        return result
    except UnicodeDecodeError :
        pass
result = unicode ( string , 'utf-8' , 'replace' )
return result


signature:
def http_get_provider ( provider , request_url , params , token_secret , token_cookie = None ) :
    '''Handle HTTP GET requests on an authentication endpoint .
 Authentication flow begins when ` ` params ` ` has a ` ` login ` ` key with a value
 of ` ` start ` ` . For instance , ` ` / auth / twitter ? login = start ` ` .
 : param str provider : An provider to obtain a user ID from .
 : param str request _ url : The authentication endpoint / callback .
 : param dict params : GET parameters from the query string .
 : param str token _ secret : An app secret to encode / decode JSON web tokens .
 : param str token _ cookie : The current JSON web token , if available .
 : return : A dict containing any of the following possible keys :
 ` ` status ` ` : an HTTP status code the server should sent
 ` ` redirect ` ` : where the client should be directed to continue the flow
 ` ` set _ token _ cookie ` ` : contains a JSON web token and should be stored by
 the client and passed in the next call .
 ` ` provider _ user _ id ` ` : the user ID from the login provider
 ` ` provider _ user _ name ` ` : the user name from the login provider'''

implementation:
if not validate_provider ( provider ) :
    raise InvalidUsage ( 'Provider not supported' )
klass = getattr ( socialauth . providers , provider . capitalize ( ) )
provider = klass ( request_url , params , token_secret , token_cookie )
if provider . status == 302 :
    ret = dict ( status = 302 , redirect = provider . redirect )
    tc = getattr ( provider , 'set_token_cookie' , None )
    if tc is not None :
        ret [ 'set_token_cookie' ] = tc
    return ret
if provider . status == 200 and provider . user_id is not None :
    ret = dict ( status = 200 , provider_user_id = provider . user_id )
    if provider . user_name is not None :
        ret [ 'provider_user_name' ] = provider . user_name
    return ret
raise InvalidUsage ( 'Invalid request' )


signature:
def to_json ( self ) :
    """Method to call to get a serializable object for json . dump or jsonify based on the target
 : return : dict"""

implementation:
if self . subreference is not None :
    return { "source" : self . objectId , "selector" : { "type" : "FragmentSelector" , "conformsTo" : "http://ontology-dts.org/terms/subreference" , "value" : self . subreference } }
else :
    return { "source" : self . objectId }


signature:
def read ( self ) :
    """Read the contents of the Annotation Resource
 : return : the contents of the resource
 : rtype : str or bytes or flask . response"""

implementation:
if not self . __content__ :
    self . __retriever__ = self . __resolver__ . resolve ( self . uri )
    self . __content__ , self . __mimetype__ = self . __retriever__ . read ( self . uri )
return self . __content__


signature:
def build_index_and_mapping ( triples ) :
    """index all triples into indexes and return their mappings"""

implementation:
ents = bidict ( )
rels = bidict ( )
ent_id = 0
rel_id = 0
collected = [ ]
for t in triples :
    for e in ( t . head , t . tail ) :
        if e not in ents :
            ents [ e ] = ent_id
            ent_id += 1
    if t . relation not in rels :
        rels [ t . relation ] = rel_id
        rel_id += 1
    collected . append ( kgedata . TripleIndex ( ents [ t . head ] , rels [ t . relation ] , ents [ t . tail ] ) )
return collected , ents , rels


signature:
def recover_triples_from_mapping ( indexes , ents : bidict , rels : bidict ) :
    """recover triples from mapping ."""

implementation:
triples = [ ]
for t in indexes :
    triples . append ( kgedata . Triple ( ents . inverse [ t . head ] , rels . inverse [ t . relation ] , ents . inverse [ t . tail ] ) )
return triples


signature:
def split_golden_set ( triples , valid_ratio , test_ratio ) :
    """Split the data into train / valid / test sets ."""

implementation:
assert valid_ratio >= 0.0
assert test_ratio >= 0.0
num_valid = int ( len ( triples ) * valid_ratio )
num_test = int ( len ( triples ) * test_ratio )
valid_set = triples [ : num_valid ]
test_set = triples [ num_valid : num_valid + num_test ]
train_set = triples [ num_valid + num_test : ]
assert len ( valid_set ) + len ( test_set ) + len ( train_set ) == len ( triples )
return train_set , valid_set , test_set


signature:
def _transform_triple_numpy ( x ) :
    """Transform triple index into a 1 - D numpy array ."""

implementation: return np . array ( [ x . head , x . relation , x . tail ] , dtype = np . int64 )


signature:
def pack_triples_numpy ( triples ) :
    """Packs a list of triple indexes into a 2D numpy array ."""

implementation:
if len ( triples ) == 0 :
    return np . array ( [ ] , dtype = np . int64 )
return np . stack ( list ( map ( _transform_triple_numpy , triples ) ) , axis = 0 )


signature:
def remove_near_duplicate_relation ( triples , threshold = 0.97 ) :
    """If entity pairs in a relation is as close as another relations , only keep one relation of such set ."""

implementation:
logging . debug ( "remove duplicate" )
_assert_threshold ( threshold )
duplicate_rel_counter = defaultdict ( list )
relations = set ( )
for t in triples :
    duplicate_rel_counter [ t . relation ] . append ( f"{t.head} {t.tail}" )
    relations . add ( t . relation )
relations = list ( relations )
num_triples = len ( triples )
removal_relation_set = set ( )
for rel , values in duplicate_rel_counter . items ( ) :
    duplicate_rel_counter [ rel ] = Superminhash ( values )
for i in relations :
    for j in relations :
        if i == j or i in removal_relation_set or j in removal_relation_set :
            continue
        close_relations = [ i ]
        if _set_close_to ( duplicate_rel_counter [ i ] , duplicate_rel_counter [ j ] , threshold ) :
            close_relations . append ( j )
    if len ( close_relations ) > 1 :
        close_relations . pop ( np . random . randint ( len ( close_relations ) ) )
        removal_relation_set |= set ( close_relations )
logging . info ( "Removing {} relations: {}" . format ( len ( removal_relation_set ) , str ( removal_relation_set ) ) )
return list ( filterfalse ( lambda x : x . relation in removal_relation_set , triples ) )


signature:
def remove_direct_link_triples ( train , valid , test ) :
    """Remove direct links in the training sets ."""

implementation:
pairs = set ( )
merged = valid + test
for t in merged :
    pairs . add ( ( t . head , t . tail ) )
filtered = filterfalse ( lambda t : ( t . head , t . tail ) in pairs or ( t . tail , t . head ) in pairs , train )
return list ( filtered )


signature:
def shrink_indexes_in_place ( self , triples ) :
    """Uses a union find to find segment ."""

implementation:
_ent_roots = self . UnionFind ( self . _ent_id )
_rel_roots = self . UnionFind ( self . _rel_id )
for t in triples :
    _ent_roots . add ( t . head )
    _ent_roots . add ( t . tail )
    _rel_roots . add ( t . relation )
for i , t in enumerate ( triples ) :
    h = _ent_roots . find ( t . head )
    r = _rel_roots . find ( t . relation )
    t = _ent_roots . find ( t . tail )
    triples [ i ] = kgedata . TripleIndex ( h , r , t )
ents = bidict ( )
available_ent_idx = 0
for previous_idx , ent_exist in enumerate ( _ent_roots . roots ( ) ) :
    if not ent_exist :
        self . _ents . inverse . pop ( previous_idx )
    else :
        ents [ self . _ents . inverse [ previous_idx ] ] = available_ent_idx
    available_ent_idx += 1
rels = bidict ( )
available_rel_idx = 0
for previous_idx , rel_exist in enumerate ( _rel_roots . roots ( ) ) :
    if not rel_exist :
        self . _rels . inverse . pop ( previous_idx )
    else :
        rels [ self . _rels . inverse [ previous_idx ] ] = available_rel_idx
    available_rel_idx += 1
self . _ents = ents
self . _rels = rels
self . _ent_id = available_ent_idx
self . _rel_id = available_rel_idx


signature:
def freeze ( self ) :
    """Create a usable data structure for serializing ."""

implementation:
data = super ( IndexBuilder , self ) . freeze ( )
try : # Sphinx > = 1.5 format
# Due to changes from github . com / sphinx - doc / sphinx / pull / 2454
    base_file_names = data [ 'docnames' ]
except KeyError : # Sphinx < 1.5 format
    base_file_names = data [ 'filenames' ]
store = { }
c = itertools . count ( )
for prefix , items in iteritems ( data [ 'objects' ] ) :
    for name , ( index , typeindex , _ , shortanchor ) in iteritems ( items ) :
        objtype = data [ 'objtypes' ] [ typeindex ]
        if objtype . startswith ( 'cpp:' ) :
            split = name . rsplit ( '::' , 1 )
            if len ( split ) != 2 :
                warnings . warn ( "What's up with %s?" % str ( ( prefix , name , objtype ) ) )
                continue
            prefix , name = split
            last_prefix = prefix . split ( '::' ) [ - 1 ]
        else :
            last_prefix = prefix . split ( '.' ) [ - 1 ]
        store [ next ( c ) ] = { 'filename' : base_file_names [ index ] , 'objtype' : objtype , 'prefix' : prefix , 'last_prefix' : last_prefix , 'name' : name , 'shortanchor' : shortanchor , }
data . update ( { 'store' : store } )
return data


signature:
def log_entity_creation ( entity , params = None ) :
    """Logs an entity creation"""

implementation:
p = { 'entity' : entity }
if params :
    p [ 'params' ] = params
_log ( TYPE_CODES . CREATE , p )


signature:
def log_entity_deletion ( entity , params = None ) :
    """Logs an entity creation"""

implementation:
p = { 'entity' : entity }
if params :
    p [ 'params' ] = params
_log ( TYPE_CODES . DELETE , p )


signature:
def log_operation ( entities , operation_name , params = None ) :
    """Logs an operation done on an entity , possibly with other arguments"""

implementation:
if isinstance ( entities , ( list , tuple ) ) :
    entities = list ( entities )
else :
    entities = [ entities ]
p = { 'name' : operation_name , 'on' : entities }
if params :
    p [ 'params' ] = params
_log ( TYPE_CODES . OPERATION , p )


signature:
def log_state ( entity , state ) :
    """Logs a new state of an entity"""

implementation:
p = { 'on' : entity , 'state' : state }
_log ( TYPE_CODES . STATE , p )


signature:
def log_update ( entity , update ) :
    """Logs an update done on an entity"""

implementation:
p = { 'on' : entity , 'update' : update }
_log ( TYPE_CODES . UPDATE , p )


signature:
def log_error ( error , result ) :
    """Logs an error"""

implementation:
p = { 'error' : error , 'result' : result }
_log ( TYPE_CODES . ERROR , p )


signature:
def dict_cursor ( func ) :
    """Decorator that provides a dictionary cursor to the calling function
 Adds the cursor as the second argument to the calling functions
 Requires that the function being decorated is an instance of a class or object
 that yields a cursor from a get _ cursor ( cursor _ type = CursorType . DICT ) coroutine or provides such an object
 as the first argument in its signature
 Yields :
 A client - side dictionary cursor"""

implementation:
@ wraps ( func )
def wrapper ( cls , * args , ** kwargs ) :
    with ( yield from cls . get_cursor ( _CursorType . DICT ) ) as c :
        return ( yield from func ( cls , c , * args , ** kwargs ) )
return wrapper


signature:
def cursor ( func ) :
    """Decorator that provides a cursor to the calling function
 Adds the cursor as the second argument to the calling functions
 Requires that the function being decorated is an instance of a class or object
 that yields a cursor from a get _ cursor ( ) coroutine or provides such an object
 as the first argument in its signature
 Yields :
 A client - side cursor"""

implementation:
@ wraps ( func )
def wrapper ( cls , * args , ** kwargs ) :
    with ( yield from cls . get_cursor ( ) ) as c :
        return ( yield from func ( cls , c , * args , ** kwargs ) )
return wrapper


signature:
def nt_cursor ( func ) :
    """Decorator that provides a namedtuple cursor to the calling function
 Adds the cursor as the second argument to the calling functions
 Requires that the function being decorated is an instance of a class or object
 that yields a cursor from a get _ cursor ( cursor _ type = CursorType . NAMEDTUPLE ) coroutine or provides such an object
 as the first argument in its signature
 Yields :
 A client - side namedtuple cursor"""

implementation:
@ wraps ( func )
def wrapper ( cls , * args , ** kwargs ) :
    with ( yield from cls . get_cursor ( _CursorType . NAMEDTUPLE ) ) as c :
        return ( yield from func ( cls , c , * args , ** kwargs ) )
return wrapper


signature:
def transaction ( func ) :
    """Provides a transacted cursor which will run in autocommit = false mode
 For any exception the transaction will be rolled back .
 Requires that the function being decorated is an instance of a class or object
 that yields a cursor from a get _ cursor ( cursor _ type = CursorType . NAMEDTUPLE ) coroutine or provides such an object
 as the first argument in its signature
 Yields :
 A client - side transacted named cursor"""

implementation:
@ wraps ( func )
def wrapper ( cls , * args , ** kwargs ) :
    with ( yield from cls . get_cursor ( _CursorType . NAMEDTUPLE ) ) as c :
        try :
            yield from c . execute ( 'BEGIN' )
            result = ( yield from func ( cls , c , * args , ** kwargs ) )
        except Exception :
            yield from c . execute ( 'ROLLBACK' )
        else :
            yield from c . execute ( 'COMMIT' )
            return result
return wrapper


signature:
def connect ( cls , database : str , user : str , password : str , host : str , port : int , * , use_pool : bool = True , enable_ssl : bool = False , minsize = 1 , maxsize = 50 , keepalives_idle = 5 , keepalives_interval = 4 , echo = False , ** kwargs ) :
    """Sets connection parameters
 For more information on the parameters that is accepts ,
 see : http : / / www . postgresql . org / docs / 9.2 / static / libpq - connect . html"""

implementation:
cls . _connection_params [ 'database' ] = database
cls . _connection_params [ 'user' ] = user
cls . _connection_params [ 'password' ] = password
cls . _connection_params [ 'host' ] = host
cls . _connection_params [ 'port' ] = port
cls . _connection_params [ 'sslmode' ] = 'prefer' if enable_ssl else 'disable'
cls . _connection_params [ 'minsize' ] = minsize
cls . _connection_params [ 'maxsize' ] = maxsize
cls . _connection_params [ 'keepalives_idle' ] = keepalives_idle
cls . _connection_params [ 'keepalives_interval' ] = keepalives_interval
cls . _connection_params [ 'echo' ] = echo
cls . _connection_params . update ( kwargs )
cls . _use_pool = use_pool


signature:
def get_pool ( cls ) -> Pool :
    """Yields :
 existing db connection pool"""

implementation:
if len ( cls . _connection_params ) < 5 :
    raise ConnectionError ( 'Please call SQLStore.connect before calling this method' )
if not cls . _pool :
    cls . _pool = yield from create_pool ( ** cls . _connection_params )
return cls . _pool


signature:
def get_cursor ( cls , cursor_type = _CursorType . PLAIN ) -> Cursor :
    """Yields :
 new client - side cursor from existing db connection pool"""

implementation:
_cur = None
if cls . _use_pool :
    _connection_source = yield from cls . get_pool ( )
else :
    _connection_source = yield from aiopg . connect ( echo = False , ** cls . _connection_params )
if cursor_type == _CursorType . PLAIN :
    _cur = yield from _connection_source . cursor ( )
if cursor_type == _CursorType . NAMEDTUPLE :
    _cur = yield from _connection_source . cursor ( cursor_factory = psycopg2 . extras . NamedTupleCursor )
if cursor_type == _CursorType . DICT :
    _cur = yield from _connection_source . cursor ( cursor_factory = psycopg2 . extras . DictCursor )
if not cls . _use_pool :
    _cur = cursor_context_manager ( _connection_source , _cur )
return _cur


signature:
def count ( cls , cur , table : str , where_keys : list = None ) :
    """gives the number of records in the table
 Args :
 table : a string indicating the name of the table
 Returns :
 an integer indicating the number of records in the table"""

implementation:
if where_keys :
    where_clause , values = cls . _get_where_clause_with_values ( where_keys )
    query = cls . _count_query_where . format ( table , where_clause )
    q , t = query , values
else :
    query = cls . _count_query . format ( table )
    q , t = query , ( )
yield from cur . execute ( q , t )
result = yield from cur . fetchone ( )
return int ( result [ 0 ] )


signature:
def insert ( cls , cur , table : str , values : dict ) :
    """Creates an insert statement with only chosen fields
 Args :
 table : a string indicating the name of the table
 values : a dict of fields and values to be inserted
 Returns :
 A ' Record ' object with table columns as properties"""

implementation:
keys = cls . _COMMA . join ( values . keys ( ) )
value_place_holder = cls . _PLACEHOLDER * len ( values )
query = cls . _insert_string . format ( table , keys , value_place_holder [ : - 1 ] )
yield from cur . execute ( query , tuple ( values . values ( ) ) )
return ( yield from cur . fetchone ( ) )


signature:
def update ( cls , cur , table : str , values : dict , where_keys : list ) -> tuple :
    """Creates an update query with only chosen fields
 Supports only a single field where clause
 Args :
 table : a string indicating the name of the table
 values : a dict of fields and values to be inserted
 where _ keys : list of dictionary
 example of where keys : [ { ' name ' : ( ' > ' , ' cip ' ) , ' url ' : ( ' = ' , ' cip . com ' } , { ' type ' : { ' < = ' , ' manufacturer ' } } ]
 where _ clause will look like ( ( name > % s and url = % s ) or ( type < = % s ) )
 items within each dictionary get ' AND ' - ed and dictionaries themselves get ' OR ' - ed
 Returns :
 an integer indicating count of rows deleted"""

implementation:
keys = cls . _COMMA . join ( values . keys ( ) )
value_place_holder = cls . _PLACEHOLDER * len ( values )
where_clause , where_values = cls . _get_where_clause_with_values ( where_keys )
query = cls . _update_string . format ( table , keys , value_place_holder [ : - 1 ] , where_clause )
yield from cur . execute ( query , ( tuple ( values . values ( ) ) + where_values ) )
return ( yield from cur . fetchall ( ) )


signature:
def delete ( cls , cur , table : str , where_keys : list ) :
    """Creates a delete query with where keys
 Supports multiple where clause with and or or both
 Args :
 table : a string indicating the name of the table
 where _ keys : list of dictionary
 example of where keys : [ { ' name ' : ( ' > ' , ' cip ' ) , ' url ' : ( ' = ' , ' cip . com ' } , { ' type ' : { ' < = ' , ' manufacturer ' } } ]
 where _ clause will look like ( ( name > % s and url = % s ) or ( type < = % s ) )
 items within each dictionary get ' AND ' - ed and dictionaries themselves get ' OR ' - ed
 Returns :
 an integer indicating count of rows deleted"""

implementation:
where_clause , values = cls . _get_where_clause_with_values ( where_keys )
query = cls . _delete_query . format ( table , where_clause )
yield from cur . execute ( query , values )
return cur . rowcount


signature:
def select ( cls , cur , table : str , order_by : str , columns : list = None , where_keys : list = None , limit = 100 , offset = 0 ) :
    """Creates a select query for selective columns with where keys
 Supports multiple where claus with and or or both
 Args :
 table : a string indicating the name of the table
 order _ by : a string indicating column name to order the results on
 columns : list of columns to select from
 where _ keys : list of dictionary
 limit : the limit on the number of results
 offset : offset on the results
 example of where keys : [ { ' name ' : ( ' > ' , ' cip ' ) , ' url ' : ( ' = ' , ' cip . com ' } , { ' type ' : { ' < = ' , ' manufacturer ' } } ]
 where _ clause will look like ( ( name > % s and url = % s ) or ( type < = % s ) )
 items within each dictionary get ' AND ' - ed and across dictionaries get ' OR ' - ed
 Returns :
 A list of ' Record ' object with table columns as properties"""

implementation:
if columns :
    columns_string = cls . _COMMA . join ( columns )
    if where_keys :
        where_clause , values = cls . _get_where_clause_with_values ( where_keys )
        query = cls . _select_selective_column_with_condition . format ( columns_string , table , where_clause , order_by , limit , offset )
        q , t = query , values
    else :
        query = cls . _select_selective_column . format ( columns_string , table , order_by , limit , offset )
        q , t = query , ( )
else :
    if where_keys :
        where_clause , values = cls . _get_where_clause_with_values ( where_keys )
        query = cls . _select_all_string_with_condition . format ( table , where_clause , order_by , limit , offset )
        q , t = query , values
    else :
        query = cls . _select_all_string . format ( table , order_by , limit , offset )
        q , t = query , ( )
yield from cur . execute ( q , t )
return ( yield from cur . fetchall ( ) )


signature:
def raw_sql ( cls , cur , query : str , values : tuple ) :
    """Run a raw sql query
 Args :
 query : query string to execute
 values : tuple of values to be used with the query
 Returns :
 result of query as list of named tuple"""

implementation:
yield from cur . execute ( query , values )
return ( yield from cur . fetchall ( ) )


signature:
def serialize_text ( out , text ) :
    """This method is used to append content of the ` text `
 argument to the ` out ` argument .
 Depending on how many lines in the text , a
 padding can be added to all lines except the first
 one .
 Concatenation result is appended to the ` out ` argument ."""

implementation:
padding = len ( out )
# we need to add padding to all lines
# except the first one
add_padding = padding_adder ( padding )
text = add_padding ( text , ignore_first_line = True )
return out + text


signature:
def serialize_list ( out , lst , delimiter = u'' , max_length = 20 ) :
    """This method is used to serialize list of text
 pieces like [ " some = u ' Another ' " , " blah = 124 " ]
 Depending on how many lines are in these items ,
 they are concatenated in row or as a column .
 Concatenation result is appended to the ` out ` argument ."""

implementation:
have_multiline_items = any ( map ( is_multiline , lst ) )
result_will_be_too_long = sum ( map ( len , lst ) ) > max_length
if have_multiline_items or result_will_be_too_long :
    padding = len ( out )
    add_padding = padding_adder ( padding )
    # we need to add padding to all lines
    # except the first one
    head , rest = cut_head ( lst )
    rest = map ( add_padding , rest )
    # add padding to the head , but not for it ' s first line
    head = add_padding ( head , ignore_first_line = True )
    # now join lines back
    lst = chain ( ( head , ) , rest )
    delimiter += u'\n'
else :
    delimiter += u' '
return out + delimiter . join ( lst )


signature:
def format_value ( value ) :
    """This function should return unicode representation of the value"""

implementation:
value_id = id ( value )
if value_id in recursion_breaker . processed :
    return u'<recursion>'
recursion_breaker . processed . add ( value_id )
try :
    if isinstance ( value , six . binary_type ) : # suppose , all byte strings are in unicode
    # don ' t know if everybody in the world uses anything else ?
        return u"'{0}'" . format ( value . decode ( 'utf-8' ) )
    elif isinstance ( value , six . text_type ) :
        return u"u'{0}'" . format ( value )
    elif isinstance ( value , ( list , tuple ) ) : # long lists or lists with multiline items
    # will be shown vertically
        values = list ( map ( format_value , value ) )
        result = serialize_list ( u'[' , values , delimiter = u',' ) + u']'
        return force_unicode ( result )
    elif isinstance ( value , dict ) :
        items = six . iteritems ( value )
        # format each key / value pair as a text ,
        # calling format _ value recursively
        items = ( tuple ( map ( format_value , item ) ) for item in items )
        items = list ( items )
        # sort by keys for readability
        items . sort ( )
        # for each item value
        items = [ serialize_text ( u'{0}: ' . format ( key ) , item_value ) for key , item_value in items ]
        # and serialize these pieces as a list , enclosing
        # them into a curve brackets
        result = serialize_list ( u'{' , items , delimiter = u',' ) + u'}'
        return force_unicode ( result )
    return force_unicode ( repr ( value ) )
finally :
    recursion_breaker . processed . remove ( value_id )


signature:
def make_repr ( * args , ** kwargs ) :
    """Returns _ _ repr _ _ method which returns ASCII
 representaion of the object with given fields .
 Without arguments , ` ` make _ repr ` ` generates a method
 which outputs all object ' s non - protected ( non - undercored )
 arguments which are not callables .
 Accepts ` ` * args ` ` , which should be a names of object ' s
 attributes to be included in the output : :
 _ _ repr _ _ = make _ repr ( ' foo ' , ' bar ' )
 If you want to generate attribute ' s content on the fly ,
 then you should use keyword arguments and pass a callable
 of one argument : :
 _ _ repr _ _ = make _ repr ( foo = lambda obj : obj . blah + 100500)"""

implementation:
def method ( self ) :
    cls_name = self . __class__ . __name__
    if args :
        field_names = args
    else :
        def undercored ( name ) :
            return name . startswith ( '_' )
        def is_method ( name ) :
            return callable ( getattr ( self , name ) )
        def good_name ( name ) :
            return not undercored ( name ) and not is_method ( name )
        field_names = filter ( good_name , dir ( self ) )
        field_names = sorted ( field_names )
    # on this stage , we make from field _ names an
    # attribute getters
    field_getters = zip ( field_names , map ( attrgetter , field_names ) )
    # now process keyword args , they must
    # contain callables of one argument
    # and callable should return a field ' s value
    field_getters = chain ( field_getters , kwargs . items ( ) )
    fields = ( ( name , format_value ( getter ( self ) ) ) for name , getter in field_getters )
    # prepare key strings
    fields = ( ( u'{0}=' . format ( name ) , value ) for name , value in fields )
    # join values with they respective keys
    fields = list ( starmap ( serialize_text , fields ) )
    beginning = u'<{cls_name} ' . format ( cls_name = cls_name , )
    result = serialize_list ( beginning , fields )
    # append closing braket
    result += u'>'
    if ON_PYTHON2 : # on python 2 . x repr returns bytes , but on python3 - unicode strings
        result = result . encode ( 'utf-8' )
    return result
return method


signature:
def connect ( self , host , port , minsize = 5 , maxsize = 10 , loop = asyncio . get_event_loop ( ) ) :
    """Setup a connection pool
 : param host : Redis host
 : param port : Redis port
 : param loop : Event loop"""

implementation: self . _pool = yield from aioredis . create_pool ( ( host , port ) , minsize = minsize , maxsize = maxsize , loop = loop )


signature:
def set_key ( self , key , value , namespace = None , expire = 0 ) :
    """Set a key in a cache .
 : param key : Key name
 : param value : Value
 : param namespace : Namespace to associate the key with
 : param expire : expiration
 : return :"""

implementation:
with ( yield from self . _pool ) as redis :
    if namespace is not None :
        key = self . _get_key ( namespace , key )
    yield from redis . set ( key , value , expire = expire )


signature:
def traverse ( element , query , deep = False ) :
    """Helper function to traverse an element tree rooted at element , yielding nodes matching the query ."""

implementation:
# Grab the next part of the query ( it will be chopped from the front each iteration ) .
part = query [ 0 ]
if not part : # If the part is blank , we encountered a / / , meaning search all sub - nodes .
    query = query [ 1 : ]
    part = query [ 0 ]
    deep = True
# Parse out any predicate ( tag [ pred ] ) from this part of the query .
part , predicate = xpath_re . match ( query [ 0 ] ) . groups ( )
for c in element . _children :
    if part in ( '*' , c . tagname ) and c . _match ( predicate ) : # A potential matching branch : this child matches the next query part ( and predicate ) .
        if len ( query ) == 1 : # If this is the last part of the query , we found a matching element , yield it .
            yield c
        else : # Otherwise , check the children of this child against the next query part .
            for e in traverse ( c , query [ 1 : ] ) :
                yield e
    if deep : # If we ' re searching all sub - nodes , traverse with the same query , regardless of matching .
    # This basically creates a recursion branch to search EVERYWHERE for anything after / / .
        for e in traverse ( c , query , deep = True ) :
            yield e


signature:
def parse_query ( query ) :
    """Given a simplified XPath query string , returns an array of normalized query parts ."""

implementation:
parts = query . split ( '/' )
norm = [ ]
for p in parts :
    p = p . strip ( )
    if p :
        norm . append ( p )
    elif '' not in norm :
        norm . append ( '' )
return norm


signature:
def parse ( url_or_path , encoding = None , handler_class = DrillHandler ) :
    """: param url _ or _ path : A file - like object , a filesystem path , a URL , or a string containing XML
 : rtype : : class : ` XmlElement `"""

implementation:
handler = handler_class ( )
parser = expat . ParserCreate ( encoding )
parser . buffer_text = 1
parser . StartElementHandler = handler . start_element
parser . EndElementHandler = handler . end_element
parser . CharacterDataHandler = handler . characters
if isinstance ( url_or_path , basestring ) :
    if '://' in url_or_path [ : 20 ] :
        with contextlib . closing ( url_lib . urlopen ( url_or_path ) ) as f :
            parser . ParseFile ( f )
    elif url_or_path [ : 100 ] . strip ( ) . startswith ( '<' ) :
        if isinstance ( url_or_path , unicode ) :
            if encoding is None :
                encoding = 'utf-8'
            url_or_path = url_or_path . encode ( encoding )
        parser . Parse ( url_or_path , True )
    else :
        with open ( url_or_path , 'rb' ) as f :
            parser . ParseFile ( f )
elif PY3 and isinstance ( url_or_path , bytes ) :
    parser . ParseFile ( bytes_io ( url_or_path ) )
else :
    parser . ParseFile ( url_or_path )
return handler . root


signature:
def iterparse ( filelike , encoding = None , handler_class = DrillHandler , xpath = None ) :
    """: param filelike : A file - like object with a ` ` read ` ` method
 : returns : An iterator yielding : class : ` XmlElement ` objects"""

implementation:
parser = expat . ParserCreate ( encoding )
elem_iter = DrillElementIterator ( filelike , parser )
handler = handler_class ( elem_iter , xpath )
parser . buffer_text = 1
parser . StartElementHandler = handler . start_element
parser . EndElementHandler = handler . end_element
parser . CharacterDataHandler = handler . characters
return elem_iter


signature:
def write ( self , writer ) :
    """Writes an XML representation of this node ( including descendants ) to the specified file - like object .
 : param writer : An : class : ` XmlWriter ` instance to write this node to"""

implementation:
multiline = bool ( self . _children )
newline_start = multiline and not bool ( self . data )
writer . start ( self . tagname , self . attrs , newline = newline_start )
if self . data :
    writer . data ( self . data , newline = bool ( self . _children ) )
for c in self . _children :
    c . write ( writer )
writer . end ( self . tagname , indent = multiline )


signature:
def xml ( self , ** kwargs ) :
    """Returns an XML representation of this node ( including descendants ) . This method automatically creates an
 : class : ` XmlWriter ` instance internally to handle the writing .
 : param * * kwargs : Any named arguments are passed along to the : class : ` XmlWriter ` constructor"""

implementation:
s = bytes_io ( )
writer = XmlWriter ( s , ** kwargs )
self . write ( writer )
return s . getvalue ( )


signature:
def append ( self , name , attrs = None , data = None ) :
    """Called when the parser detects a start tag ( child element ) while in this node . Internally creates an
 : class : ` XmlElement ` and adds it to the end of this node ' s children .
 : param name : The tag name to add
 : param attrs : Attributes for the new tag
 : param data : CDATA for the new tag
 : returns : The newly - created element
 : rtype : : class : ` XmlElement `"""

implementation:
elem = self . __class__ ( name , attrs , data , parent = self , index = len ( self . _children ) )
self . _children . append ( elem )
return elem


signature:
def insert ( self , before , name , attrs = None , data = None ) :
    """Inserts a new element as a child of this element , before the specified index or sibling .
 : param before : An : class : ` XmlElement ` or a numeric index to insert the new node before
 : param name : The tag name to add
 : param attrs : Attributes for the new tag
 : param data : CDATA for the new tag
 : returns : The newly - created element
 : rtype : : class : ` XmlElement `"""

implementation:
if isinstance ( before , self . __class__ ) :
    if before . parent != self :
        raise ValueError ( 'Cannot insert before an element with a different parent.' )
    before = before . index
# Make sure 0 < = before < = len ( _ children ) .
before = min ( max ( 0 , before ) , len ( self . _children ) )
elem = self . __class__ ( name , attrs , data , parent = self , index = before )
self . _children . insert ( before , elem )
# Re - index all the children .
for idx , c in enumerate ( self . _children ) :
    c . index = idx
return elem


signature:
def items ( self ) :
    """A generator yielding ` ` ( key , value ) ` ` attribute pairs , sorted by key name ."""

implementation:
for key in sorted ( self . attrs ) :
    yield key , self . attrs [ key ]


signature:
def children ( self , name = None , reverse = False ) :
    """A generator yielding children of this node .
 : param name : If specified , only consider elements with this tag name
 : param reverse : If ` ` True ` ` , children will be yielded in reverse declaration order"""

implementation:
elems = self . _children
if reverse :
    elems = reversed ( elems )
for elem in elems :
    if name is None or elem . tagname == name :
        yield elem


signature:
def _match ( self , pred ) :
    """Helper function to determine if this node matches the given predicate ."""

implementation:
if not pred :
    return True
# Strip off the [ and ]
pred = pred [ 1 : - 1 ]
if pred . startswith ( '@' ) : # An attribute predicate checks the existence ( and optionally value ) of an attribute on this tag .
    pred = pred [ 1 : ]
    if '=' in pred :
        attr , value = pred . split ( '=' , 1 )
        if value [ 0 ] in ( '"' , "'" ) :
            value = value [ 1 : ]
        if value [ - 1 ] in ( '"' , "'" ) :
            value = value [ : - 1 ]
        return self . attrs . get ( attr ) == value
    else :
        return pred in self . attrs
elif num_re . match ( pred ) : # An index predicate checks whether we are the n - th child of our parent ( 0 - based ) .
    index = int ( pred )
    if index < 0 :
        if self . parent : # For negative indexes , count from the end of the list .
            return self . index == ( len ( self . parent . _children ) + index )
        else : # If we ' re the root node , the only index we could be is 0.
            return index == 0
    else :
        return index == self . index
else :
    if '=' in pred :
        tag , value = pred . split ( '=' , 1 )
        if value [ 0 ] in ( '"' , "'" ) :
            value = value [ 1 : ]
        if value [ - 1 ] in ( '"' , "'" ) :
            value = value [ : - 1 ]
        for c in self . _children :
            if c . tagname == tag and c . data == value :
                return True
    else : # A plain [ tag ] predicate means we match if we have a child with tagname " tag " .
        for c in self . _children :
            if c . tagname == pred :
                return True
return False


signature:
def path ( self , include_root = False ) :
    """Returns a canonical path to this element , relative to the root node .
 : param include _ root : If ` ` True ` ` , include the root node in the path . Defaults to ` ` False ` ` ."""

implementation:
path = '%s[%d]' % ( self . tagname , self . index or 0 )
p = self . parent
while p is not None :
    if p . parent or include_root :
        path = '%s[%d]/%s' % ( p . tagname , p . index or 0 , path )
    p = p . parent
return path


signature:
def iter ( self , name = None ) :
    """Recursively find any descendants of this node with the given tag name . If a tag name is omitted , this will
 yield every descendant node .
 : param name : If specified , only consider elements with this tag name
 : returns : A generator yielding descendants of this node"""

implementation:
for c in self . _children :
    if name is None or c . tagname == name :
        yield c
    for gc in c . find ( name ) :
        yield gc


signature:
def last ( self , name = None ) :
    """Returns the last child of this node .
 : param name : If specified , only consider elements with this tag name
 : rtype : : class : ` XmlElement `"""

implementation:
for c in self . children ( name , reverse = True ) :
    return c


signature:
def parents ( self , name = None ) :
    """Yields all parents of this element , back to the root element .
 : param name : If specified , only consider elements with this tag name"""

implementation:
p = self . parent
while p is not None :
    if name is None or p . tagname == name :
        yield p
    p = p . parent


signature:
def siblings ( self , name = None ) :
    """Yields all siblings of this node ( not including the node itself ) .
 : param name : If specified , only consider elements with this tag name"""

implementation:
if self . parent and self . index :
    for c in self . parent . _children :
        if c . index != self . index and ( name is None or name == c . tagname ) :
            yield c


signature:
def next ( self , name = None ) :
    """Returns the next sibling of this node .
 : param name : If specified , only consider elements with this tag name
 : rtype : : class : ` XmlElement `"""

implementation:
if self . parent is None or self . index is None :
    return None
for idx in xrange ( self . index + 1 , len ( self . parent ) ) :
    if name is None or self . parent [ idx ] . tagname == name :
        return self . parent [ idx ]


signature:
def prev ( self , name = None ) :
    """Returns the previous sibling of this node .
 : param name : If specified , only consider elements with this tag name
 : rtype : : class : ` XmlElement `"""

implementation:
if self . parent is None or self . index is None :
    return None
for idx in xrange ( self . index - 1 , - 1 , - 1 ) :
    if name is None or self . parent [ idx ] . tagname == name :
        return self . parent [ idx ]


signature:
def get_observations ( self ) :
    """Parses the HTML table into a list of dictionaries , each of which
 represents a single observation ."""

implementation:
if self . empty :
    return [ ]
rows = list ( self . tbody )
observations = [ ]
for row_observation , row_details in zip ( rows [ : : 2 ] , rows [ 1 : : 2 ] ) :
    data = { }
    cells = OBSERVATION_XPATH ( row_observation )
    data [ 'name' ] = _clean_cell ( cells [ 0 ] )
    data [ 'date' ] = _clean_cell ( cells [ 1 ] )
    data [ 'magnitude' ] = _clean_cell ( cells [ 3 ] )
    data [ 'obscode' ] = _clean_cell ( cells [ 6 ] )
    cells = DETAILS_XPATH ( row_details )
    data [ 'comp1' ] = _clean_cell ( cells [ 0 ] )
    data [ 'chart' ] = _clean_cell ( cells [ 3 ] ) . replace ( 'None' , '' )
    data [ 'comment_code' ] = _clean_cell ( cells [ 4 ] )
    data [ 'notes' ] = _clean_cell ( cells [ 5 ] )
    observations . append ( data )
return observations


signature:
def get_cache_key ( prefix , * args , ** kwargs ) :
    """Calculates cache key based on ` args ` and ` kwargs ` .
 ` args ` and ` kwargs ` must be instances of hashable types ."""

implementation:
hash_args_kwargs = hash ( tuple ( kwargs . iteritems ( ) ) + args )
return '{}_{}' . format ( prefix , hash_args_kwargs )


signature:
def cache_method ( func = None , prefix = '' ) :
    """Cache result of function execution into the ` self ` object ( mostly useful in models ) .
 Calculate cache key based on ` args ` and ` kwargs ` of the function ( except ` self ` ) ."""

implementation:
def decorator ( func ) :
    @ wraps ( func )
    def wrapper ( self , * args , ** kwargs ) :
        cache_key_prefix = prefix or '_cache_{}' . format ( func . __name__ )
        cache_key = get_cache_key ( cache_key_prefix , * args , ** kwargs )
        if not hasattr ( self , cache_key ) :
            setattr ( self , cache_key , func ( self ) )
        return getattr ( self , cache_key )
    return wrapper
if func is None :
    return decorator
else :
    return decorator ( func )


signature:
def cache_func ( prefix , method = False ) :
    """Cache result of function execution into the django cache backend .
 Calculate cache key based on ` prefix ` , ` args ` and ` kwargs ` of the function .
 For using like object method set ` method = True ` ."""

implementation:
def decorator ( func ) :
    @ wraps ( func )
    def wrapper ( * args , ** kwargs ) :
        cache_args = args
        if method :
            cache_args = args [ 1 : ]
        cache_key = get_cache_key ( prefix , * cache_args , ** kwargs )
        cached_value = cache . get ( cache_key )
        if cached_value is None :
            cached_value = func ( * args , ** kwargs )
            cache . set ( cache_key , cached_value )
        return cached_value
    return wrapper
return decorator


signature:
def get_or_default ( func = None , default = None ) :
    """Wrapper around Django ' s ORM ` get ` functionality .
 Wrap anything that raises ObjectDoesNotExist exception
 and provide the default value if necessary .
 ` default ` by default is None . ` default ` can be any callable ,
 if it is callable it will be called when ObjectDoesNotExist
 exception will be raised ."""

implementation:
def decorator ( func ) :
    @ wraps ( func )
    def wrapper ( * args , ** kwargs ) :
        try :
            return func ( * args , ** kwargs )
        except ObjectDoesNotExist :
            if callable ( default ) :
                return default ( )
            else :
                return default
    return wrapper
if func is None :
    return decorator
else :
    return decorator ( func )


signature:
def _get_column_nums_from_args ( columns ) :
    """Turn column inputs from user into list of simple numbers .
 Inputs can be :
 - individual number : 1
 - range : 1-3
 - comma separated list : 1,2,3,4-6"""

implementation:
nums = [ ]
for c in columns :
    for p in c . split ( ',' ) :
        p = p . strip ( )
        try :
            c = int ( p )
            nums . append ( c )
        except ( TypeError , ValueError ) :
            start , ignore , end = p . partition ( '-' )
            try :
                start = int ( start )
                end = int ( end )
            except ( TypeError , ValueError ) :
                raise ValueError ( 'Did not understand %r, expected digit-digit' % c )
            inc = 1 if start < end else - 1
            nums . extend ( range ( start , end + inc , inc ) )
# The user will pass us 1 - based indexes , but we need to use
# 0 - based indexing with the row .
return [ n - 1 for n in nums ]


signature:
def _get_printable_columns ( columns , row ) :
    """Return only the part of the row which should be printed ."""

implementation:
if not columns :
    return row
# Extract the column values , in the order specified .
return tuple ( row [ c ] for c in columns )


signature:
def writerow ( self , observation_data ) :
    """Writes a single observation to the output file .
 If the ` ` observation _ data ` ` parameter is a dictionary , it is
 converted to a list to keep a consisted field order ( as described
 in format specification ) . Otherwise it is assumed that the data
 is a raw record ready to be written to file .
 : param observation _ data : a single observation as a dictionary or list"""

implementation:
if isinstance ( observation_data , ( list , tuple ) ) :
    row = observation_data
else :
    row = self . dict_to_row ( observation_data )
self . writer . writerow ( row )


signature:
def dict_to_row ( cls , observation_data ) :
    """Takes a dictionary of observation data and converts it to a list
 of fields according to AAVSO visual format specification .
 : param cls : current class
 : param observation _ data : a single observation as a dictionary"""

implementation:
row = [ ]
row . append ( observation_data [ 'name' ] )
row . append ( observation_data [ 'date' ] )
row . append ( observation_data [ 'magnitude' ] )
comment_code = observation_data . get ( 'comment_code' , 'na' )
if not comment_code :
    comment_code = 'na'
row . append ( comment_code )
comp1 = observation_data . get ( 'comp1' , 'na' )
if not comp1 :
    comp1 = 'na'
row . append ( comp1 )
comp2 = observation_data . get ( 'comp2' , 'na' )
if not comp2 :
    comp2 = 'na'
row . append ( comp2 )
chart = observation_data . get ( 'chart' , 'na' )
if not chart :
    chart = 'na'
row . append ( chart )
notes = observation_data . get ( 'notes' , 'na' )
if not notes :
    notes = 'na'
row . append ( notes )
return row


signature:
def row_to_dict ( cls , row ) :
    """Converts a raw input record to a dictionary of observation data .
 : param cls : current class
 : param row : a single observation as a list or tuple"""

implementation:
comment_code = row [ 3 ]
if comment_code . lower ( ) == 'na' :
    comment_code = ''
comp1 = row [ 4 ]
if comp1 . lower ( ) == 'na' :
    comp1 = ''
comp2 = row [ 5 ]
if comp2 . lower ( ) == 'na' :
    comp2 = ''
chart = row [ 6 ]
if chart . lower ( ) == 'na' :
    chart = ''
notes = row [ 7 ]
if notes . lower ( ) == 'na' :
    notes = ''
return { 'name' : row [ 0 ] , 'date' : row [ 1 ] , 'magnitude' : row [ 2 ] , 'comment_code' : comment_code , 'comp1' : comp1 , 'comp2' : comp2 , 'chart' : chart , 'notes' : notes , }


signature:
def get_default_tag ( app ) :
    '''Get the name of the view function used to prevent having to set the tag
 manually for every endpoint'''

implementation:
view_func = get_view_function ( app , request . path , request . method )
if view_func :
    return view_func . __name__


signature:
def get_view_function ( app , url , method ) :
    """Match a url and return the view and arguments
 it will be called with , or None if there is no view .
 Creds : http : / / stackoverflow . com / a / 38488506"""

implementation:
# pylint : disable = too - many - return - statements
adapter = app . create_url_adapter ( request )
try :
    match = adapter . match ( url , method = method )
except RequestRedirect as ex : # recursively match redirects
    return get_view_function ( app , ex . new_url , method )
except ( MethodNotAllowed , NotFound ) : # no match
    return None
try :
    return app . view_functions [ match [ 0 ] ]
except KeyError : # no view is associated with the endpoint
    return None


signature:
def download_observations ( observer_code ) :
    """Downloads all variable star observations by a given observer .
 Performs a series of HTTP requests to AAVSO ' s WebObs search and
 downloads the results page by page . Each page is then passed to
 : py : class : ` ~ pyaavso . parsers . webobs . WebObsResultsParser ` and parse results
 are added to the final observation list ."""

implementation:
page_number = 1
observations = [ ]
while True :
    logger . info ( 'Downloading page %d...' , page_number )
    response = requests . get ( WEBOBS_RESULTS_URL , params = { 'obscode' : observer_code , 'num_results' : 200 , 'obs_types' : 'all' , 'page' : page_number , } )
    logger . debug ( response . request . url )
    parser = WebObsResultsParser ( response . text )
    observations . extend ( parser . get_observations ( ) )
    # kinda silly , but there ' s no need for lxml machinery here
    if '>Next</a>' not in response . text :
        break
    page_number += 1
return observations


signature:
def get_random_filename ( instance , filename ) :
    """Generates random filename for uploading file using uuid4 hashes
 You need to define UPLOADS _ ROOT in your django settings
 something like this
 UPLOADS _ ROOT = rel ( MEDIA _ ROOT , ' uploads ' )"""

implementation:
folder = settings . UPLOADS_ROOT
ext = filename . split ( '.' ) [ - 1 ]
filename = '{}.{}' . format ( str ( uuid4 ( ) ) , ext )
return os . path . join ( folder , filename )


signature:
def image_path ( instance , filename ) :
    """Generates likely unique image path using md5 hashes"""

implementation:
filename , ext = os . path . splitext ( filename . lower ( ) )
instance_id_hash = hashlib . md5 ( str ( instance . id ) ) . hexdigest ( )
filename_hash = '' . join ( random . sample ( hashlib . md5 ( filename . encode ( 'utf-8' ) ) . hexdigest ( ) , 8 ) )
return '{}/{}{}' . format ( instance_id_hash , filename_hash , ext )


signature:
async def get_ltd_product_urls ( session ) :
    """Get URLs for LSST the Docs ( LTD ) products from the LTD Keeper API .
 Parameters
 session : ` aiohttp . ClientSession `
 Your application ' s aiohttp client session .
 See http : / / aiohttp . readthedocs . io / en / stable / client . html .
 Returns
 product _ urls : ` list `
 List of product URLs ."""

implementation:
product_url = 'https://keeper.lsst.codes/products/'
async with session . get ( product_url ) as response :
    data = await response . json ( )
return data [ 'products' ]


signature:
async def get_ltd_product ( session , slug = None , url = None ) :
    """Get the product resource ( JSON document ) from the LSST the Docs API .
 Parameters
 session : ` aiohttp . ClientSession `
 Your application ' s aiohttp client session .
 See http : / / aiohttp . readthedocs . io / en / stable / client . html .
 slug : ` str ` , optional
 Slug identfying the product . This is the same as the subdomain .
 For example , ` ` ' ldm - 151 ' ` ` is the slug for ` ` ldm - 151 . lsst . io ` ` .
 A full product URL can be provided instead , see ` ` url ` ` .
 url : ` str ` , optional
 The full LTD Keeper URL for the product resource . For example ,
 ` ` ' https : / / keeper . lsst . codes / products / ldm - 151 ' ` ` . The ` ` slug ` `
 can be provided instead .
 Returns
 product : ` dict `
 Product dataset . See
 https : / / ltd - keeper . lsst . io / products . html # get - - products - ( slug )
 for fields ."""

implementation:
if url is None :
    url = 'https://keeper.lsst.codes/products/{}' . format ( slug )
async with session . get ( url ) as response :
    data = await response . json ( )
return data


signature:
async def process_lander_page ( session , github_api_token , ltd_product_data , mongo_collection = None ) :
    """Extract , transform , and load metadata from Lander - based projects .
 Parameters
 session : ` aiohttp . ClientSession `
 Your application ' s aiohttp client session .
 See http : / / aiohttp . readthedocs . io / en / stable / client . html .
 github _ api _ token : ` str `
 A GitHub personal API token . See the ` GitHub personal access token
 guide ` _ .
 ltd _ product _ data : ` dict `
 Contents of ` ` metadata . yaml ` ` , obtained via ` download _ metadata _ yaml ` .
 Data for this technote from the LTD Keeper API
 ( ` ` GET / products / < slug > ` ` ) . Usually obtained via
 ` lsstprojectmeta . ltd . get _ ltd _ product ` .
 mongo _ collection : ` motor . motor _ asyncio . AsyncIOMotorCollection ` , optional
 MongoDB collection . This should be the common MongoDB collection for
 LSST projectmeta JSON - LD records . If provided , ths JSON - LD is upserted
 into the MongoDB collection .
 Returns
 metadata : ` dict `
 JSON - LD - formatted dictionary .
 Raises
 NotLanderPageError
 Raised when the LTD product cannot be interpreted as a Lander page
 because the ` ` / metadata . jsonld ` ` file is absent . This implies that
 the LTD product * could * be of a different format .
 . . ` GitHub personal access token guide ` : https : / / ls . st / 41d"""

implementation:
logger = logging . getLogger ( __name__ )
# Try to download metadata . jsonld from the Landing page site .
published_url = ltd_product_data [ 'published_url' ]
jsonld_url = urljoin ( published_url , '/metadata.jsonld' )
try :
    async with session . get ( jsonld_url ) as response :
        logger . debug ( '%s response status %r' , jsonld_url , response . status )
        response . raise_for_status ( )
        json_data = await response . text ( )
except aiohttp . ClientResponseError as err :
    logger . debug ( 'Tried to download %s, got status %d' , jsonld_url , err . code )
    raise NotLanderPageError ( )
# Use our own json parser to get datetimes
metadata = decode_jsonld ( json_data )
if mongo_collection is not None :
    await _upload_to_mongodb ( mongo_collection , metadata )
return metadata


signature:
async def _upload_to_mongodb ( collection , jsonld ) :
    """Upsert the technote resource into the projectmeta MongoDB collection .
 Parameters
 collection : ` motor . motor _ asyncio . AsyncIOMotorCollection `
 The MongoDB collection .
 jsonld : ` dict `
 The JSON - LD document that represents the document resource ."""

implementation:
document = { 'data' : jsonld }
query = { 'data.reportNumber' : jsonld [ 'reportNumber' ] }
await collection . update ( query , document , upsert = True , multi = False )


signature:
def json_doc_to_xml ( json_obj , lang = 'en' , custom_namespace = None ) :
    """Converts a Open511 JSON document to XML .
 lang : the appropriate language code
 Takes a dict deserialized from JSON , returns an lxml Element .
 Accepts only the full root - level JSON object from an Open511 response ."""

implementation:
if 'meta' not in json_obj :
    raise Exception ( "This function requires a conforming Open511 JSON document with a 'meta' section." )
json_obj = dict ( json_obj )
meta = json_obj . pop ( 'meta' )
elem = get_base_open511_element ( lang = lang , version = meta . pop ( 'version' ) )
pagination = json_obj . pop ( 'pagination' , None )
json_struct_to_xml ( json_obj , elem , custom_namespace = custom_namespace )
if pagination :
    elem . append ( json_struct_to_xml ( pagination , 'pagination' , custom_namespace = custom_namespace ) )
json_struct_to_xml ( meta , elem )
return elem


signature:
def json_struct_to_xml ( json_obj , root , custom_namespace = None ) :
    """Converts a Open511 JSON fragment to XML .
 Takes a dict deserialized from JSON , returns an lxml Element .
 This won ' t provide a conforming document if you pass in a full JSON document ;
 it ' s for translating little fragments , and is mostly used internally ."""

implementation:
if isinstance ( root , ( str , unicode ) ) :
    if root . startswith ( '!' ) :
        root = etree . Element ( '{%s}%s' % ( NS_PROTECTED , root [ 1 : ] ) )
    elif root . startswith ( '+' ) :
        if not custom_namespace :
            raise Exception ( "JSON fields starts with +, but no custom namespace provided" )
        root = etree . Element ( '{%s}%s' % ( custom_namespace , root [ 1 : ] ) )
    else :
        root = etree . Element ( root )
if root . tag in ( 'attachments' , 'grouped_events' , 'media_files' ) :
    for link in json_obj :
        root . append ( json_link_to_xml ( link ) )
elif isinstance ( json_obj , ( str , unicode ) ) :
    root . text = json_obj
elif isinstance ( json_obj , ( int , float ) ) :
    root . text = unicode ( json_obj )
elif isinstance ( json_obj , dict ) :
    if frozenset ( json_obj . keys ( ) ) == frozenset ( ( 'type' , 'coordinates' ) ) :
        root . append ( geojson_to_gml ( json_obj ) )
    else :
        for key , val in json_obj . items ( ) :
            if key == 'url' or key . endswith ( '_url' ) :
                el = json_link_to_xml ( val , json_link_key_to_xml_rel ( key ) )
            else :
                el = json_struct_to_xml ( val , key , custom_namespace = custom_namespace )
            if el is not None :
                root . append ( el )
elif isinstance ( json_obj , list ) :
    tag_name = root . tag
    if tag_name . endswith ( 'ies' ) :
        tag_name = tag_name [ : - 3 ] + 'y'
    elif tag_name . endswith ( 's' ) :
        tag_name = tag_name [ : - 1 ]
    for val in json_obj :
        el = json_struct_to_xml ( val , tag_name , custom_namespace = custom_namespace )
        if el is not None :
            root . append ( el )
elif json_obj is None :
    return None
else :
    raise NotImplementedError
return root


signature:
def geojson_to_gml ( gj , set_srs = True ) :
    """Given a dict deserialized from a GeoJSON object , returns an lxml Element
 of the corresponding GML geometry ."""

implementation:
tag = G ( gj [ 'type' ] )
if set_srs :
    tag . set ( 'srsName' , 'urn:ogc:def:crs:EPSG::4326' )
if gj [ 'type' ] == 'Point' :
    tag . append ( G . pos ( _reverse_geojson_coords ( gj [ 'coordinates' ] ) ) )
elif gj [ 'type' ] == 'LineString' :
    tag . append ( G . posList ( ' ' . join ( _reverse_geojson_coords ( ll ) for ll in gj [ 'coordinates' ] ) ) )
elif gj [ 'type' ] == 'Polygon' :
    rings = [ G . LinearRing ( G . posList ( ' ' . join ( _reverse_geojson_coords ( ll ) for ll in ring ) ) ) for ring in gj [ 'coordinates' ] ]
    tag . append ( G . exterior ( rings . pop ( 0 ) ) )
    for ring in rings :
        tag . append ( G . interior ( ring ) )
elif gj [ 'type' ] in ( 'MultiPoint' , 'MultiLineString' , 'MultiPolygon' ) :
    single_type = gj [ 'type' ] [ 5 : ]
    member_tag = single_type [ 0 ] . lower ( ) + single_type [ 1 : ] + 'Member'
    for coord in gj [ 'coordinates' ] :
        tag . append ( G ( member_tag , geojson_to_gml ( { 'type' : single_type , 'coordinates' : coord } , set_srs = False ) ) )
else :
    raise NotImplementedError
return tag


signature:
def geom_to_xml_element ( geom ) :
    """Transform a GEOS or OGR geometry object into an lxml Element
 for the GML geometry ."""

implementation:
if geom . srs . srid != 4326 :
    raise NotImplementedError ( "Only WGS 84 lat/long geometries (SRID 4326) are supported." )
# GeoJSON output is far more standard than GML , so go through that
return geojson_to_gml ( json . loads ( geom . geojson ) )


signature:
def remove_comments ( tex_source ) :
    """Delete latex comments from TeX source .
 Parameters
 tex _ source : str
 TeX source content .
 Returns
 tex _ source : str
 TeX source without comments ."""

implementation:
# Expression via http : / / stackoverflow . com / a / 13365453
return re . sub ( r'(?<!\\)%.*$' , r'' , tex_source , flags = re . M )


signature:
def read_tex_file ( root_filepath , root_dir = None ) :
    r"""Read a TeX file , automatically processing and normalizing it
 ( including other input files , removing comments , and deleting trailing
 whitespace ) .
 Parameters
 root _ filepath : ` str `
 Filepath to a TeX file .
 root _ dir : ` str `
 Root directory of the TeX project . This only needs to be set when
 recursively reading in ` ` \ input ` ` or ` ` \ include ` ` files .
 Returns
 tex _ source : ` str `
 TeX source ."""

implementation:
with open ( root_filepath , 'r' ) as f :
    tex_source = f . read ( )
if root_dir is None :
    root_dir = os . path . dirname ( root_filepath )
# Text processing pipline
tex_source = remove_comments ( tex_source )
tex_source = remove_trailing_whitespace ( tex_source )
tex_source = process_inputs ( tex_source , root_dir = root_dir )
return tex_source


signature:
def replace_macros ( tex_source , macros ) :
    r"""Replace macros in the TeX source with their content .
 Parameters
 tex _ source : ` str `
 TeX source content .
 macros : ` dict `
 Keys are macro names ( including leading ` ` \ ` ` ) and values are the
 content ( as ` str ` ) of the macros . See
 ` lsstprojectmeta . tex . scraper . get _ macros ` .
 Returns
 tex _ source : ` str `
 TeX source with known macros replaced .
 Notes
 Macros with arguments are not supported .
 Examples
 > > > macros = { r ' \ handle ' : ' LDM - nnn ' }
 > > > sample = r ' This is document \ handle . '
 > > > replace _ macros ( sample , macros )
 ' This is document LDM - nnn . '
 Any trailing slash after the macro command is also replaced by this
 function .
 > > > macros = { r ' \ product ' : ' Data Management ' }
 > > > sample = r ' \ title [ Test Plan ] { \ product \ Test Plan } '
 > > > replace _ macros ( sample , macros )
 ' \ \ title [ Test Plan ] { Data Management Test Plan } '"""

implementation:
for macro_name , macro_content in macros . items ( ) : # ' \ \ ? ' suffix matches an optional trailing ' \ ' that might be used
# for spacing .
    pattern = re . escape ( macro_name ) + r"\\?"
    # Wrap macro _ content in lambda to avoid processing escapes
    tex_source = re . sub ( pattern , lambda _ : macro_content , tex_source )
return tex_source


signature:
def ensure_format ( doc , format ) :
    """Ensures that the provided document is an lxml Element or json dict ."""

implementation:
assert format in ( 'xml' , 'json' )
if getattr ( doc , 'tag' , None ) == 'open511' :
    if format == 'json' :
        return xml_to_json ( doc )
elif isinstance ( doc , dict ) and 'meta' in doc :
    if format == 'xml' :
        return json_doc_to_xml ( doc )
else :
    raise ValueError ( "Unrecognized input document" )
return doc


signature:
def open511_convert ( input_doc , output_format , serialize = True , ** kwargs ) :
    """Convert an Open511 document between formats .
 input _ doc - either an lxml open511 Element or a deserialized JSON dict
 output _ format - short string name of a valid output format , as listed above"""

implementation:
try :
    output_format_info = FORMATS [ output_format ]
except KeyError :
    raise ValueError ( "Unrecognized output format %s" % output_format )
input_doc = ensure_format ( input_doc , output_format_info . input_format )
result = output_format_info . func ( input_doc , ** kwargs )
if serialize :
    result = output_format_info . serializer ( result )
return result


signature:
def read ( cls , root_tex_path ) :
    """Construct an ` LsstLatexDoc ` instance by reading and parsing the
 LaTeX source .
 Parameters
 root _ tex _ path : ` str `
 Path to the LaTeX source on the filesystem . For multi - file LaTeX
 projects this should be the path to the root document .
 Notes
 This method implements the following pipeline :
 1 . ` lsstprojectmeta . tex . normalizer . read _ tex _ file `
 2 . ` lsstprojectmeta . tex . scraper . get _ macros `
 3 . ` lsstprojectmeta . tex . normalizer . replace _ macros `
 Thus ` ` input ` ` and ` ` includes ` ` are resolved along with simple macros ."""

implementation:
# Read and normalize the TeX source , replacing macros with content
root_dir = os . path . dirname ( root_tex_path )
tex_source = read_tex_file ( root_tex_path )
tex_macros = get_macros ( tex_source )
tex_source = replace_macros ( tex_source , tex_macros )
return cls ( tex_source , root_dir = root_dir )


signature:
def html_title ( self ) :
    """HTML5 - formatted document title ( ` str ` ) ."""

implementation: return self . format_title ( format = 'html5' , deparagraph = True , mathjax = False , smart = True )


signature:
def html_short_title ( self ) :
    """HTML5 - formatted document short title ( ` str ` ) ."""

implementation: return self . format_short_title ( format = 'html5' , deparagraph = True , mathjax = False , smart = True )


signature:
def html_authors ( self ) :
    """HTML5 - formatted authors ( ` list ` of ` str ` ) ."""

implementation: return self . format_authors ( format = 'html5' , deparagraph = True , mathjax = False , smart = True )


signature:
def html_abstract ( self ) :
    """HTML5 - formatted document abstract ( ` str ` ) ."""

implementation: return self . format_abstract ( format = 'html5' , deparagraph = False , mathjax = False , smart = True )


signature:
def is_draft ( self ) :
    """Document is a draft if ` ` ' lsstdoc ' ` ` is included in the
 documentclass options ( ` bool ` ) ."""

implementation:
if not hasattr ( self , '_document_options' ) :
    self . _parse_documentclass ( )
if 'lsstdraft' in self . _document_options :
    return True
else :
    return False


signature:
def format_content ( self , format = 'plain' , mathjax = False , smart = True , extra_args = None ) :
    """Get the document content in the specified markup format .
 Parameters
 format : ` str ` , optional
 Output format ( such as ` ` ' html5 ' ` ` or ` ` ' plain ' ` ` ) .
 mathjax : ` bool ` , optional
 Allow pandoc to use MathJax math markup .
 smart : ` True ` , optional
 Allow pandoc to create " smart " unicode punctuation .
 extra _ args : ` list ` , optional
 Additional command line flags to pass to Pandoc . See
 ` lsstprojectmeta . pandoc . convert . convert _ text ` .
 Returns
 output _ text : ` str `
 Converted content ."""

implementation:
output_text = convert_lsstdoc_tex ( self . _tex , format , mathjax = mathjax , smart = smart , extra_args = extra_args )
return output_text


signature:
def format_title ( self , format = 'html5' , deparagraph = True , mathjax = False , smart = True , extra_args = None ) :
    """Get the document title in the specified markup format .
 Parameters
 format : ` str ` , optional
 Output format ( such as ` ` ' html5 ' ` ` or ` ` ' plain ' ` ` ) .
 deparagraph : ` bool ` , optional
 Remove the paragraph tags from single paragraph content .
 mathjax : ` bool ` , optional
 Allow pandoc to use MathJax math markup .
 smart : ` True ` , optional
 Allow pandoc to create " smart " unicode punctuation .
 extra _ args : ` list ` , optional
 Additional command line flags to pass to Pandoc . See
 ` lsstprojectmeta . pandoc . convert . convert _ text ` .
 Returns
 output _ text : ` str `
 Converted content or ` None ` if the title is not available in
 the document ."""

implementation:
if self . title is None :
    return None
output_text = convert_lsstdoc_tex ( self . title , format , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args )
return output_text


signature:
def format_short_title ( self , format = 'html5' , deparagraph = True , mathjax = False , smart = True , extra_args = None ) :
    """Get the document short title in the specified markup format .
 Parameters
 format : ` str ` , optional
 Output format ( such as ` ` ' html5 ' ` ` or ` ` ' plain ' ` ` ) .
 deparagraph : ` bool ` , optional
 Remove the paragraph tags from single paragraph content .
 mathjax : ` bool ` , optional
 Allow pandoc to use MathJax math markup .
 smart : ` True ` , optional
 Allow pandoc to create " smart " unicode punctuation .
 extra _ args : ` list ` , optional
 Additional command line flags to pass to Pandoc . See
 ` lsstprojectmeta . pandoc . convert . convert _ text ` .
 Returns
 output _ text : ` str `
 Converted content or ` None ` if the short title is not available in
 the document ."""

implementation:
if self . short_title is None :
    return None
output_text = convert_lsstdoc_tex ( self . short_title , 'html5' , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args )
return output_text


signature:
def format_abstract ( self , format = 'html5' , deparagraph = False , mathjax = False , smart = True , extra_args = None ) :
    """Get the document abstract in the specified markup format .
 Parameters
 format : ` str ` , optional
 Output format ( such as ` ` ' html5 ' ` ` or ` ` ' plain ' ` ` ) .
 deparagraph : ` bool ` , optional
 Remove the paragraph tags from single paragraph content .
 mathjax : ` bool ` , optional
 Allow pandoc to use MathJax math markup .
 smart : ` True ` , optional
 Allow pandoc to create " smart " unicode punctuation .
 extra _ args : ` list ` , optional
 Additional command line flags to pass to Pandoc . See
 ` lsstprojectmeta . pandoc . convert . convert _ text ` .
 Returns
 output _ text : ` str `
 Converted content or ` None ` if the title is not available in
 the document ."""

implementation:
if self . abstract is None :
    return None
abstract_latex = self . _prep_snippet_for_pandoc ( self . abstract )
output_text = convert_lsstdoc_tex ( abstract_latex , format , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args )
return output_text


signature:
def format_authors ( self , format = 'html5' , deparagraph = True , mathjax = False , smart = True , extra_args = None ) :
    """Get the document authors in the specified markup format .
 Parameters
 format : ` str ` , optional
 Output format ( such as ` ` ' html5 ' ` ` or ` ` ' plain ' ` ` ) .
 deparagraph : ` bool ` , optional
 Remove the paragraph tags from single paragraph content .
 mathjax : ` bool ` , optional
 Allow pandoc to use MathJax math markup .
 smart : ` True ` , optional
 Allow pandoc to create " smart " unicode punctuation .
 extra _ args : ` list ` , optional
 Additional command line flags to pass to Pandoc . See
 ` lsstprojectmeta . pandoc . convert . convert _ text ` .
 Returns
 output _ text : ` list ` of ` str `
 Sequence of author names in the specified output markup format ."""

implementation:
formatted_authors = [ ]
for latex_author in self . authors :
    formatted_author = convert_lsstdoc_tex ( latex_author , format , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args )
    # removes Pandoc ' s terminal newlines
    formatted_author = formatted_author . strip ( )
    formatted_authors . append ( formatted_author )
return formatted_authors


signature:
def _parse_documentclass ( self ) :
    """Parse documentclass options .
 Sets the the ` ` _ document _ options ` ` attribute ."""

implementation:
command = LatexCommand ( 'documentclass' , { 'name' : 'options' , 'required' : False , 'bracket' : '[' } , { 'name' : 'class_name' , 'required' : True , 'bracket' : '{' } )
try :
    parsed = next ( command . parse ( self . _tex ) )
except StopIteration :
    self . _logger . warning ( 'lsstdoc has no documentclass' )
    self . _document_options = [ ]
try :
    content = parsed [ 'options' ]
    self . _document_options = [ opt . strip ( ) for opt in content . split ( ',' ) ]
except KeyError :
    self . _logger . warning ( 'lsstdoc has no documentclass options' )
    self . _document_options = [ ]


signature:
def _parse_title ( self ) :
    """Parse the title from TeX source .
 Sets these attributes :
 - ` ` _ title ` `
 - ` ` _ short _ title ` `"""

implementation:
command = LatexCommand ( 'title' , { 'name' : 'short_title' , 'required' : False , 'bracket' : '[' } , { 'name' : 'long_title' , 'required' : True , 'bracket' : '{' } )
try :
    parsed = next ( command . parse ( self . _tex ) )
except StopIteration :
    self . _logger . warning ( 'lsstdoc has no title' )
    self . _title = None
    self . _short_title = None
self . _title = parsed [ 'long_title' ]
try :
    self . _short_title = parsed [ 'short_title' ]
except KeyError :
    self . _logger . warning ( 'lsstdoc has no short title' )
    self . _short_title = None