signature:
def get_gdb_response ( self , timeout_sec = DEFAULT_GDB_TIMEOUT_SEC , raise_error_on_timeout = True ) :
    """Get response from GDB , and block while doing so . If GDB does not have any response ready to be read
 by timeout _ sec , an exception is raised .
 Args :
 timeout _ sec ( float ) : Maximum time to wait for reponse . Must be > = 0 . Will return after
 raise _ error _ on _ timeout ( bool ) : Whether an exception should be raised if no response was found
 after timeout _ sec
 Returns :
 List of parsed GDB responses , returned from gdbmiparser . parse _ response , with the
 additional key ' stream ' which is either ' stdout ' or ' stderr '
 Raises :
 GdbTimeoutError if response is not received within timeout _ sec
 ValueError if select returned unexpected file number
 NoGdbProcessError if there is no gdb subprocess running"""

implementation:
self . verify_valid_gdb_subprocess ( )
if timeout_sec < 0 :
    self . logger . warning ( "timeout_sec was negative, replacing with 0" )
    timeout_sec = 0
if USING_WINDOWS :
    retval = self . _get_responses_windows ( timeout_sec )
else :
    retval = self . _get_responses_unix ( timeout_sec )
if not retval and raise_error_on_timeout :
    raise GdbTimeoutError ( "Did not get response from gdb after %s seconds" % timeout_sec )
else :
    return retval


signature:
def _get_responses_windows ( self , timeout_sec ) :
    """Get responses on windows . Assume no support for select and use a while loop ."""

implementation:
timeout_time_sec = time . time ( ) + timeout_sec
responses = [ ]
while True :
    try :
        self . gdb_process . stdout . flush ( )
        if PYTHON3 :
            raw_output = self . gdb_process . stdout . readline ( ) . replace ( b"\r" , b"\n" )
        else :
            raw_output = self . gdb_process . stdout . read ( ) . replace ( b"\r" , b"\n" )
        responses += self . _get_responses_list ( raw_output , "stdout" )
    except IOError :
        pass
    try :
        self . gdb_process . stderr . flush ( )
        if PYTHON3 :
            raw_output = self . gdb_process . stderr . readline ( ) . replace ( b"\r" , b"\n" )
        else :
            raw_output = self . gdb_process . stderr . read ( ) . replace ( b"\r" , b"\n" )
        responses += self . _get_responses_list ( raw_output , "stderr" )
    except IOError :
        pass
    if time . time ( ) > timeout_time_sec :
        break
return responses


signature:
def _get_responses_unix ( self , timeout_sec ) :
    """Get responses on unix - like system . Use select to wait for output ."""

implementation:
timeout_time_sec = time . time ( ) + timeout_sec
responses = [ ]
while True :
    select_timeout = timeout_time_sec - time . time ( )
    # I prefer to not pass a negative value to select
    if select_timeout <= 0 :
        select_timeout = 0
    events , _ , _ = select . select ( self . read_list , [ ] , [ ] , select_timeout )
    responses_list = None
    # to avoid infinite loop if using Python 2
    try :
        for fileno in events : # new data is ready to read
            if fileno == self . stdout_fileno :
                self . gdb_process . stdout . flush ( )
                raw_output = self . gdb_process . stdout . read ( )
                stream = "stdout"
            elif fileno == self . stderr_fileno :
                self . gdb_process . stderr . flush ( )
                raw_output = self . gdb_process . stderr . read ( )
                stream = "stderr"
            else :
                raise ValueError ( "Developer error. Got unexpected file number %d" % fileno )
            responses_list = self . _get_responses_list ( raw_output , stream )
            responses += responses_list
    except IOError : # only occurs in python 2.7
        pass
    if timeout_sec == 0 : # just exit immediately
        break
    elif responses_list and self . _allow_overwrite_timeout_times : # update timeout time to potentially be closer to now to avoid lengthy wait times when nothing is being output by gdb
        timeout_time_sec = min ( time . time ( ) + self . time_to_check_for_additional_output_sec , timeout_time_sec , )
    elif time . time ( ) > timeout_time_sec :
        break
return responses


signature:
def _get_responses_list ( self , raw_output , stream ) :
    """Get parsed response list from string output
 Args :
 raw _ output ( unicode ) : gdb output to parse
 stream ( str ) : either stdout or stderr"""

implementation:
responses = [ ]
raw_output , self . _incomplete_output [ stream ] = _buffer_incomplete_responses ( raw_output , self . _incomplete_output . get ( stream ) )
if not raw_output :
    return responses
response_list = list ( filter ( lambda x : x , raw_output . decode ( errors = "replace" ) . split ( "\n" ) ) )
# remove blank lines
# parse each response from gdb into a dict , and store in a list
for response in response_list :
    if gdbmiparser . response_is_finished ( response ) :
        pass
    else :
        parsed_response = gdbmiparser . parse_response ( response )
        parsed_response [ "stream" ] = stream
        self . logger . debug ( "%s" , pformat ( parsed_response ) )
        responses . append ( parsed_response )
return responses


signature:
def send_signal_to_gdb ( self , signal_input ) :
    """Send signal name ( case insensitive ) or number to gdb subprocess
 gdbmi . send _ signal _ to _ gdb ( 2 ) # valid
 gdbmi . send _ signal _ to _ gdb ( ' sigint ' ) # also valid
 gdbmi . send _ signal _ to _ gdb ( ' SIGINT ' ) # also valid
 raises ValueError if signal _ input is invalie
 raises NoGdbProcessError if there is no gdb process to send a signal to"""

implementation:
try :
    signal = int ( signal_input )
except Exception :
    signal = SIGNAL_NAME_TO_NUM . get ( signal_input . upper ( ) )
if not signal :
    raise ValueError ( 'Could not find signal corresponding to "%s"' % str ( signal ) )
if self . gdb_process :
    os . kill ( self . gdb_process . pid , signal )
else :
    raise NoGdbProcessError ( "Cannot send signal to gdb process because no process exists." )


signature:
def exit ( self ) :
    """Terminate gdb process
 Returns : None"""

implementation:
if self . gdb_process :
    self . gdb_process . terminate ( )
    self . gdb_process . communicate ( )
self . gdb_process = None
return None


signature:
def main ( verbose = True ) :
    """Build and debug an application programatically
 For a list of GDB MI commands , see https : / / www . sourceware . org / gdb / onlinedocs / gdb / GDB _ 002fMI . html"""

implementation:
# Build C program
find_executable ( MAKE_CMD )
if not find_executable ( MAKE_CMD ) :
    print ( 'Could not find executable "%s". Ensure it is installed and on your $PATH.' % MAKE_CMD )
    exit ( 1 )
subprocess . check_output ( [ MAKE_CMD , "-C" , SAMPLE_C_CODE_DIR , "--quiet" ] )
# Initialize object that manages gdb subprocess
gdbmi = GdbController ( verbose = verbose )
# Send gdb commands . Gdb machine interface commands are easier to script around ,
# hence the name " machine interface " .
# Responses are automatically printed as they are received if verbose is True .
# Responses are returned after writing , by default .
# Load the file
responses = gdbmi . write ( "-file-exec-and-symbols %s" % SAMPLE_C_BINARY )
# Get list of source files used to compile the binary
responses = gdbmi . write ( "-file-list-exec-source-files" )
# Add breakpoint
responses = gdbmi . write ( "-break-insert main" )
# Run
responses = gdbmi . write ( "-exec-run" )
responses = gdbmi . write ( "-exec-next" )
responses = gdbmi . write ( "-exec-next" )
responses = gdbmi . write ( "-exec-continue" )
# noqa : F841
# gdbmi . gdb _ process will be None because the gdb subprocess ( and its inferior
# program ) will be terminated
gdbmi . exit ( )


signature:
def read ( self , count ) :
    """Read count characters starting at self . index ,
 and return those characters as a string"""

implementation:
new_index = self . index + count
if new_index > self . len :
    buf = self . raw_text [ self . index : ]
    # return to the end , don ' t fail
else :
    buf = self . raw_text [ self . index : new_index ]
self . index = new_index
return buf


signature:
def advance_past_chars ( self , chars ) :
    """Advance the index past specific chars
 Args chars ( list ) : list of characters to advance past
 Return substring that was advanced past"""

implementation:
start_index = self . index
while True :
    current_char = self . raw_text [ self . index ]
    self . index += 1
    if current_char in chars :
        break
    elif self . index == self . len :
        break
return self . raw_text [ start_index : self . index - 1 ]


signature:
def advance_past_string_with_gdb_escapes ( self , chars_to_remove_gdb_escape = None ) :
    """characters that gdb escapes that should not be
 escaped by this parser"""

implementation:
if chars_to_remove_gdb_escape is None :
    chars_to_remove_gdb_escape = [ '"' ]
buf = ""
while True :
    c = self . raw_text [ self . index ]
    self . index += 1
    logging . debug ( "%s" , fmt_cyan ( c ) )
    if c == "\\" : # We are on a backslash and there is another character after the backslash
    # to parse . Handle this case specially since gdb escaped it for us
    # Get the next char that is being escaped
        c2 = self . raw_text [ self . index ]
        self . index += 1
        # only store the escaped character in the buffer ; don ' t store the backslash
        # ( don ' t leave it escaped )
        buf += c2
    elif c == '"' : # Quote is closed . Exit ( and don ' t include the end quote ) .
        break
    else : # capture this character , and keep capturing
        buf += c
return buf


signature:
def parse_response ( gdb_mi_text ) :
    """Parse gdb mi text and turn it into a dictionary .
 See https : / / sourceware . org / gdb / onlinedocs / gdb / GDB _ 002fMI - Stream - Records . html # GDB _ 002fMI - Stream - Records
 for details on types of gdb mi output .
 Args :
 gdb _ mi _ text ( str ) : String output from gdb
 Returns :
 dict with the following keys :
 type ( either ' notify ' , ' result ' , ' console ' , ' log ' , ' target ' , ' done ' ) ,
 message ( str or None ) ,
 payload ( str , list , dict , or None )"""

implementation:
stream = StringStream ( gdb_mi_text , debug = _DEBUG )
if _GDB_MI_NOTIFY_RE . match ( gdb_mi_text ) :
    token , message , payload = _get_notify_msg_and_payload ( gdb_mi_text , stream )
    return { "type" : "notify" , "message" : message , "payload" : payload , "token" : token , }
elif _GDB_MI_RESULT_RE . match ( gdb_mi_text ) :
    token , message , payload = _get_result_msg_and_payload ( gdb_mi_text , stream )
    return { "type" : "result" , "message" : message , "payload" : payload , "token" : token , }
elif _GDB_MI_CONSOLE_RE . match ( gdb_mi_text ) :
    return { "type" : "console" , "message" : None , "payload" : _GDB_MI_CONSOLE_RE . match ( gdb_mi_text ) . groups ( ) [ 0 ] , }
elif _GDB_MI_LOG_RE . match ( gdb_mi_text ) :
    return { "type" : "log" , "message" : None , "payload" : _GDB_MI_LOG_RE . match ( gdb_mi_text ) . groups ( ) [ 0 ] , }
elif _GDB_MI_TARGET_OUTPUT_RE . match ( gdb_mi_text ) :
    return { "type" : "target" , "message" : None , "payload" : _GDB_MI_TARGET_OUTPUT_RE . match ( gdb_mi_text ) . groups ( ) [ 0 ] , }
elif response_is_finished ( gdb_mi_text ) :
    return { "type" : "done" , "message" : None , "payload" : None }
else : # This was not gdb mi output , so it must have just been printed by
# the inferior program that ' s being debugged
    return { "type" : "output" , "message" : None , "payload" : gdb_mi_text }


signature:
def assert_match ( actual_char_or_str , expected_char_or_str ) :
    """If values don ' t match , print them and raise a ValueError , otherwise ,
 continue
 Raises : ValueError if argumetns do not match"""

implementation:
if expected_char_or_str != actual_char_or_str :
    print ( "Expected" )
    pprint ( expected_char_or_str )
    print ( "" )
    print ( "Got" )
    pprint ( actual_char_or_str )
    raise ValueError ( )


signature:
def _get_notify_msg_and_payload ( result , stream ) :
    """Get notify message and payload dict"""

implementation:
token = stream . advance_past_chars ( [ "=" , "*" ] )
token = int ( token ) if token != "" else None
logger . debug ( "%s" , fmt_green ( "parsing message" ) )
message = stream . advance_past_chars ( [ "," ] )
logger . debug ( "parsed message" )
logger . debug ( "%s" , fmt_green ( message ) )
payload = _parse_dict ( stream )
return token , message . strip ( ) , payload


signature:
def _get_result_msg_and_payload ( result , stream ) :
    """Get result message and payload dict"""

implementation:
groups = _GDB_MI_RESULT_RE . match ( result ) . groups ( )
token = int ( groups [ 0 ] ) if groups [ 0 ] != "" else None
message = groups [ 1 ]
if groups [ 2 ] is None :
    payload = None
else :
    stream . advance_past_chars ( [ "," ] )
    payload = _parse_dict ( stream )
return token , message , payload


signature:
def _parse_dict ( stream ) :
    """Parse dictionary , with optional starting character ' { '
 return ( tuple ) :
 Number of characters parsed from to _ parse
 Parsed dictionary"""

implementation:
obj = { }
logger . debug ( "%s" , fmt_green ( "parsing dict" ) )
while True :
    c = stream . read ( 1 )
    if c in _WHITESPACE :
        pass
    elif c in [ "{" , "," ] :
        pass
    elif c in [ "}" , "" ] : # end of object , exit loop
        break
    else :
        stream . seek ( - 1 )
        key , val = _parse_key_val ( stream )
        if key in obj : # This is a gdb bug . We should never get repeated keys in a dict !
        # See https : / / sourceware . org / bugzilla / show _ bug . cgi ? id = 22217
        # and https : / / github . com / cs01 / pygdbmi / issues / 19
        # Example :
        # thread - ids = { thread - id = " 1 " , thread - id = " 2 " }
        # Results in :
        # thread - ids : { { ' thread - id ' : [ ' 1 ' , ' 2 ' ] } }
        # Rather than the lossy
        # thread - ids : { ' thread - id ' : 2 } # ' 1 ' got overwritten !
            if isinstance ( obj [ key ] , list ) :
                obj [ key ] . append ( val )
            else :
                obj [ key ] = [ obj [ key ] , val ]
        else :
            obj [ key ] = val
        look_ahead_for_garbage = True
        c = stream . read ( 1 )
        while look_ahead_for_garbage :
            if c in [ "}" , "," , "" ] :
                look_ahead_for_garbage = False
            else : # got some garbage text , skip it . for example :
            # name = " gdb " gargage # skip over ' garbage '
            # name = " gdb " \ n # skip over ' \ n '
                logger . debug ( "skipping unexpected charcter: " + c )
                c = stream . read ( 1 )
        stream . seek ( - 1 )
logger . debug ( "parsed dict" )
logger . debug ( "%s" , fmt_green ( obj ) )
return obj


signature:
def _parse_key_val ( stream ) :
    """Parse key , value combination
 return ( tuple ) :
 Parsed key ( string )
 Parsed value ( either a string , array , or dict )"""

implementation:
logger . debug ( "parsing key/val" )
key = _parse_key ( stream )
val = _parse_val ( stream )
logger . debug ( "parsed key/val" )
logger . debug ( "%s" , fmt_green ( key ) )
logger . debug ( "%s" , fmt_green ( val ) )
return key , val


signature:
def _parse_key ( stream ) :
    """Parse key , value combination
 returns :
 Parsed key ( string )"""

implementation:
logger . debug ( "parsing key" )
key = stream . advance_past_chars ( [ "=" ] )
logger . debug ( "parsed key:" )
logger . debug ( "%s" , fmt_green ( key ) )
return key


signature:
def _parse_val ( stream ) :
    """Parse value from string
 returns :
 Parsed value ( either a string , array , or dict )"""

implementation:
logger . debug ( "parsing value" )
while True :
    c = stream . read ( 1 )
    if c == "{" : # Start object
        val = _parse_dict ( stream )
        break
    elif c == "[" : # Start of an array
        val = _parse_array ( stream )
        break
    elif c == '"' : # Start of a string
        val = stream . advance_past_string_with_gdb_escapes ( )
        break
    elif _DEBUG :
        raise ValueError ( "unexpected character: %s" % c )
    else :
        print ( 'pygdbmi warning: encountered unexpected character: "%s". Continuing.' % c )
        val = ""
# this will be overwritten if there are more characters to be read
logger . debug ( "parsed value:" )
logger . debug ( "%s" , fmt_green ( val ) )
return val


signature:
def _parse_array ( stream ) :
    """Parse an array , stream should be passed the initial [
 returns :
 Parsed array"""

implementation:
logger . debug ( "parsing array" )
arr = [ ]
while True :
    c = stream . read ( 1 )
    if c in _GDB_MI_VALUE_START_CHARS :
        stream . seek ( - 1 )
        val = _parse_val ( stream )
        arr . append ( val )
    elif c in _WHITESPACE :
        pass
    elif c == "," :
        pass
    elif c == "]" : # Stop when this array has finished . Note
    # that elements of this array can be also be arrays .
        break
logger . debug ( "parsed array:" )
logger . debug ( "%s" , fmt_green ( arr ) )
return arr


signature:
def as_parameters ( * parameters , variables = None ) :
    """Dump python list as the parameter of javascript function
 : param parameters :
 : param variables :
 : return :"""

implementation:
s = json . dumps ( parameters )
s = s [ 1 : - 1 ]
if variables :
    for v in variables :
        if v in parameters :
            s = s . replace ( '"' + v + '"' , v )
return s


signature:
def generate_local_url ( self , js_name ) :
    """Generate the local url for a js file .
 : param js _ name :
 : return :"""

implementation:
host = self . _settings [ 'local_host' ] . format ( ** self . _host_context ) . rstrip ( '/' )
return '{}/{}.js' . format ( host , js_name )


signature:
def ifetch_single ( iterable , key , default = EMPTY , getter = None ) :
    """getter ( ) g ( item , key ) : pass"""

implementation:
def _getter ( item ) :
    if getter :
        custom_getter = partial ( getter , key = key )
        return custom_getter ( item )
    else :
        try :
            attrgetter = operator . attrgetter ( key )
            return attrgetter ( item )
        except AttributeError :
            pass
        try :
            itemgetter = operator . itemgetter ( key )
            return itemgetter ( item )
        except KeyError :
            pass
        if default is not EMPTY :
            return default
        raise ValueError ( 'Item %r has no attr or key for %r' % ( item , key ) )
return map ( _getter , iterable )


signature:
def _get_or_create_subscription ( self ) :
    """In a broadcast queue , workers have a unique subscription ensuring
 that every worker recieves a copy of every task ."""

implementation:
topic_path = self . _get_topic_path ( )
subscription_name = '{}-{}-{}-worker' . format ( queue . PUBSUB_OBJECT_PREFIX , self . name , uuid4 ( ) . hex )
subscription_path = self . subscriber_client . subscription_path ( self . project , subscription_name )
try :
    self . subscriber_client . get_subscription ( subscription_path )
except google . cloud . exceptions . NotFound :
    logger . info ( "Creating worker subscription {}" . format ( subscription_name ) )
    self . subscriber_client . create_subscription ( subscription_path , topic_path )
return subscription_path


signature:
def cleanup ( self ) :
    """Deletes this worker ' s subscription ."""

implementation:
if self . subscription :
    logger . info ( "Deleting worker subscription..." )
    self . subscriber_client . delete_subscription ( self . subscription )


signature:
def _get_or_create_subscription ( self ) :
    """Workers all share the same subscription so that tasks are
 distributed across all workers ."""

implementation:
topic_path = self . _get_topic_path ( )
subscription_name = '{}-{}-shared' . format ( PUBSUB_OBJECT_PREFIX , self . name )
subscription_path = self . subscriber_client . subscription_path ( self . project , subscription_name )
try :
    self . subscriber_client . get_subscription ( subscription_path )
except google . cloud . exceptions . NotFound :
    logger . info ( "Creating shared subscription {}" . format ( subscription_name ) )
    try :
        self . subscriber_client . create_subscription ( subscription_path , topic = topic_path )
    except google . cloud . exceptions . Conflict : # Another worker created the subscription before us , ignore .
        pass
return subscription_path


signature:
def enqueue ( self , f , * args , ** kwargs ) :
    """Enqueues a function for the task queue to execute ."""

implementation:
task = Task ( uuid4 ( ) . hex , f , args , kwargs )
self . storage . put_task ( task )
return self . enqueue_task ( task )


signature:
def enqueue_task ( self , task ) :
    """Enqueues a task directly . This is used when a task is retried or if
 a task was manually created .
 Note that this does not store the task ."""

implementation:
data = dumps ( task )
if self . _async :
    self . publisher_client . publish ( self . topic_path , data = data )
    logger . info ( 'Task {} queued.' . format ( task . id ) )
else :
    unpickled_task = unpickle ( data )
    logger . info ( 'Executing task {} synchronously.' . format ( unpickled_task . id ) )
    with measure_time ( ) as summary , self . queue_context ( ) :
        unpickled_task . execute ( queue = self )
        summary ( unpickled_task . summary ( ) )
return TaskResult ( task . id , self )


signature:
def unpickle ( pickled_string ) :
    """Unpickles a string , but raises a unified UnpickleError in case anything
 fails .
 This is a helper method to not have to deal with the fact that ` loads ( ) `
 potentially raises many types of exceptions ( e . g . AttributeError ,
 IndexError , TypeError , KeyError , etc . )"""

implementation:
try :
    obj = loads ( pickled_string )
except Exception as e :
    raise UnpickleError ( 'Could not unpickle' , pickled_string , e )
return obj


signature:
def main ( path , pid , queue ) :
    """Standalone PSQ worker .
 The queue argument must be the full importable path to a psq . Queue
 instance .
 Example usage :
 psqworker config . q
 psqworker - - path / opt / app queues . fast"""

implementation:
setup_logging ( )
if pid :
    with open ( os . path . expanduser ( pid ) , "w" ) as f :
        f . write ( str ( os . getpid ( ) ) )
if not path :
    path = os . getcwd ( )
sys . path . insert ( 0 , path )
queue = import_queue ( queue )
import psq
worker = psq . Worker ( queue = queue )
worker . listen ( )


signature:
def result ( self , timeout = None ) :
    """Gets the result of the task .
 Arguments :
 timeout : Maximum seconds to wait for a result before raising a
 TimeoutError . If set to None , this will wait forever . If the
 queue doesn ' t store results and timeout is None , this call will
 never return ."""

implementation:
start = time . time ( )
while True :
    task = self . get_task ( )
    if not task or task . status not in ( FINISHED , FAILED ) :
        if not timeout :
            continue
        elif time . time ( ) - start < timeout :
            continue
        else :
            raise TimeoutError ( )
    if task . status == FAILED :
        raise task . result
    return task . result


signature:
def service_start ( service = None , param = None ) :
    """Launch a Process , return his pid"""

implementation:
if service is not None :
    to_run = [ "python" , service ]
    if param is not None :
        to_run += param
    return subprocess . Popen ( to_run )
return False


signature:
def update_running_pids ( old_procs ) :
    """Update the list of the running process and return the list"""

implementation:
new_procs = [ ]
for proc in old_procs :
    if proc . poll ( ) is None and check_pid ( proc . pid ) :
        publisher . debug ( str ( proc . pid ) + ' is alive' )
        new_procs . append ( proc )
    else :
        try :
            publisher . debug ( str ( proc . pid ) + ' is gone' )
            os . kill ( proc . pid , signal . SIGKILL )
        except : # the process is just already gone
            pass
return new_procs


signature:
def run_splitted_processing ( max_simultaneous_processes , process_name , filenames ) :
    """Run processes which push the routing dump of the RIPE in a redis
 database .
 The dump has been splitted in multiple files and each process run
 on one of this files ."""

implementation:
pids = [ ]
while len ( filenames ) > 0 :
    while len ( filenames ) > 0 and len ( pids ) < max_simultaneous_processes :
        filename = filenames . pop ( )
        pids . append ( service_start ( service = process_name , param = [ '-f' , filename , '-d' , imported_day ] ) )
    while len ( pids ) == max_simultaneous_processes :
        time . sleep ( sleep_timer )
        pids = update_running_pids ( pids )
while len ( pids ) > 0 : # Wait until all the processes are finished
    time . sleep ( sleep_timer )
    pids = update_running_pids ( pids )


signature:
def fsplit ( file_to_split ) :
    """Split the file and return the list of filenames ."""

implementation:
dirname = file_to_split + '_splitted'
if not os . path . exists ( dirname ) :
    os . mkdir ( dirname )
part_file_size = os . path . getsize ( file_to_split ) / number_of_files + 1
splitted_files = [ ]
with open ( file_to_split , "r" ) as f :
    number = 0
    actual = 0
    while 1 :
        prec = actual
        # Jump of " size " from the current place in the file
        f . seek ( part_file_size , os . SEEK_CUR )
        # find the next separator or EOF
        s = f . readline ( )
        if len ( s ) == 0 :
            s = f . readline ( )
        while len ( s ) != 0 and s != separator :
            s = f . readline ( )
        # Get the current place
        actual = f . tell ( )
        new_file = os . path . join ( dirname , str ( number ) )
        # Create the new file
        with open ( file_to_split , "r" ) as temp :
            temp . seek ( prec )
            # Get the text we want to put in the new file
            copy = temp . read ( actual - prec )
            # Write the new file
            open ( new_file , 'w' ) . write ( copy )
        splitted_files . append ( new_file )
        number += 1
        # End of file
        if len ( s ) == 0 :
            break
return splitted_files


signature:
def __entry_point ( ) :
    """Function called when an query is made on / json . Expects a JSON
 object with at least a ' method ' entry ."""

implementation:
ip = request . remote_addr
ua = request . headers . get ( 'User-Agent' , 'Empty User-Agent' )
method = request . json . get ( 'method' )
if method is None :
    __query_logging ( ip , ua , method , level = 'warning' )
    return json . dumps ( { 'error' : 'No method provided.' } )
if method not in authorized_methods : # unauthorized query
    __query_logging ( ip , ua , method , level = 'warning' )
    return json . dumps ( { 'error' : 'Unauthorized method.' } )
fct = globals ( ) . get ( method )
if fct is None : # unknown method , the method is authorized , but does not exists . . .
    __query_logging ( ip , ua , method , level = 'warning' )
    return json . dumps ( { 'error' : 'Unknown method.' } )
if request . json . get ( 'ip' ) is None :
    __query_logging ( ip , ua , method , level = 'warning' )
    return json . dumps ( { 'error' : 'No IP provided, not going to work.' } )
try :
    result = fct ( request . json )
    __query_logging ( ip , ua , method , request . json . get ( 'ip' ) , request . json . get ( 'announce_date' ) , request . json . get ( 'days_limit' ) )
    return result
except Exception :
    __query_logging ( ip , ua , method , request . json . get ( 'ip' ) , level = 'error' )
    return json . dumps ( { 'error' : 'Something went wrong.' } )


signature:
def asn ( self , ip , announce_date = None ) :
    """Give an IP , maybe a date , get the ASN .
 This is the fastest command .
 : param ip : IP address to search for
 : param announce _ date : Date of the announcement
 : rtype : String , ASN ."""

implementation:
assignations , announce_date , _ = self . run ( ip , announce_date )
return next ( ( assign for assign in assignations if assign is not None ) , None ) , announce_date


signature:
def date_asn_block ( self , ip , announce_date = None ) :
    """Get the ASN and the IP Block announcing the IP at a specific date .
 : param ip : IP address to search for
 : param announce _ date : Date of the announcement
 : rtype : tuple
 . . code - block : : python
 ( announce _ date , asn , block )
 . . note : :
 the returned announce _ date might be different of the one
 given in parameter because some raw files are missing and we
 don ' t have the information . In this case , the nearest known
 date will be chosen ,"""

implementation:
assignations , announce_date , keys = self . run ( ip , announce_date )
pos = next ( ( i for i , j in enumerate ( assignations ) if j is not None ) , None )
if pos is not None :
    block = keys [ pos ]
    if block != '0.0.0.0/0' :
        return announce_date , assignations [ pos ] , block
return None


signature:
def history ( self , ip , days_limit = None ) :
    """Get the full history of an IP . It takes time .
 : param ip : IP address to search for
 : param days _ limit : Max amount of days to query . ( None means no limit )
 : rtype : list . For each day in the database : day , asn , block"""

implementation:
all_dates = sorted ( self . routing_db . smembers ( 'imported_dates' ) , reverse = True )
if days_limit is not None :
    all_dates = all_dates [ : days_limit ]
return [ self . date_asn_block ( ip , date ) for date in all_dates ]


signature:
def aggregate_history ( self , ip , days_limit = None ) :
    """Get the full history of an IP , aggregate the result instead of
 returning one line per day .
 : param ip : IP address to search for
 : param days _ limit : Max amount of days to query . ( None means no limit )
 : rtype : list . For each change : FirstDay , LastDay , ASN , Block"""

implementation:
first_date = None
last_date = None
prec_asn = None
prec_block = None
for entry in self . history ( ip , days_limit ) :
    if entry is None :
        continue
    date , asn , block = entry
    if first_date is None :
        last_date = date
        first_date = date
        prec_asn = asn
        prec_block = block
    elif prec_asn == asn and prec_block == block :
        first_date = date
    else :
        yield first_date , last_date , prec_asn , prec_block
        last_date = date
        first_date = date
        prec_asn = asn
        prec_block = block
if first_date is not None :
    yield first_date , last_date , prec_asn , prec_block


signature:
def downloadURL ( url , filename ) :
    """Inconditianilly download the URL in a temporary directory .
 When finished , the file is moved in the real directory .
 Like this an other process will not attempt to extract an inclomplete file ."""

implementation:
path_temp_bviewfile = os . path . join ( c . raw_data , c . bview_dir , 'tmp' , filename )
path_bviewfile = os . path . join ( c . raw_data , c . bview_dir , filename )
try :
    f = urlopen ( url )
except :
    return False
if f . getcode ( ) != 200 :
    publisher . warning ( '{} unavailable, code: {}' . format ( url , f . getcode ( ) ) )
    return False
try :
    with open ( path_temp_bviewfile , 'w' ) as outfile :
        outfile . write ( f . read ( ) )
    os . rename ( path_temp_bviewfile , path_bviewfile )
except :
    os . remove ( path_temp_bviewfile )
    return False
return True


signature:
def already_downloaded ( filename ) :
    """Verify that the file has not already been downloaded ."""

implementation:
cur_file = os . path . join ( c . bview_dir , filename )
old_file = os . path . join ( c . bview_dir , 'old' , filename )
if not os . path . exists ( cur_file ) and not os . path . exists ( old_file ) :
    return False
return True


signature:
def to_download ( ) :
    """Build interval of urls to download .
 We always get the first file of the next day .
 Ex : 2013-01-01 = > 2013-01-02.0000"""

implementation:
first_day = parse ( interval_first )
last_day = parse ( interval_last )
format_change = parse ( '2010-06-14' )
one_day = datetime . timedelta ( 1 )
cur_day = first_day
url_list = [ ]
while cur_day < last_day :
    fname = filename . format ( day = cur_day . strftime ( "%Y%m%d" ) )
    if cur_day > format_change :
        cur_day += one_day
        url = base_url . format ( year_month = cur_day . strftime ( "%Y.%m" ) , file_day = cur_day . strftime ( "%Y%m%d" ) )
    else :
        url = base_url_old . format ( year_month = cur_day . strftime ( "%Y.%m" ) , file_day = cur_day . strftime ( "%Y%m%d" ) )
        cur_day += one_day
    url_list . append ( ( fname , url ) )
return sorted ( url_list , key = lambda tup : tup [ 0 ] , reverse = True )


signature:
def strToBool ( val ) :
    """Helper function to turn a string representation of " true " into
 boolean True ."""

implementation:
if isinstance ( val , str ) :
    val = val . lower ( )
return val in [ 'true' , 'on' , 'yes' , True ]


signature:
def get_page_url ( page_num , current_app , url_view_name , url_extra_args , url_extra_kwargs , url_param_name , url_get_params , url_anchor ) :
    """Helper function to return a valid URL string given the template tag parameters"""

implementation:
if url_view_name is not None : # Add page param to the kwargs list . Overrides any previously set parameter of the same name .
    url_extra_kwargs [ url_param_name ] = page_num
    try :
        url = reverse ( url_view_name , args = url_extra_args , kwargs = url_extra_kwargs , current_app = current_app )
    except NoReverseMatch as e : # Attempt to load view from application root , allowing the use of non - namespaced view names if your view is defined in the root application
        if settings . SETTINGS_MODULE :
            if django . VERSION < ( 1 , 9 , 0 ) :
                separator = '.'
            else :
                separator = ':'
            # Namespace separator changed to colon after 1.8
            project_name = settings . SETTINGS_MODULE . split ( '.' ) [ 0 ]
            try :
                url = reverse ( project_name + separator + url_view_name , args = url_extra_args , kwargs = url_extra_kwargs , current_app = current_app )
            except NoReverseMatch :
                raise e
                # Raise the original exception so the error message doesn ' t confusingly include something the Developer didn ' t add to the view name themselves
        else :
            raise e
# We can ' t determine the project name so just re - throw the exception
else :
    url = ''
    url_get_params = url_get_params or QueryDict ( url )
    url_get_params = url_get_params . copy ( )
    url_get_params [ url_param_name ] = str ( page_num )
if len ( url_get_params ) > 0 :
    if not isinstance ( url_get_params , QueryDict ) :
        tmp = QueryDict ( mutable = True )
        tmp . update ( url_get_params )
        url_get_params = tmp
    url += '?' + url_get_params . urlencode ( )
if ( url_anchor is not None ) :
    url += '#' + url_anchor
return url


signature:
def bootstrap_paginate ( parser , token ) :
    """Renders a Page object as a Twitter Bootstrap styled pagination bar .
 Compatible with Bootstrap 3 . x and 4 . x only .
 Example : :
 { % bootstrap _ paginate page _ obj range = 10 % }
 Named Parameters : :
 range - The size of the pagination bar ( ie , if set to 10 then , at most ,
 10 page numbers will display at any given time ) Defaults to
 None , which shows all pages .
 size - Accepts " small " , and " large " . Defaults to
 None which is the standard size .
 show _ prev _ next - Accepts " true " or " false " . Determines whether or not
 to show the previous and next page links . Defaults to
 " true "
 show _ first _ last - Accepts " true " or " false " . Determines whether or not
 to show the first and last page links . Defaults to
 " false "
 previous _ label - The text to display for the previous page link .
 Defaults to " & larr ; "
 next _ label - The text to display for the next page link . Defaults to
 " & rarr ; "
 first _ label - The text to display for the first page link . Defaults to
 " & laquo ; "
 last _ label - The text to display for the last page link . Defaults to
 " & raquo ; "
 url _ view _ name - The named URL to use . Defaults to None . If None , then the
 default template simply appends the url parameter as a
 relative URL link , eg : < a href = " ? page = 1 " > 1 < / a >
 url _ param _ name - The name of the parameter to use in the URL . If
 url _ view _ name is set to None , this string is used as the
 parameter name in the relative URL path . If a URL
 name is specified , this string is used as the
 parameter name passed into the reverse ( ) method for
 the URL .
 url _ extra _ args - This is used only in conjunction with url _ view _ name .
 When referencing a URL , additional arguments may be
 passed in as a list .
 url _ extra _ kwargs - This is used only in conjunction with url _ view _ name .
 When referencing a URL , additional named arguments
 may be passed in as a dictionary .
 url _ get _ params - The other get parameters to pass , only the page
 number will be overwritten . Use this to preserve
 filters .
 url _ anchor - The anchor to use in URLs . Defaults to None .
 extra _ pagination _ classes - A space separated list of CSS class names
 that will be added to the top level < ul >
 HTML element . In particular , this can be
 utilized in Bootstrap 4 installatinos to
 add the appropriate alignment classes from
 Flexbox utilites , eg : justify - content - center"""

implementation:
bits = token . split_contents ( )
if len ( bits ) < 2 :
    raise TemplateSyntaxError ( "'%s' takes at least one argument" " (Page object reference)" % bits [ 0 ] )
page = parser . compile_filter ( bits [ 1 ] )
kwargs = { }
bits = bits [ 2 : ]
kwarg_re = re . compile ( r'(\w+)=(.+)' )
if len ( bits ) :
    for bit in bits :
        match = kwarg_re . match ( bit )
        if not match :
            raise TemplateSyntaxError ( "Malformed arguments to bootstrap_pagination paginate tag" )
        name , value = match . groups ( )
        kwargs [ name ] = parser . compile_filter ( value )
return BootstrapPaginationNode ( page , kwargs )


signature:
def configure_devel_jobs ( config_url , rosdistro_name , source_build_name , groovy_script = None , dry_run = False , whitelist_repository_names = None ) :
    """Configure all Jenkins devel jobs .
 L { configure _ release _ job } will be invoked for source repository and target
 which matches the build file criteria ."""

implementation:
config = get_config_index ( config_url )
build_files = get_source_build_files ( config , rosdistro_name )
build_file = build_files [ source_build_name ]
index = get_index ( config . rosdistro_index_url )
dist_cache = None
if build_file . notify_maintainers :
    dist_cache = get_distribution_cache ( index , rosdistro_name )
# get targets
targets = [ ]
for os_name in build_file . targets . keys ( ) :
    for os_code_name in build_file . targets [ os_name ] . keys ( ) :
        for arch in build_file . targets [ os_name ] [ os_code_name ] :
            targets . append ( ( os_name , os_code_name , arch ) )
print ( 'The build file contains the following targets:' )
for os_name , os_code_name , arch in targets :
    print ( '  -' , os_name , os_code_name , arch )
dist_file = get_distribution_file ( index , rosdistro_name , build_file )
if not dist_file :
    print ( 'No distribution file matches the build file' )
    return
devel_view_name = get_devel_view_name ( rosdistro_name , source_build_name , pull_request = False )
pull_request_view_name = get_devel_view_name ( rosdistro_name , source_build_name , pull_request = True )
# all further configuration will be handled by either the Jenkins API
# or by a generated groovy script
from ros_buildfarm . jenkins import connect
jenkins = connect ( config . jenkins_url ) if groovy_script is None else False
view_configs = { }
views = { }
if build_file . test_commits_force is not False :
    views [ devel_view_name ] = configure_devel_view ( jenkins , devel_view_name , dry_run = dry_run )
if build_file . test_pull_requests_force is not False :
    views [ pull_request_view_name ] = configure_devel_view ( jenkins , pull_request_view_name , dry_run = dry_run )
if not jenkins :
    view_configs . update ( views )
groovy_data = { 'dry_run' : dry_run , 'expected_num_views' : len ( view_configs ) , }
repo_names = dist_file . repositories . keys ( )
filtered_repo_names = build_file . filter_repositories ( repo_names )
devel_job_names = [ ]
pull_request_job_names = [ ]
job_configs = OrderedDict ( )
for repo_name in sorted ( repo_names ) :
    if whitelist_repository_names :
        if repo_name not in whitelist_repository_names :
            print ( "Skipping repository '%s' not in explicitly passed list" % repo_name , file = sys . stderr )
            continue
    is_disabled = repo_name not in filtered_repo_names
    if is_disabled and build_file . skip_ignored_repositories :
        print ( "Skipping ignored repository '%s'" % repo_name , file = sys . stderr )
        continue
    repo = dist_file . repositories [ repo_name ]
    if not repo . source_repository :
        print ( "Skipping repository '%s': no source section" % repo_name )
        continue
    if not repo . source_repository . version :
        print ( "Skipping repository '%s': no source version" % repo_name )
        continue
    job_types = [ ]
    # check for testing commits
    if build_file . test_commits_force is False :
        print ( ( "Skipping repository '%s': 'test_commits' is forced to " + "false in the build file" ) % repo_name )
    elif repo . source_repository . test_commits is False :
        print ( ( "Skipping repository '%s': 'test_commits' of the " + "repository set to false" ) % repo_name )
    elif repo . source_repository . test_commits is None and not build_file . test_commits_default :
        print ( ( "Skipping repository '%s': 'test_commits' defaults to " + "false in the build file" ) % repo_name )
    else :
        job_types . append ( 'commit' )
    if not is_disabled : # check for testing pull requests
        if build_file . test_pull_requests_force is False : # print ( ( " Skipping repository ' % s ' : ' test _ pull _ requests ' " +
        # " is forced to false in the build file " ) % repo _ name )
            pass
        elif repo . source_repository . test_pull_requests is False : # print ( ( " Skipping repository ' % s ' : ' test _ pull _ requests ' of " +
        # " the repository set to false " ) % repo _ name )
            pass
        elif repo . source_repository . test_pull_requests is None and not build_file . test_pull_requests_default : # print ( ( " Skipping repository ' % s ' : ' test _ pull _ requests ' " +
        # " defaults to false in the build file " ) % repo _ name )
            pass
        else :
            print ( "Pull request job for repository '%s'" % repo_name )
            job_types . append ( 'pull_request' )
    for job_type in job_types :
        pull_request = job_type == 'pull_request'
        for os_name , os_code_name , arch in targets :
            try :
                job_name , job_config = configure_devel_job ( config_url , rosdistro_name , source_build_name , repo_name , os_name , os_code_name , arch , pull_request , config = config , build_file = build_file , index = index , dist_file = dist_file , dist_cache = dist_cache , jenkins = jenkins , views = views , is_disabled = is_disabled , groovy_script = groovy_script , dry_run = dry_run )
                if not pull_request :
                    devel_job_names . append ( job_name )
                else :
                    pull_request_job_names . append ( job_name )
                if groovy_script is not None :
                    print ( "Configuration for job '%s'" % job_name )
                    job_configs [ job_name ] = job_config
            except JobValidationError as e :
                print ( e . message , file = sys . stderr )
groovy_data [ 'expected_num_jobs' ] = len ( job_configs )
groovy_data [ 'job_prefixes_and_names' ] = { }
devel_job_prefix = '%s__' % devel_view_name
pull_request_job_prefix = '%s__' % pull_request_view_name
if not whitelist_repository_names :
    groovy_data [ 'job_prefixes_and_names' ] [ 'devel' ] = ( devel_job_prefix , devel_job_names )
    groovy_data [ 'job_prefixes_and_names' ] [ 'pull_request' ] = ( pull_request_job_prefix , pull_request_job_names )
    if groovy_script is None : # delete obsolete jobs in these views
        from ros_buildfarm . jenkins import remove_jobs
        print ( 'Removing obsolete devel jobs' )
        remove_jobs ( jenkins , devel_job_prefix , devel_job_names , dry_run = dry_run )
        print ( 'Removing obsolete pull request jobs' )
        remove_jobs ( jenkins , pull_request_job_prefix , pull_request_job_names , dry_run = dry_run )
if groovy_script is not None :
    print ( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % ( groovy_script , len ( view_configs ) , len ( job_configs ) ) )
    content = expand_template ( 'snippet/reconfigure_jobs.groovy.em' , groovy_data )
    write_groovy_script_and_configs ( groovy_script , content , job_configs , view_configs = view_configs )


signature:
def configure_devel_job ( config_url , rosdistro_name , source_build_name , repo_name , os_name , os_code_name , arch , pull_request = False , config = None , build_file = None , index = None , dist_file = None , dist_cache = None , jenkins = None , views = None , is_disabled = False , groovy_script = None , source_repository = None , build_targets = None , dry_run = False ) :
    """Configure a single Jenkins devel job .
 This includes the following steps :
 - clone the source repository to use
 - clone the ros _ buildfarm repository
 - write the distribution repository keys into files
 - invoke the release / run _ devel _ job . py script"""

implementation:
if config is None :
    config = get_config_index ( config_url )
if build_file is None :
    build_files = get_source_build_files ( config , rosdistro_name )
    build_file = build_files [ source_build_name ]
# Overwrite build _ file . targets if build _ targets is specified
if build_targets is not None :
    build_file . targets = build_targets
if index is None :
    index = get_index ( config . rosdistro_index_url )
if dist_file is None :
    dist_file = get_distribution_file ( index , rosdistro_name , build_file )
    if not dist_file :
        raise JobValidationError ( 'No distribution file matches the build file' )
repo_names = dist_file . repositories . keys ( )
if repo_name is not None :
    if repo_name not in repo_names :
        raise JobValidationError ( "Invalid repository name '%s' " % repo_name + 'choose one of the following: %s' % ', ' . join ( sorted ( repo_names ) ) )
    repo = dist_file . repositories [ repo_name ]
    if not repo . source_repository :
        raise JobValidationError ( "Repository '%s' has no source section" % repo_name )
    if not repo . source_repository . version :
        raise JobValidationError ( "Repository '%s' has no source version" % repo_name )
    source_repository = repo . source_repository
if os_name not in build_file . targets . keys ( ) :
    raise JobValidationError ( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets . keys ( ) ) ) )
if os_code_name not in build_file . targets [ os_name ] . keys ( ) :
    raise JobValidationError ( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets [ os_name ] . keys ( ) ) ) )
if arch not in build_file . targets [ os_name ] [ os_code_name ] :
    raise JobValidationError ( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', ' . join ( sorted ( build_file . targets [ os_name ] [ os_code_name ] ) ) )
if dist_cache is None and build_file . notify_maintainers :
    dist_cache = get_distribution_cache ( index , rosdistro_name )
if jenkins is None :
    from ros_buildfarm . jenkins import connect
    jenkins = connect ( config . jenkins_url )
if views is None :
    view_name = get_devel_view_name ( rosdistro_name , source_build_name , pull_request = pull_request )
    configure_devel_view ( jenkins , view_name , dry_run = dry_run )
job_name = get_devel_job_name ( rosdistro_name , source_build_name , repo_name , os_name , os_code_name , arch , pull_request )
job_config = _get_devel_job_config ( index , config , rosdistro_name , source_build_name , build_file , os_name , os_code_name , arch , source_repository , repo_name , pull_request , job_name , dist_cache = dist_cache , is_disabled = is_disabled )
# jenkinsapi . jenkins . Jenkins evaluates to false if job count is zero
if isinstance ( jenkins , object ) and jenkins is not False :
    from ros_buildfarm . jenkins import configure_job
    configure_job ( jenkins , job_name , job_config , dry_run = dry_run )
return job_name , job_config


signature:
def configure_release_jobs ( config_url , rosdistro_name , release_build_name , groovy_script = None , dry_run = False , whitelist_package_names = None ) :
    """Configure all Jenkins release jobs .
 L { configure _ release _ job } will be invoked for every released package and
 target which matches the build file criteria .
 Additionally a job to import Debian packages into the Debian repository is
 created ."""

implementation:
config = get_config_index ( config_url )
build_files = get_release_build_files ( config , rosdistro_name )
build_file = build_files [ release_build_name ]
index = get_index ( config . rosdistro_index_url )
# get targets
platforms = [ ]
for os_name in build_file . targets . keys ( ) :
    for os_code_name in build_file . targets [ os_name ] . keys ( ) :
        platforms . append ( ( os_name , os_code_name ) )
print ( 'The build file contains the following targets:' )
for os_name , os_code_name in platforms :
    print ( '  - %s %s: %s' % ( os_name , os_code_name , ', ' . join ( build_file . targets [ os_name ] [ os_code_name ] ) ) )
dist_file = get_distribution_file ( index , rosdistro_name , build_file )
if not dist_file :
    print ( 'No distribution file matches the build file' )
    return
pkg_names = dist_file . release_packages . keys ( )
filtered_pkg_names = build_file . filter_packages ( pkg_names )
explicitly_ignored_pkg_names = set ( pkg_names ) - set ( filtered_pkg_names )
if explicitly_ignored_pkg_names :
    print ( ( 'The following packages are being %s because of ' + 'white-/blacklisting:' ) % ( 'ignored' if build_file . skip_ignored_packages else 'disabled' ) )
    for pkg_name in sorted ( explicitly_ignored_pkg_names ) :
        print ( '  -' , pkg_name )
dist_cache = get_distribution_cache ( index , rosdistro_name )
if explicitly_ignored_pkg_names : # get direct dependencies from distro cache for each package
    direct_dependencies = { }
    for pkg_name in pkg_names :
        direct_dependencies [ pkg_name ] = _get_direct_dependencies ( pkg_name , dist_cache , pkg_names ) or set ( [ ] )
    # find recursive downstream deps for all explicitly ignored packages
    ignored_pkg_names = set ( explicitly_ignored_pkg_names )
    while True :
        implicitly_ignored_pkg_names = _get_downstream_package_names ( ignored_pkg_names , direct_dependencies )
        if implicitly_ignored_pkg_names - ignored_pkg_names :
            ignored_pkg_names |= implicitly_ignored_pkg_names
            continue
        break
    implicitly_ignored_pkg_names = ignored_pkg_names - explicitly_ignored_pkg_names
    if implicitly_ignored_pkg_names :
        print ( ( 'The following packages are being %s because their ' + 'dependencies are being ignored:' ) % ( 'ignored' if build_file . skip_ignored_packages else 'disabled' ) )
        for pkg_name in sorted ( implicitly_ignored_pkg_names ) :
            print ( '  -' , pkg_name )
        filtered_pkg_names = set ( filtered_pkg_names ) - implicitly_ignored_pkg_names
# all further configuration will be handled by either the Jenkins API
# or by a generated groovy script
jenkins = False
if groovy_script is None :
    from ros_buildfarm . jenkins import connect
    jenkins = connect ( config . jenkins_url )
all_view_configs = { }
all_job_configs = OrderedDict ( )
job_name , job_config = configure_import_package_job ( config_url , rosdistro_name , release_build_name , config = config , build_file = build_file , jenkins = jenkins , dry_run = dry_run )
if not jenkins :
    all_job_configs [ job_name ] = job_config
job_name , job_config = configure_sync_packages_to_main_job ( config_url , rosdistro_name , release_build_name , config = config , build_file = build_file , jenkins = jenkins , dry_run = dry_run )
if not jenkins :
    all_job_configs [ job_name ] = job_config
for os_name , os_code_name in platforms :
    for arch in sorted ( build_file . targets [ os_name ] [ os_code_name ] ) :
        job_name , job_config = configure_sync_packages_to_testing_job ( config_url , rosdistro_name , release_build_name , os_code_name , arch , config = config , build_file = build_file , jenkins = jenkins , dry_run = dry_run )
        if not jenkins :
            all_job_configs [ job_name ] = job_config
targets = [ ]
for os_name , os_code_name in platforms :
    targets . append ( ( os_name , os_code_name , 'source' ) )
    for arch in build_file . targets [ os_name ] [ os_code_name ] :
        targets . append ( ( os_name , os_code_name , arch ) )
views = configure_release_views ( jenkins , rosdistro_name , release_build_name , targets , dry_run = dry_run )
if not jenkins :
    all_view_configs . update ( views )
groovy_data = { 'dry_run' : dry_run , 'expected_num_views' : len ( views ) , }
# binary jobs must be generated in topological order
from catkin_pkg . package import parse_package_string
from ros_buildfarm . common import topological_order_packages
pkgs = { }
for pkg_name in pkg_names :
    if pkg_name not in dist_cache . release_package_xmls :
        print ( "Skipping package '%s': no released package.xml in cache" % ( pkg_name ) , file = sys . stderr )
        continue
    pkg_xml = dist_cache . release_package_xmls [ pkg_name ]
    pkg = parse_package_string ( pkg_xml )
    pkgs [ pkg_name ] = pkg
ordered_pkg_tuples = topological_order_packages ( pkgs )
other_build_files = [ v for k , v in build_files . items ( ) if k != release_build_name ]
all_source_job_names = [ ]
all_binary_job_names = [ ]
for pkg_name in [ p . name for _ , p in ordered_pkg_tuples ] :
    if whitelist_package_names :
        if pkg_name not in whitelist_package_names :
            print ( "Skipping package '%s' not in the explicitly passed list" % pkg_name , file = sys . stderr )
            continue
    pkg = dist_file . release_packages [ pkg_name ]
    repo_name = pkg . repository_name
    repo = dist_file . repositories [ repo_name ]
    is_disabled = pkg_name not in filtered_pkg_names
    if is_disabled and build_file . skip_ignored_packages :
        print ( "Skipping ignored package '%s' in repository '%s'" % ( pkg_name , repo_name ) , file = sys . stderr )
        continue
    if not repo . release_repository :
        print ( ( "Skipping package '%s' in repository '%s': no release " + "section" ) % ( pkg_name , repo_name ) , file = sys . stderr )
        continue
    if not repo . release_repository . version :
        print ( ( "Skipping package '%s' in repository '%s': no release " + "version" ) % ( pkg_name , repo_name ) , file = sys . stderr )
        continue
    for os_name , os_code_name in platforms :
        other_build_files_same_platform = [ ]
        for other_build_file in other_build_files :
            if os_name not in other_build_file . targets :
                continue
            if os_code_name not in other_build_file . targets [ os_name ] :
                continue
            other_build_files_same_platform . append ( other_build_file )
        try :
            source_job_names , binary_job_names , job_configs = configure_release_job ( config_url , rosdistro_name , release_build_name , pkg_name , os_name , os_code_name , config = config , build_file = build_file , index = index , dist_file = dist_file , dist_cache = dist_cache , jenkins = jenkins , views = views , generate_import_package_job = False , generate_sync_packages_jobs = False , is_disabled = is_disabled , other_build_files_same_platform = other_build_files_same_platform , groovy_script = groovy_script , dry_run = dry_run )
            all_source_job_names += source_job_names
            all_binary_job_names += binary_job_names
            if groovy_script is not None :
                print ( 'Configuration for jobs: ' + ', ' . join ( source_job_names + binary_job_names ) )
                for source_job_name in source_job_names :
                    all_job_configs [ source_job_name ] = job_configs [ source_job_name ]
                for binary_job_name in binary_job_names :
                    all_job_configs [ binary_job_name ] = job_configs [ binary_job_name ]
        except JobValidationError as e :
            print ( e . message , file = sys . stderr )
groovy_data [ 'expected_num_jobs' ] = len ( all_job_configs )
groovy_data [ 'job_prefixes_and_names' ] = { }
# with an explicit list of packages we don ' t delete obsolete jobs
if not whitelist_package_names : # delete obsolete binary jobs
    for os_name , os_code_name in platforms :
        for arch in build_file . targets [ os_name ] [ os_code_name ] :
            binary_view = get_release_binary_view_name ( rosdistro_name , release_build_name , os_name , os_code_name , arch )
            binary_job_prefix = '%s__' % binary_view
            excluded_job_names = set ( [ j for j in all_binary_job_names if j . startswith ( binary_job_prefix ) ] )
            if groovy_script is None :
                print ( "Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix )
                from ros_buildfarm . jenkins import remove_jobs
                remove_jobs ( jenkins , binary_job_prefix , excluded_job_names , dry_run = dry_run )
            else :
                binary_key = 'binary_%s_%s_%s' % ( os_name , os_code_name , arch )
                groovy_data [ 'job_prefixes_and_names' ] [ binary_key ] = ( binary_job_prefix , excluded_job_names )
    # delete obsolete source jobs
    # requires knowledge about all other release build files
    for os_name , os_code_name in platforms :
        other_source_job_names = [ ]
        # get source job names for all other release build files
        for other_release_build_name in [ k for k in build_files . keys ( ) if k != release_build_name ] :
            other_build_file = build_files [ other_release_build_name ]
            other_dist_file = get_distribution_file ( index , rosdistro_name , other_build_file )
            if not other_dist_file :
                continue
            if os_name not in other_build_file . targets or os_code_name not in other_build_file . targets [ os_name ] :
                continue
            if other_build_file . skip_ignored_packages :
                filtered_pkg_names = other_build_file . filter_packages ( pkg_names )
            else :
                filtered_pkg_names = pkg_names
            for pkg_name in sorted ( filtered_pkg_names ) :
                pkg = other_dist_file . release_packages [ pkg_name ]
                repo_name = pkg . repository_name
                repo = other_dist_file . repositories [ repo_name ]
                if not repo . release_repository :
                    continue
                if not repo . release_repository . version :
                    continue
                other_job_name = get_sourcedeb_job_name ( rosdistro_name , other_release_build_name , pkg_name , os_name , os_code_name )
                other_source_job_names . append ( other_job_name )
        source_view_prefix = get_release_source_view_name ( rosdistro_name , os_name , os_code_name )
        source_job_prefix = '%s__' % source_view_prefix
        excluded_job_names = set ( [ j for j in ( all_source_job_names + other_source_job_names ) if j . startswith ( source_job_prefix ) ] )
        if groovy_script is None :
            print ( "Removing obsolete source jobs with prefix '%s'" % source_job_prefix )
            from ros_buildfarm . jenkins import remove_jobs
            remove_jobs ( jenkins , source_job_prefix , excluded_job_names , dry_run = dry_run )
        else :
            source_key = 'source_%s_%s' % ( os_name , os_code_name )
            groovy_data [ 'job_prefixes_and_names' ] [ source_key ] = ( source_job_prefix , excluded_job_names )
if groovy_script is not None :
    print ( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % ( groovy_script , len ( all_view_configs ) , len ( all_job_configs ) ) )
    content = expand_template ( 'snippet/reconfigure_jobs.groovy.em' , groovy_data )
    write_groovy_script_and_configs ( groovy_script , content , all_job_configs , view_configs = all_view_configs )


signature:
def configure_release_job ( config_url , rosdistro_name , release_build_name , pkg_name , os_name , os_code_name , config = None , build_file = None , index = None , dist_file = None , dist_cache = None , jenkins = None , views = None , generate_import_package_job = True , generate_sync_packages_jobs = True , is_disabled = False , other_build_files_same_platform = None , groovy_script = None , filter_arches = None , dry_run = False ) :
    """Configure a Jenkins release job .
 The following jobs are created for each package :
 - M source jobs , one for each OS node name
 - M * N binary jobs , one for each combination of OS code name and arch"""

implementation:
if config is None :
    config = get_config_index ( config_url )
if build_file is None :
    build_files = get_release_build_files ( config , rosdistro_name )
    build_file = build_files [ release_build_name ]
if index is None :
    index = get_index ( config . rosdistro_index_url )
if dist_file is None :
    dist_file = get_distribution_file ( index , rosdistro_name , build_file )
    if not dist_file :
        raise JobValidationError ( 'No distribution file matches the build file' )
pkg_names = dist_file . release_packages . keys ( )
if pkg_name not in pkg_names :
    raise JobValidationError ( "Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', ' . join ( sorted ( pkg_names ) ) )
pkg = dist_file . release_packages [ pkg_name ]
repo_name = pkg . repository_name
repo = dist_file . repositories [ repo_name ]
if not repo . release_repository :
    raise JobValidationError ( "Repository '%s' has no release section" % repo_name )
if not repo . release_repository . version :
    raise JobValidationError ( "Repository '%s' has no release version" % repo_name )
if os_name not in build_file . targets . keys ( ) :
    raise JobValidationError ( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets . keys ( ) ) ) )
if os_code_name not in build_file . targets [ os_name ] . keys ( ) :
    raise JobValidationError ( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets [ os_name ] . keys ( ) ) ) )
if dist_cache is None and ( build_file . notify_maintainers or build_file . abi_incompatibility_assumed ) :
    dist_cache = get_distribution_cache ( index , rosdistro_name )
if jenkins is None :
    from ros_buildfarm . jenkins import connect
    jenkins = connect ( config . jenkins_url )
if views is None :
    targets = [ ]
    targets . append ( ( os_name , os_code_name , 'source' ) )
    for arch in build_file . targets [ os_name ] [ os_code_name ] :
        targets . append ( ( os_name , os_code_name , arch ) )
    configure_release_views ( jenkins , rosdistro_name , release_build_name , targets , dry_run = dry_run )
if generate_import_package_job :
    configure_import_package_job ( config_url , rosdistro_name , release_build_name , config = config , build_file = build_file , jenkins = jenkins , dry_run = dry_run )
if generate_sync_packages_jobs :
    configure_sync_packages_to_main_job ( config_url , rosdistro_name , release_build_name , config = config , build_file = build_file , jenkins = jenkins , dry_run = dry_run )
    for arch in build_file . targets [ os_name ] [ os_code_name ] :
        configure_sync_packages_to_testing_job ( config_url , rosdistro_name , release_build_name , os_code_name , arch , config = config , build_file = build_file , jenkins = jenkins , dry_run = dry_run )
source_job_names = [ ]
binary_job_names = [ ]
job_configs = { }
# sourcedeb job
# since sourcedeb jobs are potentially being shared across multiple build
# files the configuration has to take all of them into account in order to
# generate a job which all build files agree on
source_job_name = get_sourcedeb_job_name ( rosdistro_name , release_build_name , pkg_name , os_name , os_code_name )
# while the package is disabled in the current build file
# it might be used by sibling build files
is_source_disabled = is_disabled
if is_source_disabled and other_build_files_same_platform : # check if sourcedeb job is used by any other build file with the same platform
    for other_build_file in other_build_files_same_platform :
        if other_build_file . filter_packages ( [ pkg_name ] ) :
            is_source_disabled = False
            break
job_config = _get_sourcedeb_job_config ( config_url , rosdistro_name , release_build_name , config , build_file , os_name , os_code_name , pkg_name , repo_name , repo . release_repository , dist_cache = dist_cache , is_disabled = is_source_disabled , other_build_files_same_platform = other_build_files_same_platform )
# jenkinsapi . jenkins . Jenkins evaluates to false if job count is zero
if isinstance ( jenkins , object ) and jenkins is not False :
    from ros_buildfarm . jenkins import configure_job
    configure_job ( jenkins , source_job_name , job_config , dry_run = dry_run )
source_job_names . append ( source_job_name )
job_configs [ source_job_name ] = job_config
dependency_names = [ ]
if build_file . abi_incompatibility_assumed :
    dependency_names = _get_direct_dependencies ( pkg_name , dist_cache , pkg_names )
    # if dependencies are not yet available in rosdistro cache
    # skip binary jobs
    if dependency_names is None :
        print ( ( "Skipping binary jobs for package '%s' because it is not " + "yet in the rosdistro cache" ) % pkg_name , file = sys . stderr )
        return source_job_names , binary_job_names , job_configs
# binarydeb jobs
for arch in build_file . targets [ os_name ] [ os_code_name ] :
    if filter_arches and arch not in filter_arches :
        continue
    job_name = get_binarydeb_job_name ( rosdistro_name , release_build_name , pkg_name , os_name , os_code_name , arch )
    upstream_job_names = [ source_job_name ] + [ get_binarydeb_job_name ( rosdistro_name , release_build_name , dependency_name , os_name , os_code_name , arch ) for dependency_name in dependency_names ]
    job_config = _get_binarydeb_job_config ( config_url , rosdistro_name , release_build_name , config , build_file , os_name , os_code_name , arch , pkg_name , repo_name , repo . release_repository , dist_cache = dist_cache , upstream_job_names = upstream_job_names , is_disabled = is_disabled )
    # jenkinsapi . jenkins . Jenkins evaluates to false if job count is zero
    if isinstance ( jenkins , object ) and jenkins is not False :
        configure_job ( jenkins , job_name , job_config , dry_run = dry_run )
    binary_job_names . append ( job_name )
    job_configs [ job_name ] = job_config
return source_job_names , binary_job_names , job_configs


signature:
def configure_doc_jobs ( config_url , rosdistro_name , doc_build_name , groovy_script = None , dry_run = False , whitelist_repository_names = None ) :
    """Configure all Jenkins doc jobs .
 L { configure _ doc _ job } will be invoked for doc repository and target
 which matches the build file criteria ."""

implementation:
config = get_config_index ( config_url )
build_files = get_doc_build_files ( config , rosdistro_name )
build_file = build_files [ doc_build_name ]
index = get_index ( config . rosdistro_index_url )
dist_cache = None
if build_file . notify_maintainers :
    dist_cache = get_distribution_cache ( index , rosdistro_name )
# get targets
targets = [ ]
for os_name in build_file . targets . keys ( ) :
    for os_code_name in build_file . targets [ os_name ] . keys ( ) :
        for arch in build_file . targets [ os_name ] [ os_code_name ] :
            targets . append ( ( os_name , os_code_name , arch ) )
print ( 'The build file contains the following targets:' )
for os_name , os_code_name , arch in targets :
    print ( '  -' , os_name , os_code_name , arch )
dist_file = get_distribution_file ( index , rosdistro_name , build_file )
if not dist_file :
    print ( 'No distribution file matches the build file' )
    return
doc_view_name = get_doc_view_name ( rosdistro_name , doc_build_name )
# all further configuration will be handled by either the Jenkins API
# or by a generated groovy script
from ros_buildfarm . jenkins import connect
jenkins = connect ( config . jenkins_url ) if groovy_script is None else False
view_configs = { }
views = { }
views [ doc_view_name ] = configure_doc_view ( jenkins , doc_view_name , dry_run = dry_run )
if not jenkins :
    view_configs . update ( views )
groovy_data = { 'dry_run' : dry_run , 'expected_num_views' : len ( view_configs ) , }
repo_names = dist_file . repositories . keys ( )
filtered_repo_names = build_file . filter_repositories ( repo_names )
job_names = [ ]
job_configs = OrderedDict ( )
for repo_name in sorted ( repo_names ) :
    if whitelist_repository_names :
        if repo_name not in whitelist_repository_names :
            print ( "Skipping repository '%s' not in explicitly passed list" % repo_name , file = sys . stderr )
            continue
    is_disabled = repo_name not in filtered_repo_names
    if is_disabled and build_file . skip_ignored_repositories :
        print ( "Skipping ignored repository '%s'" % repo_name , file = sys . stderr )
        continue
    repo = dist_file . repositories [ repo_name ]
    if not repo . doc_repository :
        print ( "Skipping repository '%s': no doc section" % repo_name )
        continue
    if not repo . doc_repository . version :
        print ( "Skipping repository '%s': no doc version" % repo_name )
        continue
    for os_name , os_code_name , arch in targets :
        try :
            job_name , job_config = configure_doc_job ( config_url , rosdistro_name , doc_build_name , repo_name , os_name , os_code_name , arch , config = config , build_file = build_file , index = index , dist_file = dist_file , dist_cache = dist_cache , jenkins = jenkins , views = views , is_disabled = is_disabled , groovy_script = groovy_script , dry_run = dry_run )
            job_names . append ( job_name )
            if groovy_script is not None :
                print ( "Configuration for job '%s'" % job_name )
                job_configs [ job_name ] = job_config
        except JobValidationError as e :
            print ( e . message , file = sys . stderr )
groovy_data [ 'expected_num_jobs' ] = len ( job_configs )
groovy_data [ 'job_prefixes_and_names' ] = { }
job_prefix = '%s__' % doc_view_name
if not whitelist_repository_names :
    groovy_data [ 'job_prefixes_and_names' ] [ 'doc' ] = ( job_prefix , job_names )
    if groovy_script is None : # delete obsolete jobs in this view
        from ros_buildfarm . jenkins import remove_jobs
        print ( 'Removing obsolete doc jobs' )
        remove_jobs ( jenkins , job_prefix , job_names , dry_run = dry_run )
if groovy_script is not None :
    print ( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % ( groovy_script , len ( view_configs ) , len ( job_configs ) ) )
    content = expand_template ( 'snippet/reconfigure_jobs.groovy.em' , groovy_data )
    write_groovy_script_and_configs ( groovy_script , content , job_configs , view_configs = view_configs )


signature:
def configure_doc_job ( config_url , rosdistro_name , doc_build_name , repo_name , os_name , os_code_name , arch , config = None , build_file = None , index = None , dist_file = None , dist_cache = None , jenkins = None , views = None , is_disabled = False , groovy_script = None , doc_repository = None , dry_run = False ) :
    """Configure a single Jenkins doc job .
 This includes the following steps :
 - clone the doc repository to use
 - clone the ros _ buildfarm repository
 - write the distribution repository keys into files
 - invoke the run _ doc _ job . py script"""

implementation:
if config is None :
    config = get_config_index ( config_url )
if build_file is None :
    build_files = get_doc_build_files ( config , rosdistro_name )
    build_file = build_files [ doc_build_name ]
if index is None :
    index = get_index ( config . rosdistro_index_url )
if dist_file is None :
    dist_file = get_distribution_file ( index , rosdistro_name , build_file )
    if not dist_file :
        raise JobValidationError ( 'No distribution file matches the build file' )
repo_names = dist_file . repositories . keys ( )
if repo_name is not None :
    if repo_name not in repo_names :
        raise JobValidationError ( "Invalid repository name '%s' " % repo_name + 'choose one of the following: %s' % ', ' . join ( sorted ( repo_names ) ) )
    repo = dist_file . repositories [ repo_name ]
    if not repo . doc_repository :
        raise JobValidationError ( "Repository '%s' has no doc section" % repo_name )
    if not repo . doc_repository . version :
        raise JobValidationError ( "Repository '%s' has no doc version" % repo_name )
    doc_repository = repo . doc_repository
if os_name not in build_file . targets . keys ( ) :
    raise JobValidationError ( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets . keys ( ) ) ) )
if os_code_name not in build_file . targets [ os_name ] . keys ( ) :
    raise JobValidationError ( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets [ os_name ] . keys ( ) ) ) )
if arch not in build_file . targets [ os_name ] [ os_code_name ] :
    raise JobValidationError ( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', ' . join ( sorted ( build_file . targets [ os_name ] [ os_code_name ] ) ) )
if dist_cache is None and build_file . notify_maintainers :
    dist_cache = get_distribution_cache ( index , rosdistro_name )
if jenkins is None :
    from ros_buildfarm . jenkins import connect
    jenkins = connect ( config . jenkins_url )
if views is None :
    view_name = get_doc_view_name ( rosdistro_name , doc_build_name )
    configure_doc_view ( jenkins , view_name , dry_run = dry_run )
job_name = get_doc_job_name ( rosdistro_name , doc_build_name , repo_name , os_name , os_code_name , arch )
job_config = _get_doc_job_config ( config , config_url , rosdistro_name , doc_build_name , build_file , os_name , os_code_name , arch , doc_repository , repo_name , dist_cache = dist_cache , is_disabled = is_disabled )
# jenkinsapi . jenkins . Jenkins evaluates to false if job count is zero
if isinstance ( jenkins , object ) and jenkins is not False :
    from ros_buildfarm . jenkins import configure_job
    configure_job ( jenkins , job_name , job_config , dry_run = dry_run )
return job_name , job_config


signature:
def get_affected_by_sync ( package_descriptors , targets , testing_repo_data , main_repo_data ) :
    """For each package and target check if it is affected by a sync .
 This is the case when the package version in the testing repo is different
 from the version in the main repo .
 : return : a dict indexed by package names containing
 dicts indexed by targets containing a boolean flag"""

implementation:
affected_by_sync = { }
for package_descriptor in package_descriptors . values ( ) :
    pkg_name = package_descriptor . pkg_name
    debian_pkg_name = package_descriptor . debian_pkg_name
    affected_by_sync [ pkg_name ] = { }
    for target in targets :
        testing_version = _strip_version_suffix ( testing_repo_data . get ( target , { } ) . get ( debian_pkg_name , None ) )
        main_version = _strip_version_suffix ( main_repo_data . get ( target , { } ) . get ( debian_pkg_name , None ) )
        affected_by_sync [ pkg_name ] [ target ] = testing_version != main_version
return affected_by_sync


signature:
def get_regressions ( package_descriptors , targets , building_repo_data , testing_repo_data , main_repo_data ) :
    """For each package and target check if it is a regression .
 This is the case if the main repo contains a package version which is
 higher then in any of the other repos or if any of the other repos does not
 contain that package at all .
 : return : a dict indexed by package names containing
 dicts indexed by targets containing a boolean flag"""

implementation:
regressions = { }
for package_descriptor in package_descriptors . values ( ) :
    pkg_name = package_descriptor . pkg_name
    debian_pkg_name = package_descriptor . debian_pkg_name
    regressions [ pkg_name ] = { }
    for target in targets :
        regressions [ pkg_name ] [ target ] = False
        main_version = main_repo_data . get ( target , { } ) . get ( debian_pkg_name , None )
        if main_version is not None :
            main_ver_loose = LooseVersion ( main_version )
            for repo_data in [ building_repo_data , testing_repo_data ] :
                version = repo_data . get ( target , { } ) . get ( debian_pkg_name , None )
                if not version or main_ver_loose > LooseVersion ( version ) :
                    regressions [ pkg_name ] [ target ] = True
return regressions


signature:
def get_version_status ( package_descriptors , targets , repos_data , strip_version = False , strip_os_code_name = False ) :
    """For each package and target check if it is affected by a sync .
 This is the case when the package version in the testing repo is different
 from the version in the main repo .
 : return : a dict indexed by package names containing
 dicts indexed by targets containing
 a list of status strings ( one for each repo )"""

implementation:
status = { }
for package_descriptor in package_descriptors . values ( ) :
    pkg_name = package_descriptor . pkg_name
    debian_pkg_name = package_descriptor . debian_pkg_name
    ref_version = package_descriptor . version
    if strip_version :
        ref_version = _strip_version_suffix ( ref_version )
    status [ pkg_name ] = { }
    for target in targets :
        statuses = [ ]
        for repo_data in repos_data :
            version = repo_data . get ( target , { } ) . get ( debian_pkg_name , None )
            if strip_version :
                version = _strip_version_suffix ( version )
            if strip_os_code_name :
                version = _strip_os_code_name_suffix ( version , target . os_code_name )
            if ref_version :
                if not version :
                    statuses . append ( 'missing' )
                elif version . startswith ( ref_version ) : # including equal
                    statuses . append ( 'equal' )
                else :
                    if _version_is_gt_other ( version , ref_version ) :
                        statuses . append ( 'higher' )
                    else :
                        statuses . append ( 'lower' )
            else :
                if not version :
                    statuses . append ( 'ignore' )
                else :
                    statuses . append ( 'obsolete' )
        status [ pkg_name ] [ target ] = statuses
return status


signature:
def _strip_version_suffix ( version ) :
    """Remove trailing junk from the version number .
 > > > strip _ version _ suffix ( ' ' )
 > > > strip _ version _ suffix ( ' None ' )
 ' None '
 > > > strip _ version _ suffix ( ' 1.2.3-4trusty - 20140131-1359 - + 0000 ' )
 '1.2.3-4'
 > > > strip _ version _ suffix ( ' 1.2.3 - foo ' )
 '1.2.3'"""

implementation:
global version_regex
if not version :
    return version
match = version_regex . search ( version )
return match . group ( 0 ) if match else version


signature:
def get_homogeneous ( package_descriptors , targets , repos_data ) :
    """For each package check if the version in one repo is equal for all targets .
 The version could be different in different repos though .
 : return : a dict indexed by package names containing a boolean flag"""

implementation:
homogeneous = { }
for package_descriptor in package_descriptors . values ( ) :
    pkg_name = package_descriptor . pkg_name
    debian_pkg_name = package_descriptor . debian_pkg_name
    versions = [ ]
    for repo_data in repos_data :
        versions . append ( set ( [ ] ) )
        for target in targets :
            version = _strip_version_suffix ( repo_data . get ( target , { } ) . get ( debian_pkg_name , None ) )
            versions [ - 1 ] . add ( version )
    homogeneous [ pkg_name ] = max ( [ len ( v ) for v in versions ] ) == 1
return homogeneous


signature:
def get_package_counts ( package_descriptors , targets , repos_data ) :
    """Get the number of packages per target and repository .
 : return : a dict indexed by targets containing
 a list of integer values ( one for each repo )"""

implementation:
counts = { }
for target in targets :
    counts [ target ] = [ 0 ] * len ( repos_data )
for package_descriptor in package_descriptors . values ( ) :
    debian_pkg_name = package_descriptor . debian_pkg_name
    for target in targets :
        for i , repo_data in enumerate ( repos_data ) :
            version = repo_data . get ( target , { } ) . get ( debian_pkg_name , None )
            if version :
                counts [ target ] [ i ] += 1
return counts


signature:
def get_jenkins_job_urls ( rosdistro_name , jenkins_url , release_build_name , targets ) :
    """Get the Jenkins job urls for each target .
 The placeholder { pkg } needs to be replaced with the ROS package name .
 : return : a dict indexed by targets containing a string"""

implementation:
urls = { }
for target in targets :
    view_name = get_release_view_name ( rosdistro_name , release_build_name , target . os_name , target . os_code_name , target . arch )
    base_url = jenkins_url + '/view/%s/job/%s__{pkg}__' % ( view_name , view_name )
    if target . arch == 'source' :
        urls [ target ] = base_url + '%s_%s__source' % ( target . os_name , target . os_code_name )
    else :
        urls [ target ] = base_url + '%s_%s_%s__binary' % ( target . os_name , target . os_code_name , target . arch )
return urls


signature:
def configure_ci_jobs ( config_url , rosdistro_name , ci_build_name , groovy_script = None , dry_run = False ) :
    """Configure all Jenkins CI jobs ."""

implementation:
config = get_config_index ( config_url )
build_files = get_ci_build_files ( config , rosdistro_name )
build_file = build_files [ ci_build_name ]
index = get_index ( config . rosdistro_index_url )
# get targets
targets = [ ]
for os_name in build_file . targets . keys ( ) :
    for os_code_name in build_file . targets [ os_name ] . keys ( ) :
        for arch in build_file . targets [ os_name ] [ os_code_name ] :
            targets . append ( ( os_name , os_code_name , arch ) )
print ( 'The build file contains the following targets:' )
for os_name , os_code_name , arch in targets :
    print ( '  -' , os_name , os_code_name , arch )
dist_file = get_distribution_file ( index , rosdistro_name , build_file )
if not dist_file :
    print ( 'No distribution file matches the build file' )
    return
ci_view_name = get_ci_view_name ( rosdistro_name )
# all further configuration will be handled by either the Jenkins API
# or by a generated groovy script
from ros_buildfarm . jenkins import connect
jenkins = connect ( config . jenkins_url ) if groovy_script is None else False
view_configs = { }
views = { ci_view_name : configure_ci_view ( jenkins , ci_view_name , dry_run = dry_run ) }
if not jenkins :
    view_configs . update ( views )
groovy_data = { 'dry_run' : dry_run , 'expected_num_views' : len ( view_configs ) , }
ci_job_names = [ ]
job_configs = OrderedDict ( )
is_disabled = False
for os_name , os_code_name , arch in targets :
    try :
        job_name , job_config = configure_ci_job ( config_url , rosdistro_name , ci_build_name , os_name , os_code_name , arch , config = config , build_file = build_file , index = index , dist_file = dist_file , jenkins = jenkins , views = views , is_disabled = is_disabled , groovy_script = groovy_script , dry_run = dry_run , trigger_timer = build_file . jenkins_job_schedule )
        ci_job_names . append ( job_name )
        if groovy_script is not None :
            print ( "Configuration for job '%s'" % job_name )
            job_configs [ job_name ] = job_config
    except JobValidationError as e :
        print ( e . message , file = sys . stderr )
groovy_data [ 'expected_num_jobs' ] = len ( job_configs )
groovy_data [ 'job_prefixes_and_names' ] = { }
if groovy_script is not None :
    print ( "Writing groovy script '%s' to reconfigure %d jobs" % ( groovy_script , len ( job_configs ) ) )
    content = expand_template ( 'snippet/reconfigure_jobs.groovy.em' , groovy_data )
    write_groovy_script_and_configs ( groovy_script , content , job_configs , view_configs )


signature:
def configure_ci_job ( config_url , rosdistro_name , ci_build_name , os_name , os_code_name , arch , config = None , build_file = None , index = None , dist_file = None , jenkins = None , views = None , is_disabled = False , groovy_script = None , build_targets = None , dry_run = False , underlay_source_paths = None , trigger_timer = None ) :
    """Configure a single Jenkins CI job .
 This includes the following steps :
 - clone the ros _ buildfarm repository
 - write the distribution repository keys into files
 - invoke the ci / run _ ci _ job . py script"""

implementation:
if config is None :
    config = get_config_index ( config_url )
if build_file is None :
    build_files = get_ci_build_files ( config , rosdistro_name )
    build_file = build_files [ ci_build_name ]
# Overwrite build _ file . targets if build _ targets is specified
if build_targets is not None :
    build_file . targets = build_targets
if index is None :
    index = get_index ( config . rosdistro_index_url )
if dist_file is None :
    dist_file = get_distribution_file ( index , rosdistro_name , build_file )
    if not dist_file :
        raise JobValidationError ( 'No distribution file matches the build file' )
if os_name not in build_file . targets . keys ( ) :
    raise JobValidationError ( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets . keys ( ) ) ) )
if os_code_name not in build_file . targets [ os_name ] . keys ( ) :
    raise JobValidationError ( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', ' . join ( sorted ( build_file . targets [ os_name ] . keys ( ) ) ) )
if arch not in build_file . targets [ os_name ] [ os_code_name ] :
    raise JobValidationError ( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', ' . join ( sorted ( build_file . targets [ os_name ] [ os_code_name ] ) ) )
if len ( build_file . underlay_from_ci_jobs ) > 1 :
    raise JobValidationError ( 'Only a single underlay job is currently supported, but the ' + 'build file lists %d.' % len ( build_file . underlay_from_ci_jobs ) )
underlay_source_job = None
if build_file . underlay_from_ci_jobs :
    underlay_source_job = get_ci_job_name ( rosdistro_name , os_name , os_code_name , arch , build_file . underlay_from_ci_jobs [ 0 ] )
    underlay_source_paths = ( underlay_source_paths or [ ] ) + [ '$UNDERLAY_JOB_SPACE' ]
if jenkins is None :
    from ros_buildfarm . jenkins import connect
    jenkins = connect ( config . jenkins_url )
if views is None :
    view_name = get_ci_view_name ( rosdistro_name )
    configure_ci_view ( jenkins , view_name , dry_run = dry_run )
job_name = get_ci_job_name ( rosdistro_name , os_name , os_code_name , arch , ci_build_name )
job_config = _get_ci_job_config ( index , rosdistro_name , build_file , os_name , os_code_name , arch , build_file . repos_files , underlay_source_job , underlay_source_paths , trigger_timer , is_disabled = is_disabled )
# jenkinsapi . jenkins . Jenkins evaluates to false if job count is zero
if isinstance ( jenkins , object ) and jenkins is not False :
    from ros_buildfarm . jenkins import configure_job
    configure_job ( jenkins , job_name , job_config , dry_run = dry_run )
return job_name , job_config


signature:
def write_groovy_script_and_configs ( filename , content , job_configs , view_configs = None ) :
    """Write out the groovy script and configs to file .
 This writes the reconfigure script to the file location
 and places the expanded configs in subdirectories ' view _ configs ' /
 ' job _ configs ' that the script can then access when run ."""

implementation:
with open ( filename , 'w' ) as h :
    h . write ( content )
if view_configs :
    view_config_dir = os . path . join ( os . path . dirname ( filename ) , 'view_configs' )
    if not os . path . isdir ( view_config_dir ) :
        os . makedirs ( view_config_dir )
    for config_name , config_body in view_configs . items ( ) :
        config_filename = os . path . join ( view_config_dir , config_name )
        with open ( config_filename , 'w' ) as config_fh :
            config_fh . write ( config_body )
job_config_dir = os . path . join ( os . path . dirname ( filename ) , 'job_configs' )
if not os . path . isdir ( job_config_dir ) :
    os . makedirs ( job_config_dir )
# prefix each config file with a serial number to maintain order
format_str = '%0' + str ( len ( str ( len ( job_configs ) ) ) ) + 'd'
i = 0
for config_name , config_body in job_configs . items ( ) :
    i += 1
    config_filename = os . path . join ( job_config_dir , format_str % i + ' ' + config_name )
    with open ( config_filename , 'w' ) as config_fh :
        config_fh . write ( config_body )


signature:
def topological_order_packages ( packages ) :
    """Order packages topologically .
 First returning packages which have message generators and then
 the rest based on all direct depends and indirect recursive run _ depends .
 : param packages : A dict mapping relative paths to ` ` Package ` ` objects ` ` dict ` `
 : returns : A list of tuples containing the relative path and a ` ` Package ` ` object , ` ` list ` `"""

implementation:
from catkin_pkg . topological_order import _PackageDecorator
from catkin_pkg . topological_order import _sort_decorated_packages
decorators_by_name = { }
for path , package in packages . items ( ) :
    decorators_by_name [ package . name ] = _PackageDecorator ( package , path )
# calculate transitive dependencies
for decorator in decorators_by_name . values ( ) :
    decorator . depends_for_topological_order = set ( [ ] )
    all_depends = decorator . package . build_depends + decorator . package . buildtool_depends + decorator . package . run_depends + decorator . package . test_depends
    # skip external dependencies , meaning names that are not known packages
    unique_depend_names = set ( [ d . name for d in all_depends if d . name in decorators_by_name . keys ( ) ] )
    for name in unique_depend_names :
        if name in decorator . depends_for_topological_order : # avoid function call to improve performance
        # check within the loop since the set changes every cycle
            continue
        decorators_by_name [ name ] . _add_recursive_run_depends ( decorators_by_name , decorator . depends_for_topological_order )
ordered_pkg_tuples = _sort_decorated_packages ( decorators_by_name )
for pkg_path , pkg in ordered_pkg_tuples :
    if pkg_path is None :
        raise RuntimeError ( 'Circular dependency in: %s' % pkg )
return ordered_pkg_tuples


signature:
def parse_public ( data ) :
    """Loads a public key from a DER or PEM - formatted file . Supports RSA , DSA and
 EC public keys . For RSA keys , both the old RSAPublicKey and
 SubjectPublicKeyInfo structures are supported . Also allows extracting a
 public key from an X . 509 certificate .
 : param data :
 A byte string to load the public key from
 : raises :
 ValueError - when the data does not appear to contain a public key
 : return :
 An asn1crypto . keys . PublicKeyInfo object"""

implementation:
if not isinstance ( data , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            data must be a byte string, not %s
            ''' , type_name ( data ) ) )
key_type = None
# Appears to be PEM formatted
if data [ 0 : 5 ] == b'-----' :
    key_type , algo , data = _unarmor_pem ( data )
    if key_type == 'private key' :
        raise ValueError ( pretty_message ( '''
                The data specified does not appear to be a public key or
                certificate, but rather a private key
                ''' ) )
    # When a public key returning from _ unarmor _ pem has a known algorithm
    # of RSA , that means the DER structure is of the type RSAPublicKey , so
    # we need to wrap it in the PublicKeyInfo structure .
    if algo == 'rsa' :
        return keys . PublicKeyInfo . wrap ( data , 'rsa' )
if key_type is None or key_type == 'public key' :
    try :
        pki = keys . PublicKeyInfo . load ( data )
        # Call . native to fully parse since asn1crypto is lazy
        pki . native
        return pki
    except ( ValueError ) :
        pass
    # Data was not PublicKeyInfo
    try :
        rpk = keys . RSAPublicKey . load ( data )
        # Call . native to fully parse since asn1crypto is lazy
        rpk . native
        return keys . PublicKeyInfo . wrap ( rpk , 'rsa' )
    except ( ValueError ) :
        pass
# Data was not an RSAPublicKey
if key_type is None or key_type == 'certificate' :
    try :
        parsed_cert = x509 . Certificate . load ( data )
        key_info = parsed_cert [ 'tbs_certificate' ] [ 'subject_public_key_info' ]
        return key_info
    except ( ValueError ) :
        pass
# Data was not a cert
raise ValueError ( 'The data specified does not appear to be a known public key or certificate format' )


signature:
def parse_certificate ( data ) :
    """Loads a certificate from a DER or PEM - formatted file . Supports X . 509
 certificates only .
 : param data :
 A byte string to load the certificate from
 : raises :
 ValueError - when the data does not appear to contain a certificate
 : return :
 An asn1crypto . x509 . Certificate object"""

implementation:
if not isinstance ( data , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            data must be a byte string, not %s
            ''' , type_name ( data ) ) )
key_type = None
# Appears to be PEM formatted
if data [ 0 : 5 ] == b'-----' :
    key_type , _ , data = _unarmor_pem ( data )
    if key_type == 'private key' :
        raise ValueError ( pretty_message ( '''
                The data specified does not appear to be a certificate, but
                rather a private key
                ''' ) )
    if key_type == 'public key' :
        raise ValueError ( pretty_message ( '''
                The data specified does not appear to be a certificate, but
                rather a public key
                ''' ) )
if key_type is None or key_type == 'certificate' :
    try :
        return x509 . Certificate . load ( data )
    except ( ValueError ) :
        pass
# Data was not a Certificate
raise ValueError ( pretty_message ( '''
        The data specified does not appear to be a known certificate format
        ''' ) )


signature:
def parse_private ( data , password = None ) :
    """Loads a private key from a DER or PEM - formatted file . Supports RSA , DSA and
 EC private keys . Works with the follow formats :
 - RSAPrivateKey ( PKCS # 1)
 - ECPrivateKey ( SECG SEC1 V2)
 - DSAPrivateKey ( OpenSSL )
 - PrivateKeyInfo ( RSA / DSA / EC - PKCS # 8)
 - EncryptedPrivateKeyInfo ( RSA / DSA / EC - PKCS # 8)
 - Encrypted RSAPrivateKey ( PEM only , OpenSSL )
 - Encrypted DSAPrivateKey ( PEM only , OpenSSL )
 - Encrypted ECPrivateKey ( PEM only , OpenSSL )
 : param data :
 A byte string to load the private key from
 : param password :
 The password to unencrypt the private key
 : raises :
 ValueError - when the data does not appear to contain a private key , or the password is invalid
 : return :
 An asn1crypto . keys . PrivateKeyInfo object"""

implementation:
if not isinstance ( data , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            data must be a byte string, not %s
            ''' , type_name ( data ) ) )
if password is not None :
    if not isinstance ( password , byte_cls ) :
        raise TypeError ( pretty_message ( '''
                password must be a byte string, not %s
                ''' , type_name ( password ) ) )
else :
    password = b''
# Appears to be PEM formatted
if data [ 0 : 5 ] == b'-----' :
    key_type , _ , data = _unarmor_pem ( data , password )
    if key_type == 'public key' :
        raise ValueError ( pretty_message ( '''
                The data specified does not appear to be a private key, but
                rather a public key
                ''' ) )
    if key_type == 'certificate' :
        raise ValueError ( pretty_message ( '''
                The data specified does not appear to be a private key, but
                rather a certificate
                ''' ) )
try :
    pki = keys . PrivateKeyInfo . load ( data )
    # Call . native to fully parse since asn1crypto is lazy
    pki . native
    return pki
except ( ValueError ) :
    pass
# Data was not PrivateKeyInfo
try :
    parsed_wrapper = keys . EncryptedPrivateKeyInfo . load ( data )
    encryption_algorithm_info = parsed_wrapper [ 'encryption_algorithm' ]
    encrypted_data = parsed_wrapper [ 'encrypted_data' ] . native
    decrypted_data = _decrypt_encrypted_data ( encryption_algorithm_info , encrypted_data , password )
    pki = keys . PrivateKeyInfo . load ( decrypted_data )
    # Call . native to fully parse since asn1crypto is lazy
    pki . native
    return pki
except ( ValueError ) :
    pass
# Data was not EncryptedPrivateKeyInfo
try :
    parsed = keys . RSAPrivateKey . load ( data )
    # Call . native to fully parse since asn1crypto is lazy
    parsed . native
    return keys . PrivateKeyInfo . wrap ( parsed , 'rsa' )
except ( ValueError ) :
    pass
# Data was not an RSAPrivateKey
try :
    parsed = keys . DSAPrivateKey . load ( data )
    # Call . native to fully parse since asn1crypto is lazy
    parsed . native
    return keys . PrivateKeyInfo . wrap ( parsed , 'dsa' )
except ( ValueError ) :
    pass
# Data was not a DSAPrivateKey
try :
    parsed = keys . ECPrivateKey . load ( data )
    # Call . native to fully parse since asn1crypto is lazy
    parsed . native
    return keys . PrivateKeyInfo . wrap ( parsed , 'ec' )
except ( ValueError ) :
    pass
# Data was not an ECPrivateKey
raise ValueError ( pretty_message ( '''
        The data specified does not appear to be a known private key format
        ''' ) )