signature:
def create_packet ( header , data ) :
    """Creates an IncomingPacket object from header and data
 This method is for testing purposes"""

implementation:
packet = IncomingPacket ( )
packet . header = header
packet . data = data
if len ( header ) == HeronProtocol . HEADER_SIZE :
    packet . is_header_read = True
    if len ( data ) == packet . get_datasize ( ) :
        packet . is_complete = True
return packet


signature:
def read ( self , dispatcher ) :
    """Reads incoming data from asyncore . dispatcher"""

implementation:
try :
    if not self . is_header_read : # try reading header
        to_read = HeronProtocol . HEADER_SIZE - len ( self . header )
        self . header += dispatcher . recv ( to_read )
        if len ( self . header ) == HeronProtocol . HEADER_SIZE :
            self . is_header_read = True
        else :
            Log . debug ( "Header read incomplete; read %d bytes of header" % len ( self . header ) )
            return
    if self . is_header_read and not self . is_complete : # try reading data
        to_read = self . get_datasize ( ) - len ( self . data )
        self . data += dispatcher . recv ( to_read )
        if len ( self . data ) == self . get_datasize ( ) :
            self . is_complete = True
except socket . error as e :
    if e . errno == socket . errno . EAGAIN or e . errno == socket . errno . EWOULDBLOCK : # Try again later - > call continue _ read later
        Log . debug ( "Try again error" )
    else : # Fatal error
        Log . debug ( "Fatal error when reading IncomingPacket" )
        raise RuntimeError ( "Fatal error occured in IncomingPacket.read()" )


signature:
def generate ( ) :
    """Generates a random REQID for request"""

implementation:
data_bytes = bytearray ( random . getrandbits ( 8 ) for i in range ( REQID . REQID_SIZE ) )
return REQID ( data_bytes )


signature:
def create_parser ( subparsers ) :
    ''': param subparsers :
 : return :'''

implementation:
parser = subparsers . add_parser ( 'restart' , help = 'Restart a topology' , usage = "%(prog)s [options] cluster/[role]/[env] <topology-name> [container-id]" , add_help = True )
args . add_titles ( parser )
args . add_cluster_role_env ( parser )
args . add_topology ( parser )
parser . add_argument ( 'container-id' , nargs = '?' , type = int , default = - 1 , help = 'Identifier of the container to be restarted' )
args . add_config ( parser )
args . add_service_url ( parser )
args . add_verbose ( parser )
parser . set_defaults ( subcommand = 'restart' )
return parser


signature:
def run ( command , parser , cl_args , unknown_args ) :
    ''': param command :
 : param parser :
 : param cl _ args :
 : param unknown _ args :
 : return :'''

implementation:
Log . debug ( "Restart Args: %s" , cl_args )
container_id = cl_args [ 'container-id' ]
if cl_args [ 'deploy_mode' ] == config . SERVER_MODE :
    dict_extra_args = { "container_id" : str ( container_id ) }
    return cli_helper . run_server ( command , cl_args , "restart topology" , extra_args = dict_extra_args )
else :
    list_extra_args = [ "--container_id" , str ( container_id ) ]
    return cli_helper . run_direct ( command , cl_args , "restart topology" , extra_args = list_extra_args )


signature:
def get_heron_config ( ) :
    '''Get config opts from the global variable
 : return :'''

implementation:
opt_list = [ ]
for ( key , value ) in config_opts . items ( ) :
    opt_list . append ( '%s=%s' % ( key , value ) )
all_opts = ( ',' . join ( opt_list ) ) . replace ( ' ' , '%%%%' )
return all_opts


signature:
def yaml_config_reader ( config_path ) :
    """Reads yaml config file and returns auto - typed config _ dict"""

implementation:
if not config_path . endswith ( ".yaml" ) :
    raise ValueError ( "Config file not yaml" )
with open ( config_path , 'r' ) as f :
    config = yaml . load ( f )
return config


signature:
def handle_new_tuple_set_2 ( self , hts2 ) :
    """Called when new HeronTupleSet2 arrives
 Convert ( Assemble ) HeronTupleSet2 ( raw byte array ) to HeronTupleSet
 See more at GitHub PR # 1421
 : param tuple _ msg _ set : HeronTupleSet2 type"""

implementation:
if self . my_pplan_helper is None or self . my_instance is None :
    Log . error ( "Got tuple set when no instance assigned yet" )
else :
    hts = tuple_pb2 . HeronTupleSet ( )
    if hts2 . HasField ( 'control' ) :
        hts . control . CopyFrom ( hts2 . control )
    else :
        hdts = tuple_pb2 . HeronDataTupleSet ( )
        hdts . stream . CopyFrom ( hts2 . data . stream )
        try :
            for trunk in hts2 . data . tuples :
                added_tuple = hdts . tuples . add ( )
                added_tuple . ParseFromString ( trunk )
        except Exception :
            Log . exception ( 'Fail to deserialize HeronDataTuple' )
        hts . data . CopyFrom ( hdts )
    self . in_stream . offer ( hts )
    if self . my_pplan_helper . is_topology_running ( ) :
        self . my_instance . py_class . process_incoming_tuples ( )


signature:
def handle_initiate_stateful_checkpoint ( self , ckptmsg ) :
    """Called when we get InitiateStatefulCheckpoint message
 : param ckptmsg : InitiateStatefulCheckpoint type"""

implementation:
self . in_stream . offer ( ckptmsg )
if self . my_pplan_helper . is_topology_running ( ) :
    self . my_instance . py_class . process_incoming_tuples ( )


signature:
def handle_start_stateful_processing ( self , start_msg ) :
    """Called when we receive StartInstanceStatefulProcessing message
 : param start _ msg : StartInstanceStatefulProcessing type"""

implementation:
Log . info ( "Received start stateful processing for %s" % start_msg . checkpoint_id )
self . is_stateful_started = True
self . start_instance_if_possible ( )


signature:
def handle_restore_instance_state ( self , restore_msg ) :
    """Called when we receive RestoreInstanceStateRequest message
 : param restore _ msg : RestoreInstanceStateRequest type"""

implementation:
Log . info ( "Restoring instance state to checkpoint %s" % restore_msg . state . checkpoint_id )
# Stop the instance
if self . is_stateful_started :
    self . my_instance . py_class . stop ( )
    self . my_instance . py_class . clear_collector ( )
    self . is_stateful_started = False
# Clear all buffers
self . in_stream . clear ( )
self . out_stream . clear ( )
# Deser the state
if self . stateful_state is not None :
    self . stateful_state . clear ( )
if restore_msg . state . state is not None and restore_msg . state . state :
    try :
        self . stateful_state = self . serializer . deserialize ( restore_msg . state . state )
    except Exception as e :
        raise RuntimeError ( "Could not serialize state during restore " + str ( e ) )
else :
    Log . info ( "The restore request does not have an actual state" )
if self . stateful_state is None :
    self . stateful_state = HashMapState ( )
Log . info ( "Instance restore state deserialized" )
# Send the response back
resp = ckptmgr_pb2 . RestoreInstanceStateResponse ( )
resp . status . status = common_pb2 . StatusCode . Value ( "OK" )
resp . checkpoint_id = restore_msg . state . checkpoint_id
self . _stmgr_client . send_message ( resp )


signature:
def send_buffered_messages ( self ) :
    """Send messages in out _ stream to the Stream Manager"""

implementation:
while not self . out_stream . is_empty ( ) and self . _stmgr_client . is_registered :
    tuple_set = self . out_stream . poll ( )
    if isinstance ( tuple_set , tuple_pb2 . HeronTupleSet ) :
        tuple_set . src_task_id = self . my_pplan_helper . my_task_id
        self . gateway_metrics . update_sent_packet ( tuple_set . ByteSize ( ) )
    self . _stmgr_client . send_message ( tuple_set )


signature:
def _handle_state_change_msg ( self , new_helper ) :
    """Called when state change is commanded by stream manager"""

implementation:
assert self . my_pplan_helper is not None
assert self . my_instance is not None and self . my_instance . py_class is not None
if self . my_pplan_helper . get_topology_state ( ) != new_helper . get_topology_state ( ) : # handle state change
# update the pplan _ helper
    self . my_pplan_helper = new_helper
    if new_helper . is_topology_running ( ) :
        if not self . is_instance_started :
            self . start_instance_if_possible ( )
        self . my_instance . py_class . invoke_activate ( )
    elif new_helper . is_topology_paused ( ) :
        self . my_instance . py_class . invoke_deactivate ( )
    else :
        raise RuntimeError ( "Unexpected TopologyState update: %s" % new_helper . get_topology_state ( ) )
else :
    Log . info ( "Topology state remains the same." )


signature:
def handle_assignment_msg ( self , pplan ) :
    """Called when new NewInstanceAssignmentMessage arrives
 Tells this instance to become either spout / bolt .
 : param pplan : PhysicalPlan proto"""

implementation:
new_helper = PhysicalPlanHelper ( pplan , self . instance . instance_id , self . topo_pex_file_abs_path )
if self . my_pplan_helper is not None and ( self . my_pplan_helper . my_component_name != new_helper . my_component_name or self . my_pplan_helper . my_task_id != new_helper . my_task_id ) :
    raise RuntimeError ( "Our Assignment has changed. We will die to pick it." )
new_helper . set_topology_context ( self . metrics_collector )
if self . my_pplan_helper is None :
    Log . info ( "Received a new Physical Plan" )
    Log . info ( "Push the new pplan_helper to Heron Instance" )
    self . _handle_assignment_msg ( new_helper )
else :
    Log . info ( "Received a new Physical Plan with the same assignment -- State Change" )
    Log . info ( "Old state: %s, new state: %s." , self . my_pplan_helper . get_topology_state ( ) , new_helper . get_topology_state ( ) )
    self . _handle_state_change_msg ( new_helper )


signature:
def check_output_schema ( self , stream_id , tup ) :
    """Checks if a given stream _ id and tuple matches with the output schema
 : type stream _ id : str
 : param stream _ id : stream id into which tuple is sent
 : type tup : list
 : param tup : tuple that is going to be sent"""

implementation:
# do some checking to make sure that the number of fields match what ' s expected
size = self . _output_schema . get ( stream_id , None )
if size is None :
    raise RuntimeError ( "%s emitting to stream %s but was not declared in output fields" % ( self . my_component_name , stream_id ) )
elif size != len ( tup ) :
    raise RuntimeError ( "Number of fields emitted in stream %s does not match what's expected. " "Expected: %s, Observed: %s" % ( stream_id , size , len ( tup ) ) )


signature:
def get_topology_config ( self ) :
    """Returns the topology config"""

implementation:
if self . pplan . topology . HasField ( "topology_config" ) :
    return self . _get_dict_from_config ( self . pplan . topology . topology_config )
else :
    return { }


signature:
def set_topology_context ( self , metrics_collector ) :
    """Sets a new topology context"""

implementation:
Log . debug ( "Setting topology context" )
cluster_config = self . get_topology_config ( )
cluster_config . update ( self . _get_dict_from_config ( self . my_component . config ) )
task_to_component_map = self . _get_task_to_comp_map ( )
self . context = TopologyContextImpl ( cluster_config , self . pplan . topology , task_to_component_map , self . my_task_id , metrics_collector , self . topology_pex_abs_path )


signature:
def _get_dict_from_config ( topology_config ) :
    """Converts Config protobuf message to python dictionary
 Values are converted according to the rules below :
 - Number string ( e . g . " 12 " or " 1.2 " ) is appropriately converted to ` ` int ` ` or ` ` float ` `
 - Boolean string ( " true " , " True " , " false " or " False " ) is converted to built - in boolean type
 ( i . e . ` ` True ` ` or ` ` False ` ` )
 - Normal string is inserted to dict as is
 - Serialized value is deserialized and inserted as a corresponding Python object"""

implementation:
config = { }
for kv in topology_config . kvs :
    if kv . HasField ( "value" ) :
        assert kv . type == topology_pb2 . ConfigValueType . Value ( "STRING_VALUE" )
        # value is string
        if PhysicalPlanHelper . _is_number ( kv . value ) :
            config [ kv . key ] = PhysicalPlanHelper . _get_number ( kv . value )
        elif kv . value . lower ( ) in ( "true" , "false" ) :
            config [ kv . key ] = True if kv . value . lower ( ) == "true" else False
        else :
            config [ kv . key ] = kv . value
    elif kv . HasField ( "serialized_value" ) and kv . type == topology_pb2 . ConfigValueType . Value ( "PYTHON_SERIALIZED_VALUE" ) : # deserialize that
        config [ kv . key ] = default_serializer . deserialize ( kv . serialized_value )
    else :
        assert kv . HasField ( "type" )
        Log . error ( "Unsupported config <key:value> found: %s, with type: %s" % ( str ( kv ) , str ( kv . type ) ) )
        continue
return config


signature:
def _setup_custom_grouping ( self , topology ) :
    """Checks whether there are any bolts that consume any of my streams using custom grouping"""

implementation:
for i in range ( len ( topology . bolts ) ) :
    for in_stream in topology . bolts [ i ] . inputs :
        if in_stream . stream . component_name == self . my_component_name and in_stream . gtype == topology_pb2 . Grouping . Value ( "CUSTOM" ) : # this bolt takes my output in custom grouping manner
            if in_stream . type == topology_pb2 . CustomGroupingObjectType . Value ( "PYTHON_OBJECT" ) :
                custom_grouping_obj = default_serializer . deserialize ( in_stream . custom_grouping_object )
                if isinstance ( custom_grouping_obj , str ) :
                    pex_loader . load_pex ( self . topology_pex_abs_path )
                    grouping_cls = pex_loader . import_and_get_class ( self . topology_pex_abs_path , custom_grouping_obj )
                    custom_grouping_obj = grouping_cls ( )
                assert isinstance ( custom_grouping_obj , ICustomGrouping )
                self . custom_grouper . add ( in_stream . stream . id , self . _get_taskids_for_component ( topology . bolts [ i ] . comp . name ) , custom_grouping_obj , self . my_component_name )
            elif in_stream . type == topology_pb2 . CustomGroupingObjectType . Value ( "JAVA_OBJECT" ) :
                raise NotImplementedError ( "Java-serialized custom grouping is not yet supported " "for python topology" )
            else :
                raise ValueError ( "Unrecognized custom grouping type found: %s" % str ( in_stream . type ) )


signature:
def add ( self , stream_id , task_ids , grouping , source_comp_name ) :
    """Adds the target component
 : type stream _ id : str
 : param stream _ id : stream id into which tuples are emitted
 : type task _ ids : list of str
 : param task _ ids : list of task ids to which tuples are emitted
 : type grouping : ICustomStreamGrouping object
 : param grouping : custom grouping to use
 : type source _ comp _ name : str
 : param source _ comp _ name : source component name"""

implementation:
if stream_id not in self . targets :
    self . targets [ stream_id ] = [ ]
self . targets [ stream_id ] . append ( Target ( task_ids , grouping , source_comp_name ) )


signature:
def prepare ( self , context ) :
    """Prepares the custom grouping for this component"""

implementation:
for stream_id , targets in self . targets . items ( ) :
    for target in targets :
        target . prepare ( context , stream_id )


signature:
def choose_tasks ( self , stream_id , values ) :
    """Choose tasks for a given stream _ id and values and Returns a list of target tasks"""

implementation:
if stream_id not in self . targets :
    return [ ]
ret = [ ]
for target in self . targets [ stream_id ] :
    ret . extend ( target . choose_tasks ( values ) )
return ret


signature:
def prepare ( self , context , stream_id ) :
    """Invoke prepare ( ) of this custom grouping"""

implementation: self . grouping . prepare ( context , self . source_comp_name , stream_id , self . task_ids )


signature:
def choose_tasks ( self , values ) :
    """Invoke choose _ tasks ( ) of this custom grouping"""

implementation:
ret = self . grouping . choose_tasks ( values )
if not isinstance ( ret , list ) :
    raise TypeError ( "Returned object after custom grouping's choose_tasks() " "needs to be a list, given: %s" % str ( type ( ret ) ) )
else :
    for i in ret :
        if not isinstance ( i , int ) :
            raise TypeError ( "Returned object after custom grouping's choose_tasks() " "contained non-integer: %s" % str ( i ) )
        if i not in self . task_ids :
            raise ValueError ( "Returned object after custom grouping's choose_tasks() contained " "a task id that is not registered: %d" % i )
    return ret


signature:
def add_config ( parser ) :
    ''': param parser :
 : return :'''

implementation:
# the default config path
default_config_path = config . get_heron_conf_dir ( )
parser . add_argument ( '--config-path' , default = os . path . join ( config . get_heron_dir ( ) , default_config_path ) , help = 'Path to cluster configuration files' )
parser . add_argument ( '--config-property' , metavar = 'PROPERTY=VALUE' , action = 'append' , default = [ ] , help = 'Configuration properties that overrides default options' )
return parser


signature:
def add_dry_run ( parser ) :
    ''': param parser :
 : return :'''

implementation:
default_format = 'table'
resp_formats = [ 'raw' , 'table' , 'colored_table' , 'json' ]
available_options = ', ' . join ( [ '%s' % opt for opt in resp_formats ] )
def dry_run_resp_format ( value ) :
    if value not in resp_formats :
        raise argparse . ArgumentTypeError ( 'Invalid dry-run response format: %s. Available formats: %s' % ( value , available_options ) )
    return value
parser . add_argument ( '--dry-run' , default = False , action = 'store_true' , help = 'Enable dry-run mode. Information about ' 'the command will print but no action will be taken on the topology' )
parser . add_argument ( '--dry-run-format' , metavar = 'DRY_RUN_FORMAT' , default = 'colored_table' if sys . stdout . isatty ( ) else 'table' , type = dry_run_resp_format , help = 'The format of the dry-run output ([%s], default=%s). ' 'Ignored when dry-run mode is not enabled' % ( '|' . join ( resp_formats ) , default_format ) )
return parser


signature:
def read_server_mode_cluster_definition ( cluster , cl_args ) :
    '''Read the cluster definition for server mode
 : param cluster :
 : param cl _ args :
 : param config _ file :
 : return :'''

implementation:
client_confs = dict ( )
client_confs [ cluster ] = cliconfig . cluster_config ( cluster )
# now check if the service - url from command line is set , if so override it
if cl_args . get ( 'service_url' , None ) :
    client_confs [ cluster ] [ 'service_url' ] = cl_args [ 'service_url' ]
# the return value of yaml . load can be None if conf _ file is an empty file
# or there is no service - url in command line , if needed .
return client_confs


signature:
def check_direct_mode_cluster_definition ( cluster , config_path ) :
    '''Check the cluster definition for direct mode
 : param cluster :
 : param config _ path :
 : return :'''

implementation:
config_path = config . get_heron_cluster_conf_dir ( cluster , config_path )
if not os . path . isdir ( config_path ) :
    return False
return True


signature:
def get ( self , path ) :
    '''get method'''

implementation:
if not path :
    path = "."
if not utils . check_path ( path ) :
    self . write ( "Only relative paths are allowed" )
    self . set_status ( 403 )
    self . finish ( )
    return
t = Template ( utils . get_asset ( "browse.html" ) )
args = dict ( path = path , listing = utils . get_listing ( path ) , format_prefix = utils . format_prefix , stat = stat , get_stat = utils . get_stat , os = os , css = utils . get_asset ( "bootstrap.css" ) )
self . write ( t . generate ( ** args ) )
self . finish ( )


signature:
def format_mtime ( mtime ) :
    """Format the date associated with a file to be displayed in directory listing ."""

implementation:
now = datetime . now ( )
dt = datetime . fromtimestamp ( mtime )
return '%s %2d %5s' % ( dt . strftime ( '%b' ) , dt . day , dt . year if dt . year != now . year else dt . strftime ( '%H:%M' ) )


signature:
def format_prefix ( filename , sres ) :
    """Prefix to a filename in the directory listing . This is to make the
 listing similar to an output of " ls - alh " ."""

implementation:
try :
    pwent = pwd . getpwuid ( sres . st_uid )
    user = pwent . pw_name
except KeyError :
    user = sres . st_uid
try :
    grent = grp . getgrgid ( sres . st_gid )
    group = grent . gr_name
except KeyError :
    group = sres . st_gid
return '%s %3d %10s %10s %10d %s' % ( format_mode ( sres ) , sres . st_nlink , user , group , sres . st_size , format_mtime ( sres . st_mtime ) , )


signature:
def get_listing ( path ) :
    """Returns the list of files and directories in a path .
 Prepents a " . . " ( parent directory link ) if path is not current dir ."""

implementation:
if path != "." :
    listing = sorted ( [ '..' ] + os . listdir ( path ) )
else :
    listing = sorted ( os . listdir ( path ) )
return listing


signature:
def get_stat ( path , filename ) :
    '''get stat'''

implementation: return os . stat ( os . path . join ( path , filename ) )


signature:
def read_chunk ( filename , offset = - 1 , length = - 1 , escape_data = False ) :
    """Read a chunk of a file from an offset upto the length ."""

implementation:
try :
    length = int ( length )
    offset = int ( offset )
except ValueError :
    return { }
if not os . path . isfile ( filename ) :
    return { }
try :
    fstat = os . stat ( filename )
except Exception :
    return { }
if offset == - 1 :
    offset = fstat . st_size
if length == - 1 :
    length = fstat . st_size - offset
with open ( filename , "r" ) as fp :
    fp . seek ( offset )
    try :
        data = fp . read ( length )
    except IOError :
        return { }
if data :
    data = _escape_data ( data ) if escape_data else data
    return dict ( offset = offset , length = len ( data ) , data = data )
return dict ( offset = offset , length = 0 )


signature:
def pipe ( prev_proc , to_cmd ) :
    """Pipes output of prev _ proc into to _ cmd .
 Returns piped process"""

implementation:
stdin = None if prev_proc is None else prev_proc . stdout
process = subprocess . Popen ( to_cmd , stdout = subprocess . PIPE , stdin = stdin )
if prev_proc is not None :
    prev_proc . stdout . close ( )
    # Allow prev _ proc to receive a SIGPIPE
return process


signature:
def str_cmd ( cmd , cwd , env ) :
    """Runs the command and returns its stdout and stderr ."""

implementation:
process = subprocess . Popen ( cmd , stdout = subprocess . PIPE , stderr = subprocess . PIPE , cwd = cwd , env = env )
stdout_builder , stderr_builder = proc . async_stdout_stderr_builder ( process )
process . wait ( )
stdout , stderr = stdout_builder . result ( ) , stderr_builder . result ( )
return { 'command' : ' ' . join ( cmd ) , 'stderr' : stderr , 'stdout' : stdout }


signature:
def chain ( cmd_list ) :
    """Feed output of one command to the next and return final output
 Returns string output of chained application of commands ."""

implementation:
command = ' | ' . join ( map ( lambda x : ' ' . join ( x ) , cmd_list ) )
chained_proc = functools . reduce ( pipe , [ None ] + cmd_list )
stdout_builder = proc . async_stdout_builder ( chained_proc )
chained_proc . wait ( )
return { 'command' : command , 'stdout' : stdout_builder . result ( ) }


signature:
def create_parser ( subparsers ) :
    """create parser"""

implementation:
metrics_parser = subparsers . add_parser ( 'metrics' , help = 'Display info of a topology\'s metrics' , usage = "%(prog)s cluster/[role]/[env] topology-name [options]" , add_help = False )
args . add_cluster_role_env ( metrics_parser )
args . add_topology_name ( metrics_parser )
args . add_verbose ( metrics_parser )
args . add_tracker_url ( metrics_parser )
args . add_config ( metrics_parser )
args . add_component_name ( metrics_parser )
metrics_parser . set_defaults ( subcommand = 'metrics' )
containers_parser = subparsers . add_parser ( 'containers' , help = 'Display info of a topology\'s containers metrics' , usage = "%(prog)s cluster/[role]/[env] topology-name [options]" , add_help = False )
args . add_cluster_role_env ( containers_parser )
args . add_topology_name ( containers_parser )
args . add_verbose ( containers_parser )
args . add_tracker_url ( containers_parser )
args . add_config ( containers_parser )
args . add_container_id ( containers_parser )
containers_parser . set_defaults ( subcommand = 'containers' )
return subparsers


signature:
def parse_topo_loc ( cl_args ) :
    """parse topology location"""

implementation:
try :
    topo_loc = cl_args [ 'cluster/[role]/[env]' ] . split ( '/' )
    topo_name = cl_args [ 'topology-name' ]
    topo_loc . append ( topo_name )
    if len ( topo_loc ) != 4 :
        raise
    return topo_loc
except Exception :
    Log . error ( 'Invalid topology location' )
    raise


signature:
def to_table ( metrics ) :
    """normalize raw metrics API result to table"""

implementation:
all_queries = tracker_access . metric_queries ( )
m = tracker_access . queries_map ( )
names = metrics . values ( ) [ 0 ] . keys ( )
stats = [ ]
for n in names :
    info = [ n ]
    for field in all_queries :
        try :
            info . append ( str ( metrics [ field ] [ n ] ) )
        except KeyError :
            pass
    stats . append ( info )
header = [ 'container id' ] + [ m [ k ] for k in all_queries if k in metrics . keys ( ) ]
return stats , header


signature:
def run_metrics ( command , parser , cl_args , unknown_args ) :
    """run metrics subcommand"""

implementation:
cluster , role , env = cl_args [ 'cluster' ] , cl_args [ 'role' ] , cl_args [ 'environ' ]
topology = cl_args [ 'topology-name' ]
try :
    result = tracker_access . get_topology_info ( cluster , env , topology , role )
    spouts = result [ 'physical_plan' ] [ 'spouts' ] . keys ( )
    bolts = result [ 'physical_plan' ] [ 'bolts' ] . keys ( )
    components = spouts + bolts
    cname = cl_args [ 'component' ]
    if cname :
        if cname in components :
            components = [ cname ]
        else :
            Log . error ( 'Unknown component: \'%s\'' % cname )
            raise
except Exception :
    Log . error ( "Fail to connect to tracker: \'%s\'" , cl_args [ "tracker_url" ] )
    return False
cresult = [ ]
for comp in components :
    try :
        metrics = tracker_access . get_component_metrics ( comp , cluster , env , topology , role )
    except :
        Log . error ( "Fail to connect to tracker: \'%s\'" , cl_args [ "tracker_url" ] )
        return False
    stat , header = to_table ( metrics )
    cresult . append ( ( comp , stat , header ) )
for i , ( comp , stat , header ) in enumerate ( cresult ) :
    if i != 0 :
        print ( '' )
    print ( '\'%s\' metrics:' % comp )
    print ( tabulate ( stat , headers = header ) )
return True


signature:
def run_bolts ( command , parser , cl_args , unknown_args ) :
    """run bolts subcommand"""

implementation:
cluster , role , env = cl_args [ 'cluster' ] , cl_args [ 'role' ] , cl_args [ 'environ' ]
topology = cl_args [ 'topology-name' ]
try :
    result = tracker_access . get_topology_info ( cluster , env , topology , role )
    bolts = result [ 'physical_plan' ] [ 'bolts' ] . keys ( )
    bolt_name = cl_args [ 'bolt' ]
    if bolt_name :
        if bolt_name in bolts :
            bolts = [ bolt_name ]
        else :
            Log . error ( 'Unknown bolt: \'%s\'' % bolt_name )
            raise
except Exception :
    Log . error ( "Fail to connect to tracker: \'%s\'" , cl_args [ "tracker_url" ] )
    return False
bolts_result = [ ]
for bolt in bolts :
    try :
        metrics = tracker_access . get_component_metrics ( bolt , cluster , env , topology , role )
        stat , header = to_table ( metrics )
        bolts_result . append ( ( bolt , stat , header ) )
    except Exception :
        Log . error ( "Fail to connect to tracker: \'%s\'" , cl_args [ "tracker_url" ] )
        return False
for i , ( bolt , stat , header ) in enumerate ( bolts_result ) :
    if i != 0 :
        print ( '' )
    print ( '\'%s\' metrics:' % bolt )
    print ( tabulate ( stat , headers = header ) )
return True


signature:
def run_containers ( command , parser , cl_args , unknown_args ) :
    """run containers subcommand"""

implementation:
cluster , role , env = cl_args [ 'cluster' ] , cl_args [ 'role' ] , cl_args [ 'environ' ]
topology = cl_args [ 'topology-name' ]
container_id = cl_args [ 'id' ]
try :
    result = tracker_access . get_topology_info ( cluster , env , topology , role )
except :
    Log . error ( "Fail to connect to tracker: \'%s\'" , cl_args [ "tracker_url" ] )
    return False
containers = result [ 'physical_plan' ] [ 'stmgrs' ]
all_bolts , all_spouts = set ( ) , set ( )
for _ , bolts in result [ 'physical_plan' ] [ 'bolts' ] . items ( ) :
    all_bolts = all_bolts | set ( bolts )
for _ , spouts in result [ 'physical_plan' ] [ 'spouts' ] . items ( ) :
    all_spouts = all_spouts | set ( spouts )
stmgrs = containers . keys ( )
stmgrs . sort ( )
if container_id is not None :
    try :
        normalized_cid = container_id - 1
        if normalized_cid < 0 :
            raise
        stmgrs = [ stmgrs [ normalized_cid ] ]
    except :
        Log . error ( 'Invalid container id: %d' % container_id )
        return False
table = [ ]
for sid , name in enumerate ( stmgrs ) :
    cid = sid + 1
    host = containers [ name ] [ "host" ]
    port = containers [ name ] [ "port" ]
    pid = containers [ name ] [ "pid" ]
    instances = containers [ name ] [ "instance_ids" ]
    bolt_nums = len ( [ instance for instance in instances if instance in all_bolts ] )
    spout_nums = len ( [ instance for instance in instances if instance in all_spouts ] )
    table . append ( [ cid , host , port , pid , bolt_nums , spout_nums , len ( instances ) ] )
headers = [ "container" , "host" , "port" , "pid" , "#bolt" , "#spout" , "#instance" ]
sys . stdout . flush ( )
print ( tabulate ( table , headers = headers ) )
return True


signature:
def define_options ( address , port , tracker_url , base_url ) :
    ''': param address :
 : param port :
 : param tracker _ url :
 : return :'''

implementation:
define ( "address" , default = address )
define ( "port" , default = port )
define ( "tracker_url" , default = tracker_url )
define ( "base_url" , default = base_url )


signature:
def main ( ) :
    ''': param argv :
 : return :'''

implementation:
log . configure ( logging . DEBUG )
tornado . log . enable_pretty_logging ( )
# create the parser and parse the arguments
( parser , child_parser ) = args . create_parsers ( )
( parsed_args , remaining ) = parser . parse_known_args ( )
if remaining :
    r = child_parser . parse_args ( args = remaining , namespace = parsed_args )
    namespace = vars ( r )
    if 'version' in namespace :
        common_config . print_build_info ( zipped_pex = True )
    else :
        parser . print_help ( )
    parser . exit ( )
# log additional information
command_line_args = vars ( parsed_args )
Log . info ( "Listening at http://%s:%d%s" , command_line_args [ 'address' ] , command_line_args [ 'port' ] , command_line_args [ 'base_url' ] )
Log . info ( "Using tracker url: %s" , command_line_args [ 'tracker_url' ] )
# pass the options to tornado and start the ui server
define_options ( command_line_args [ 'address' ] , command_line_args [ 'port' ] , command_line_args [ 'tracker_url' ] , command_line_args [ 'base_url' ] )
http_server = tornado . httpserver . HTTPServer ( Application ( command_line_args [ 'base_url' ] ) )
http_server . listen ( command_line_args [ 'port' ] , address = command_line_args [ 'address' ] )
# pylint : disable = unused - argument
# stop Tornado IO loop
def signal_handler ( signum , frame ) : # start a new line after ^ C character because this looks nice
    print ( '\n' , end = '' )
    Log . debug ( 'SIGINT received. Stopping UI' )
    tornado . ioloop . IOLoop . instance ( ) . stop ( )
# associate SIGINT and SIGTERM with a handler
signal . signal ( signal . SIGINT , signal_handler )
signal . signal ( signal . SIGTERM , signal_handler )
# start Tornado IO loop
tornado . ioloop . IOLoop . instance ( ) . start ( )


signature:
def spec ( cls , name = None , inputs = None , par = 1 , config = None , optional_outputs = None ) :
    """Register this bolt to the topology and create ` ` HeronComponentSpec ` `
 This method takes an optional ` ` outputs ` ` argument for supporting dynamic output fields
 declaration . However , it is recommended that ` ` outputs ` ` should be declared as
 an attribute of your ` ` Bolt ` ` subclass . Also , some ways of declaring inputs is not supported
 in this implementation ; please read the documentation below .
 : type name : str
 : param name : Name of this bolt .
 : type inputs : dict or list
 : param inputs : Streams that feed into this Bolt .
 Two forms of this are acceptable :
 1 . A ` dict ` mapping from ` ` HeronComponentSpec ` ` to ` ` Grouping ` ` .
 In this case , default stream is used .
 2 . A ` dict ` mapping from ` ` GlobalStreamId ` ` to ` ` Grouping ` ` .
 This ` ` GlobalStreamId ` ` object itself is different from StreamParse , because
 Heron does not use thrift , although its constructor method is compatible .
 3 . A ` list ` of ` ` HeronComponentSpec ` ` . In this case , default stream with
 SHUFFLE grouping is used .
 4 . A ` list ` of ` ` GlobalStreamId ` ` . In this case , SHUFFLE grouping is used .
 : type par : int
 : param par : Parallelism hint for this spout .
 : type config : dict
 : param config : Component - specific config settings .
 : type optional _ outputs : list of ( str or Stream ) or tuple of ( str or Stream )
 : param optional _ outputs : Additional output fields for this bolt . These fields are added to
 existing ` ` outputs ` ` class attributes of your bolt . This is an optional
 argument , and exists only for supporting dynamic output field
 declaration ."""

implementation:
python_class_path = "%s.%s" % ( cls . __module__ , cls . __name__ )
if hasattr ( cls , 'outputs' ) : # avoid modification to cls . outputs
    _outputs = copy . copy ( cls . outputs )
else :
    _outputs = [ ]
if optional_outputs is not None :
    assert isinstance ( optional_outputs , ( list , tuple ) )
    for out in optional_outputs :
        assert isinstance ( out , ( str , Stream ) )
        _outputs . append ( out )
return HeronComponentSpec ( name , python_class_path , is_spout = False , par = par , inputs = inputs , outputs = _outputs , config = config )


signature:
def emit ( self , tup , stream = Stream . DEFAULT_STREAM_ID , anchors = None , direct_task = None , need_task_ids = False ) :
    """Emits a new tuple from this Bolt
 It is compatible with StreamParse API .
 : type tup : list or tuple
 : param tup : the new output Tuple to send from this bolt ,
 which should contain only serializable data .
 : type stream : str
 : param stream : the ID of the stream to emit this Tuple to .
 Leave empty to emit to the default stream .
 : type anchors : list
 : param anchors : a list of HeronTuples to which the emitted Tuples should be anchored .
 : type direct _ task : int
 : param direct _ task : the task to send the Tuple to if performing a direct emit .
 : type need _ task _ ids : bool
 : param need _ task _ ids : indicate whether or not you would like the task IDs the Tuple was emitted ."""

implementation: self . delegate . emit ( tup , stream , anchors , direct_task , need_task_ids )


signature:
def create_parser ( subparsers ) :
    """create argument parser"""

implementation:
parser = subparsers . add_parser ( 'clusters' , help = 'Display existing clusters' , usage = "%(prog)s [options]" , add_help = True )
args . add_verbose ( parser )
args . add_tracker_url ( parser )
parser . set_defaults ( subcommand = 'clusters' )
return subparsers


signature:
def run ( command , parser , cl_args , unknown_args ) :
    """run command"""

implementation:
try :
    clusters = tracker_access . get_clusters ( )
except :
    Log . error ( "Fail to connect to tracker: \'%s\'" , cl_args [ "tracker_url" ] )
    return False
print ( 'Available clusters:' )
for cluster in clusters :
    print ( '  %s' % cluster )
return True


signature:
def get_time_ranges ( ranges ) :
    ''': param ranges :
 : return :'''

implementation:
# get the current time
now = int ( time . time ( ) )
# form the new
time_slots = dict ( )
for key , value in ranges . items ( ) :
    time_slots [ key ] = ( now - value [ 0 ] , now - value [ 1 ] , value [ 2 ] )
return ( now , time_slots )


signature:
def add_arguments ( parser ) :
    """add arguments"""

implementation:
default_config_file = os . path . join ( utils . get_heron_tracker_conf_dir ( ) , constants . DEFAULT_CONFIG_FILE )
parser . add_argument ( '--config-file' , metavar = '(a string; path to config file; default: "' + default_config_file + '")' , default = default_config_file )
parser . add_argument ( '--type' , metavar = '(an string; type of state manager (zookeeper or file, etc.); example: ' + str ( constants . DEFAULT_STATE_MANAGER_TYPE ) + ')' , choices = [ "file" , "zookeeper" ] )
parser . add_argument ( '--name' , metavar = '(an string; name to be used for the state manager; example: ' + str ( constants . DEFAULT_STATE_MANAGER_NAME ) + ')' )
parser . add_argument ( '--rootpath' , metavar = '(an string; where all the states are stored; example: ' + str ( constants . DEFAULT_STATE_MANAGER_ROOTPATH ) + ')' )
parser . add_argument ( '--tunnelhost' , metavar = '(an string; if ssh tunneling needs to be established to connect to it; example: ' + str ( constants . DEFAULT_STATE_MANAGER_TUNNELHOST ) + ')' )
parser . add_argument ( '--hostport' , metavar = '(an string; only used to connect to zk, must be of the form \'host:port\';' ' example: ' + str ( constants . DEFAULT_STATE_MANAGER_HOSTPORT ) + ')' )
parser . add_argument ( '--port' , metavar = '(an integer; port to listen; default: ' + str ( constants . DEFAULT_PORT ) + ')' , type = int , default = constants . DEFAULT_PORT )
parser . add_argument ( '--verbose' , action = 'store_true' )
return parser


signature:
def create_parsers ( ) :
    """create argument parser"""

implementation:
parser = argparse . ArgumentParser ( epilog = 'For detailed documentation, go to http://github.com/apache/incubator-heron' , usage = "%(prog)s [options] [help]" , add_help = False )
parser = add_titles ( parser )
parser = add_arguments ( parser )
ya_parser = argparse . ArgumentParser ( parents = [ parser ] , formatter_class = SubcommandHelpFormatter , add_help = False )
subparsers = ya_parser . add_subparsers ( title = "Available commands" )
help_parser = subparsers . add_parser ( 'help' , help = 'Prints help' , add_help = False )
help_parser . set_defaults ( help = True )
subparsers . add_parser ( 'version' , help = 'Prints version' , add_help = True )
return parser , ya_parser


signature:
def main ( ) :
    """main"""

implementation:
# create the parser and parse the arguments
( parser , _ ) = create_parsers ( )
( args , remaining ) = parser . parse_known_args ( )
if remaining == [ 'help' ] :
    parser . print_help ( )
    parser . exit ( )
elif remaining == [ 'version' ] :
    common_config . print_build_info ( )
    parser . exit ( )
elif remaining != [ ] :
    Log . error ( 'Invalid subcommand' )
    sys . exit ( 1 )
namespace = vars ( args )
log . set_logging_level ( namespace )
# set Tornado global option
define_options ( namespace [ 'port' ] , namespace [ 'config_file' ] )
config = Config ( create_tracker_config ( namespace ) )
# create Tornado application
application = Application ( config )
# pylint : disable = unused - argument
# SIGINT handler :
# 1 . stop all the running zkstatemanager and filestatemanagers
# 2 . stop the Tornado IO loop
def signal_handler ( signum , frame ) : # start a new line after ^ C character because this looks nice
    print ( '\n' , end = '' )
    application . stop ( )
    tornado . ioloop . IOLoop . instance ( ) . stop ( )
# associate SIGINT and SIGTERM with a handler
signal . signal ( signal . SIGINT , signal_handler )
signal . signal ( signal . SIGTERM , signal_handler )
Log . info ( "Running on port: %d" , namespace [ 'port' ] )
if namespace [ "config_file" ] :
    Log . info ( "Using config file: %s" , namespace [ 'config_file' ] )
Log . info ( "Using state manager:\n" + str ( config ) )
http_server = tornado . httpserver . HTTPServer ( application )
http_server . listen ( namespace [ 'port' ] )
tornado . ioloop . IOLoop . instance ( ) . start ( )


signature:
def make_tuple ( stream , tuple_key , values , roots = None ) :
    """Creates a HeronTuple
 : param stream : protobuf message ` ` StreamId ` `
 : param tuple _ key : tuple id
 : param values : a list of values
 : param roots : a list of protobuf message ` ` RootId ` `"""

implementation:
component_name = stream . component_name
stream_id = stream . id
gen_task = roots [ 0 ] . taskid if roots is not None and len ( roots ) > 0 else None
return HeronTuple ( id = str ( tuple_key ) , component = component_name , stream = stream_id , task = gen_task , values = values , creation_time = time . time ( ) , roots = roots )


signature:
def make_tick_tuple ( ) :
    """Creates a TickTuple"""

implementation: return HeronTuple ( id = TupleHelper . TICK_TUPLE_ID , component = TupleHelper . TICK_SOURCE_COMPONENT , stream = TupleHelper . TICK_TUPLE_ID , task = None , values = None , creation_time = time . time ( ) , roots = None )


signature:
def make_root_tuple_info ( stream_id , tuple_id ) :
    """Creates a RootTupleInfo"""

implementation:
key = random . getrandbits ( TupleHelper . MAX_SFIXED64_RAND_BITS )
return RootTupleInfo ( stream_id = stream_id , tuple_id = tuple_id , insertion_time = time . time ( ) , key = key )


signature:
def fetch_backpressure ( self , cluster , metric , topology , component , instance , timerange , is_max , environ = None ) :
    ''': param cluster :
 : param metric :
 : param topology :
 : param component :
 : param instance :
 : param timerange :
 : param is _ max :
 : param environ :
 : return :'''

implementation: pass


signature:
def ParseNolintSuppressions ( filename , raw_line , linenum , error ) :
    """Updates the global list of line error - suppressions .
 Parses any NOLINT comments on the current line , updating the global
 error _ suppressions store . Reports an error if the NOLINT comment
 was malformed .
 Args :
 filename : str , the name of the input file .
 raw _ line : str , the line of input text , with comments .
 linenum : int , the number of the current line .
 error : function , an error handler ."""

implementation:
matched = Search ( r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?' , raw_line )
if matched :
    if matched . group ( 1 ) :
        suppressed_line = linenum + 1
    else :
        suppressed_line = linenum
    category = matched . group ( 2 )
    if category in ( None , '(*)' ) : # = > " suppress all "
        _error_suppressions . setdefault ( None , set ( ) ) . add ( suppressed_line )
    else :
        if category . startswith ( '(' ) and category . endswith ( ')' ) :
            category = category [ 1 : - 1 ]
            if category in _ERROR_CATEGORIES :
                _error_suppressions . setdefault ( category , set ( ) ) . add ( suppressed_line )
            elif category not in _LEGACY_ERROR_CATEGORIES :
                error ( filename , linenum , 'readability/nolint' , 5 , 'Unknown NOLINT error category: %s' % category )


signature:
def ProcessGlobalSuppresions ( lines ) :
    """Updates the list of global error suppressions .
 Parses any lint directives in the file that have global effect .
 Args :
 lines : An array of strings , each representing a line of the file , with the
 last element being empty if the file is terminated with a newline ."""

implementation:
for line in lines :
    if _SEARCH_C_FILE . search ( line ) :
        for category in _DEFAULT_C_SUPPRESSED_CATEGORIES :
            _global_error_suppressions [ category ] = True
    if _SEARCH_KERNEL_FILE . search ( line ) :
        for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES :
            _global_error_suppressions [ category ] = True


signature:
def IsErrorSuppressedByNolint ( category , linenum ) :
    """Returns true if the specified error category is suppressed on this line .
 Consults the global error _ suppressions map populated by
 ParseNolintSuppressions / ProcessGlobalSuppresions / ResetNolintSuppressions .
 Args :
 category : str , the category of the error .
 linenum : int , the current line number .
 Returns :
 bool , True iff the error should be suppressed due to a NOLINT comment or
 global suppression ."""

implementation: return ( _global_error_suppressions . get ( category , False ) or linenum in _error_suppressions . get ( category , set ( ) ) or linenum in _error_suppressions . get ( None , set ( ) ) )


signature:
def Match ( pattern , s ) :
    """Matches the string with the pattern , caching the compiled regexp ."""

implementation:
# The regexp compilation caching is inlined in both Match and Search for
# performance reasons ; factoring it out into a separate function turns out
# to be noticeably expensive .
if pattern not in _regexp_compile_cache :
    _regexp_compile_cache [ pattern ] = sre_compile . compile ( pattern )
return _regexp_compile_cache [ pattern ] . match ( s )


signature:
def ReplaceAll ( pattern , rep , s ) :
    """Replaces instances of pattern in a string with a replacement .
 The compiled regex is kept in a cache shared by Match and Search .
 Args :
 pattern : regex pattern
 rep : replacement text
 s : search string
 Returns :
 string with replacements made ( or original string if no replacements )"""

implementation:
if pattern not in _regexp_compile_cache :
    _regexp_compile_cache [ pattern ] = sre_compile . compile ( pattern )
return _regexp_compile_cache [ pattern ] . sub ( rep , s )


signature:
def Search ( pattern , s ) :
    """Searches the string for the pattern , caching the compiled regexp ."""

implementation:
if pattern not in _regexp_compile_cache :
    _regexp_compile_cache [ pattern ] = sre_compile . compile ( pattern )
return _regexp_compile_cache [ pattern ] . search ( s )


signature:
def _ShouldPrintError ( category , confidence , linenum ) :
    """If confidence > = verbose , category passes filter and is not suppressed ."""

implementation:
# There are three ways we might decide not to print an error message :
# a " NOLINT ( category ) " comment appears in the source ,
# the verbosity level isn ' t high enough , or the filters filter it out .
if IsErrorSuppressedByNolint ( category , linenum ) :
    return False
if confidence < _cpplint_state . verbose_level :
    return False
is_filtered = False
for one_filter in _Filters ( ) :
    if one_filter . startswith ( '-' ) :
        if category . startswith ( one_filter [ 1 : ] ) :
            is_filtered = True
    elif one_filter . startswith ( '+' ) :
        if category . startswith ( one_filter [ 1 : ] ) :
            is_filtered = False
    else :
        assert False
        # should have been checked for in SetFilter .
if is_filtered :
    return False
return True


signature:
def IsCppString ( line ) :
    """Does line terminate so , that the next symbol is in string constant .
 This function does not consider single - line nor multi - line comments .
 Args :
 line : is a partial line of code starting from the 0 . . n .
 Returns :
 True , if next character appended to ' line ' is inside a
 string constant ."""

implementation:
line = line . replace ( r'\\' , 'XX' )
# after this , \ \ " does not match to \ "
return ( ( line . count ( '"' ) - line . count ( r'\"' ) - line . count ( "'\"'" ) ) & 1 ) == 1


signature:
def CleanseRawStrings ( raw_lines ) :
    """Removes C + + 11 raw strings from lines .
 Before :
 static const char kData [ ] = R " (
 multi - line string
 After :
 static const char kData [ ] = " "
 ( replaced by blank line )
 Args :
 raw _ lines : list of raw lines .
 Returns :
 list of lines with C + + 11 raw strings replaced by empty strings ."""

implementation:
delimiter = None
lines_without_raw_strings = [ ]
for line in raw_lines :
    if delimiter : # Inside a raw string , look for the end
        end = line . find ( delimiter )
        if end >= 0 : # Found the end of the string , match leading space for this
        # line and resume copying the original lines , and also insert
        # a " " on the last line .
            leading_space = Match ( r'^(\s*)\S' , line )
            line = leading_space . group ( 1 ) + '""' + line [ end + len ( delimiter ) : ]
            delimiter = None
        else : # Haven ' t found the end yet , append a blank line .
            line = '""'
    # Look for beginning of a raw string , and replace them with
    # empty strings . This is done in a loop to handle multiple raw
    # strings on the same line .
    while delimiter is None : # Look for beginning of a raw string .
    # See 2.14.15 [ lex . string ] for syntax .
    # Once we have matched a raw string , we check the prefix of the
    # line to make sure that the line is not part of a single line
    # comment . It ' s done this way because we remove raw strings
    # before removing comments as opposed to removing comments
    # before removing raw strings . This is because there are some
    # cpplint checks that requires the comments to be preserved , but
    # we don ' t want to check comments that are inside raw strings .
        matched = Match ( r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$' , line )
        if ( matched and not Match ( r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//' , matched . group ( 1 ) ) ) :
            delimiter = ')' + matched . group ( 2 ) + '"'
            end = matched . group ( 3 ) . find ( delimiter )
            if end >= 0 : # Raw string ended on same line
                line = ( matched . group ( 1 ) + '""' + matched . group ( 3 ) [ end + len ( delimiter ) : ] )
                delimiter = None
            else : # Start of a multi - line raw string
                line = matched . group ( 1 ) + '""'
        else :
            break
    lines_without_raw_strings . append ( line )
# TODO ( unknown ) : if delimiter is not None here , we might want to
# emit a warning for unterminated string .
return lines_without_raw_strings


signature:
def FindNextMultiLineCommentStart ( lines , lineix ) :
    """Find the beginning marker for a multiline comment ."""

implementation:
while lineix < len ( lines ) :
    if lines [ lineix ] . strip ( ) . startswith ( '/*' ) : # Only return this marker if the comment goes beyond this line
        if lines [ lineix ] . strip ( ) . find ( '*/' , 2 ) < 0 :
            return lineix
    lineix += 1
return len ( lines )


signature:
def FindNextMultiLineCommentEnd ( lines , lineix ) :
    """We are inside a comment , find the end marker ."""

implementation:
while lineix < len ( lines ) :
    if lines [ lineix ] . strip ( ) . endswith ( '*/' ) :
        return lineix
    lineix += 1
return len ( lines )


signature:
def RemoveMultiLineCommentsFromRange ( lines , begin , end ) :
    """Clears a range of lines for multi - line comments ."""

implementation:
# Having / / dummy comments makes the lines non - empty , so we will not get
# unnecessary blank line warnings later in the code .
for i in range ( begin , end ) :
    lines [ i ] = '/**/'


signature:
def RemoveMultiLineComments ( filename , lines , error ) :
    """Removes multiline ( c - style ) comments from lines ."""

implementation:
lineix = 0
while lineix < len ( lines ) :
    lineix_begin = FindNextMultiLineCommentStart ( lines , lineix )
    if lineix_begin >= len ( lines ) :
        return
    lineix_end = FindNextMultiLineCommentEnd ( lines , lineix_begin )
    if lineix_end >= len ( lines ) :
        error ( filename , lineix_begin + 1 , 'readability/multiline_comment' , 5 , 'Could not find end of multi-line comment' )
        return
    RemoveMultiLineCommentsFromRange ( lines , lineix_begin , lineix_end + 1 )
    lineix = lineix_end + 1


signature:
def CleanseComments ( line ) :
    """Removes / / - comments and single - line C - style / * * / comments .
 Args :
 line : A line of C + + source .
 Returns :
 The line with single - line comments removed ."""

implementation:
commentpos = line . find ( '//' )
if commentpos != - 1 and not IsCppString ( line [ : commentpos ] ) :
    line = line [ : commentpos ] . rstrip ( )
# get rid of / * . . . * /
return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS . sub ( '' , line )


signature:
def FindEndOfExpressionInLine ( line , startpos , stack ) :
    """Find the position just after the end of current parenthesized expression .
 Args :
 line : a CleansedLines line .
 startpos : start searching at this position .
 stack : nesting stack at startpos .
 Returns :
 On finding matching end : ( index just after matching end , None )
 On finding an unclosed expression : ( - 1 , None )
 Otherwise : ( - 1 , new stack at end of this line )"""

implementation:
for i in xrange ( startpos , len ( line ) ) :
    char = line [ i ]
    if char in '([{' : # Found start of parenthesized expression , push to expression stack
        stack . append ( char )
    elif char == '<' : # Found potential start of template argument list
        if i > 0 and line [ i - 1 ] == '<' : # Left shift operator
            if stack and stack [ - 1 ] == '<' :
                stack . pop ( )
                if not stack :
                    return ( - 1 , None )
        elif i > 0 and Search ( r'\boperator\s*$' , line [ 0 : i ] ) : # operator < , don ' t add to stack
            continue
        else : # Tentative start of template argument list
            stack . append ( '<' )
    elif char in ')]}' : # Found end of parenthesized expression .
    # If we are currently expecting a matching ' > ' , the pending ' < '
    # must have been an operator . Remove them from expression stack .
        while stack and stack [ - 1 ] == '<' :
            stack . pop ( )
        if not stack :
            return ( - 1 , None )
        if ( ( stack [ - 1 ] == '(' and char == ')' ) or ( stack [ - 1 ] == '[' and char == ']' ) or ( stack [ - 1 ] == '{' and char == '}' ) ) :
            stack . pop ( )
            if not stack :
                return ( i + 1 , None )
        else : # Mismatched parentheses
            return ( - 1 , None )
    elif char == '>' : # Found potential end of template argument list .
    # Ignore " - > " and operator functions
        if ( i > 0 and ( line [ i - 1 ] == '-' or Search ( r'\boperator\s*$' , line [ 0 : i - 1 ] ) ) ) :
            continue
        # Pop the stack if there is a matching ' < ' . Otherwise , ignore
        # this ' > ' since it must be an operator .
        if stack :
            if stack [ - 1 ] == '<' :
                stack . pop ( )
                if not stack :
                    return ( i + 1 , None )
    elif char == ';' : # Found something that look like end of statements . If we are currently
    # expecting a ' > ' , the matching ' < ' must have been an operator , since
    # template argument list should not contain statements .
        while stack and stack [ - 1 ] == '<' :
            stack . pop ( )
        if not stack :
            return ( - 1 , None )
# Did not find end of expression or unbalanced parentheses on this line
return ( - 1 , stack )


signature:
def CloseExpression ( clean_lines , linenum , pos ) :
    """If input points to ( or { or [ or < , finds the position that closes it .
 If lines [ linenum ] [ pos ] points to a ' ( ' or ' { ' or ' [ ' or ' < ' , finds the
 linenum / pos that correspond to the closing of the expression .
 TODO ( unknown ) : cpplint spends a fair bit of time matching parentheses .
 Ideally we would want to index all opening and closing parentheses once
 and have CloseExpression be just a simple lookup , but due to preprocessor
 tricks , this is not so easy .
 Args :
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 pos : A position on the line .
 Returns :
 A tuple ( line , linenum , pos ) pointer * past * the closing brace , or
 ( line , len ( lines ) , - 1 ) if we never find a close . Note we ignore
 strings and comments when matching ; and the line we return is the
 ' cleansed ' line at linenum ."""

implementation:
line = clean_lines . elided [ linenum ]
if ( line [ pos ] not in '({[<' ) or Match ( r'<[<=]' , line [ pos : ] ) :
    return ( line , clean_lines . NumLines ( ) , - 1 )
# Check first line
( end_pos , stack ) = FindEndOfExpressionInLine ( line , pos , [ ] )
if end_pos > - 1 :
    return ( line , linenum , end_pos )
# Continue scanning forward
while stack and linenum < clean_lines . NumLines ( ) - 1 :
    linenum += 1
    line = clean_lines . elided [ linenum ]
    ( end_pos , stack ) = FindEndOfExpressionInLine ( line , 0 , stack )
    if end_pos > - 1 :
        return ( line , linenum , end_pos )
# Did not find end of expression before end of file , give up
return ( line , clean_lines . NumLines ( ) , - 1 )


signature:
def FindStartOfExpressionInLine ( line , endpos , stack ) :
    """Find position at the matching start of current expression .
 This is almost the reverse of FindEndOfExpressionInLine , but note
 that the input position and returned position differs by 1.
 Args :
 line : a CleansedLines line .
 endpos : start searching at this position .
 stack : nesting stack at endpos .
 Returns :
 On finding matching start : ( index at matching start , None )
 On finding an unclosed expression : ( - 1 , None )
 Otherwise : ( - 1 , new stack at beginning of this line )"""

implementation:
i = endpos
while i >= 0 :
    char = line [ i ]
    if char in ')]}' : # Found end of expression , push to expression stack
        stack . append ( char )
    elif char == '>' : # Found potential end of template argument list .
    # Ignore it if it ' s a " - > " or " > = " or " operator > "
        if ( i > 0 and ( line [ i - 1 ] == '-' or Match ( r'\s>=\s' , line [ i - 1 : ] ) or Search ( r'\boperator\s*$' , line [ 0 : i ] ) ) ) :
            i -= 1
        else :
            stack . append ( '>' )
    elif char == '<' : # Found potential start of template argument list
        if i > 0 and line [ i - 1 ] == '<' : # Left shift operator
            i -= 1
        else : # If there is a matching ' > ' , we can pop the expression stack .
        # Otherwise , ignore this ' < ' since it must be an operator .
            if stack and stack [ - 1 ] == '>' :
                stack . pop ( )
                if not stack :
                    return ( i , None )
    elif char in '([{' : # Found start of expression .
    # If there are any unmatched ' > ' on the stack , they must be
    # operators . Remove those .
        while stack and stack [ - 1 ] == '>' :
            stack . pop ( )
        if not stack :
            return ( - 1 , None )
        if ( ( char == '(' and stack [ - 1 ] == ')' ) or ( char == '[' and stack [ - 1 ] == ']' ) or ( char == '{' and stack [ - 1 ] == '}' ) ) :
            stack . pop ( )
            if not stack :
                return ( i , None )
        else : # Mismatched parentheses
            return ( - 1 , None )
    elif char == ';' : # Found something that look like end of statements . If we are currently
    # expecting a ' < ' , the matching ' > ' must have been an operator , since
    # template argument list should not contain statements .
        while stack and stack [ - 1 ] == '>' :
            stack . pop ( )
        if not stack :
            return ( - 1 , None )
    i -= 1
return ( - 1 , stack )


signature:
def ReverseCloseExpression ( clean_lines , linenum , pos ) :
    """If input points to ) or } or ] or > , finds the position that opens it .
 If lines [ linenum ] [ pos ] points to a ' ) ' or ' } ' or ' ] ' or ' > ' , finds the
 linenum / pos that correspond to the opening of the expression .
 Args :
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 pos : A position on the line .
 Returns :
 A tuple ( line , linenum , pos ) pointer * at * the opening brace , or
 ( line , 0 , - 1 ) if we never find the matching opening brace . Note
 we ignore strings and comments when matching ; and the line we
 return is the ' cleansed ' line at linenum ."""

implementation:
line = clean_lines . elided [ linenum ]
if line [ pos ] not in ')}]>' :
    return ( line , 0 , - 1 )
# Check last line
( start_pos , stack ) = FindStartOfExpressionInLine ( line , pos , [ ] )
if start_pos > - 1 :
    return ( line , linenum , start_pos )
# Continue scanning backward
while stack and linenum > 0 :
    linenum -= 1
    line = clean_lines . elided [ linenum ]
    ( start_pos , stack ) = FindStartOfExpressionInLine ( line , len ( line ) - 1 , stack )
    if start_pos > - 1 :
        return ( line , linenum , start_pos )
# Did not find start of expression before beginning of file , give up
return ( line , 0 , - 1 )


signature:
def CheckForCopyright ( filename , lines , error ) :
    """Logs an error if no Copyright message appears at the top of the file ."""

implementation:
# We ' ll say it should occur by line 10 . Don ' t forget there ' s a
# dummy line at the front .
for line in range ( 1 , min ( len ( lines ) , 11 ) ) :
    if re . search ( r'Copyright' , lines [ line ] , re . I ) :
        break
else : # means no copyright line was found
    error ( filename , 0 , 'legal/copyright' , 5 , 'No copyright message found.  ' 'You should have a line: "Copyright [year] <Copyright Owner>"' )


signature:
def GetIndentLevel ( line ) :
    """Return the number of leading spaces in line .
 Args :
 line : A string to check .
 Returns :
 An integer count of leading spaces , possibly zero ."""

implementation:
indent = Match ( r'^( *)\S' , line )
if indent :
    return len ( indent . group ( 1 ) )
else :
    return 0


signature:
def GetHeaderGuardCPPVariable ( filename ) :
    """Returns the CPP variable that should be used as a header guard .
 Args :
 filename : The name of a C + + header file .
 Returns :
 The CPP variable that should be used as a header guard in the
 named file ."""

implementation:
# Restores original filename in case that cpplint is invoked from Emacs ' s
# flymake .
filename = re . sub ( r'_flymake\.h$' , '.h' , filename )
filename = re . sub ( r'/\.flymake/([^/]*)$' , r'/\1' , filename )
# Replace ' c + + ' with ' cpp ' .
filename = filename . replace ( 'C++' , 'cpp' ) . replace ( 'c++' , 'cpp' )
fileinfo = FileInfo ( filename )
file_path_from_root = fileinfo . RepositoryName ( )
if _root :
    suffix = os . sep
    # On Windows using directory separator will leave us with
    # " bogus escape error " unless we properly escape regex .
    if suffix == '\\' :
        suffix += '\\'
    file_path_from_root = re . sub ( '^' + _root + suffix , '' , file_path_from_root )
return re . sub ( r'[^a-zA-Z0-9]' , '_' , file_path_from_root ) . upper ( ) + '_'


signature:
def CheckForHeaderGuard ( filename , clean_lines , error ) :
    """Checks that the file contains a header guard .
 Logs an error if no # ifndef header guard is present . For other
 headers , checks that the full pathname is used .
 Args :
 filename : The name of the C + + header file .
 clean _ lines : A CleansedLines instance containing the file .
 error : The function to call with any errors found ."""

implementation:
# Don ' t check for header guards if there are error suppression
# comments somewhere in this file .
# Because this is silencing a warning for a nonexistent line , we
# only support the very specific NOLINT ( build / header _ guard ) syntax ,
# and not the general NOLINT or NOLINT ( * ) syntax .
raw_lines = clean_lines . lines_without_raw_strings
for i in raw_lines :
    if Search ( r'//\s*NOLINT\(build/header_guard\)' , i ) :
        return
# Allow pragma once instead of header guards
for i in raw_lines :
    if Search ( r'^\s*#pragma\s+once' , i ) :
        return
cppvar = GetHeaderGuardCPPVariable ( filename )
ifndef = ''
ifndef_linenum = 0
define = ''
endif = ''
endif_linenum = 0
for linenum , line in enumerate ( raw_lines ) :
    linesplit = line . split ( )
    if len ( linesplit ) >= 2 : # find the first occurrence of # ifndef and # define , save arg
        if not ifndef and linesplit [ 0 ] == '#ifndef' : # set ifndef to the header guard presented on the # ifndef line .
            ifndef = linesplit [ 1 ]
            ifndef_linenum = linenum
        if not define and linesplit [ 0 ] == '#define' :
            define = linesplit [ 1 ]
    # find the last occurrence of # endif , save entire line
    if line . startswith ( '#endif' ) :
        endif = line
        endif_linenum = linenum
if not ifndef or not define or ifndef != define :
    error ( filename , 0 , 'build/header_guard' , 5 , 'No #ifndef header guard found, suggested CPP variable is: %s' % cppvar )
    return
# The guard should be PATH _ FILE _ H _ , but we also allow PATH _ FILE _ H _ _
# for backward compatibility .
if ifndef != cppvar :
    error_level = 0
    if ifndef != cppvar + '_' :
        error_level = 5
    ParseNolintSuppressions ( filename , raw_lines [ ifndef_linenum ] , ifndef_linenum , error )
    error ( filename , ifndef_linenum , 'build/header_guard' , error_level , '#ifndef header guard has wrong style, please use: %s' % cppvar )
# Check for " / / " comments on endif line .
ParseNolintSuppressions ( filename , raw_lines [ endif_linenum ] , endif_linenum , error )
match = Match ( r'#endif\s*//\s*' + cppvar + r'(_)?\b' , endif )
if match :
    if match . group ( 1 ) == '_' : # Issue low severity warning for deprecated double trailing underscore
        error ( filename , endif_linenum , 'build/header_guard' , 0 , '#endif line should be "#endif  // %s"' % cppvar )
    return
# Didn ' t find the corresponding " / / " comment . If this file does not
# contain any " / / " comments at all , it could be that the compiler
# only wants " / * * / " comments , look for those instead .
no_single_line_comments = True
for i in xrange ( 1 , len ( raw_lines ) - 1 ) :
    line = raw_lines [ i ]
    if Match ( r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//' , line ) :
        no_single_line_comments = False
        break
if no_single_line_comments :
    match = Match ( r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/' , endif )
    if match :
        if match . group ( 1 ) == '_' : # Low severity warning for double trailing underscore
            error ( filename , endif_linenum , 'build/header_guard' , 0 , '#endif line should be "#endif  /* %s */"' % cppvar )
        return
# Didn ' t find anything
error ( filename , endif_linenum , 'build/header_guard' , 5 , '#endif line should be "#endif  // %s"' % cppvar )


signature:
def CheckHeaderFileIncluded ( filename , include_state , error ) :
    """Logs an error if a source file does not include its header ."""

implementation:
# Do not check test files
fileinfo = FileInfo ( filename )
if Search ( _TEST_FILE_SUFFIX , fileinfo . BaseName ( ) ) :
    return
for ext in GetHeaderExtensions ( ) :
    basefilename = filename [ 0 : len ( filename ) - len ( fileinfo . Extension ( ) ) ]
    headerfile = basefilename + '.' + ext
    if not os . path . exists ( headerfile ) :
        continue
    headername = FileInfo ( headerfile ) . RepositoryName ( )
    first_include = None
    for section_list in include_state . include_list :
        for f in section_list :
            if headername in f [ 0 ] or f [ 0 ] in headername :
                return
            if not first_include :
                first_include = f [ 1 ]
    error ( filename , first_include , 'build/include' , 5 , '%s should include its header file %s' % ( fileinfo . RepositoryName ( ) , headername ) )


signature:
def CheckForBadCharacters ( filename , lines , error ) :
    """Logs an error for each line containing bad characters .
 Two kinds of bad characters :
 1 . Unicode replacement characters : These indicate that either the file
 contained invalid UTF - 8 ( likely ) or Unicode replacement characters ( which
 it shouldn ' t ) . Note that it ' s possible for this to throw off line
 numbering if the invalid UTF - 8 occurred adjacent to a newline .
 2 . NUL bytes . These are problematic for some tools .
 Args :
 filename : The name of the current file .
 lines : An array of strings , each representing a line of the file .
 error : The function to call with any errors found ."""

implementation:
for linenum , line in enumerate ( lines ) :
    if unicode_escape_decode ( '\ufffd' ) in line :
        error ( filename , linenum , 'readability/utf8' , 5 , 'Line contains invalid UTF-8 (or Unicode replacement character).' )
    if '\0' in line :
        error ( filename , linenum , 'readability/nul' , 5 , 'Line contains NUL byte.' )


signature:
def CheckForNewlineAtEOF ( filename , lines , error ) :
    """Logs an error if there is no newline char at the end of the file .
 Args :
 filename : The name of the current file .
 lines : An array of strings , each representing a line of the file .
 error : The function to call with any errors found ."""

implementation:
# The array lines ( ) was created by adding two newlines to the
# original file ( go figure ) , then splitting on \ n .
# To verify that the file ends in \ n , we just have to make sure the
# last - but - two element of lines ( ) exists and is empty .
if len ( lines ) < 3 or lines [ - 2 ] :
    error ( filename , len ( lines ) - 2 , 'whitespace/ending_newline' , 5 , 'Could not find a newline character at the end of the file.' )


signature:
def CheckForMultilineCommentsAndStrings ( filename , clean_lines , linenum , error ) :
    """Logs an error if we see / * . . . * / or " . . . " that extend past one line .
 / * . . . * / comments are legit inside macros , for one line .
 Otherwise , we prefer / / comments , so it ' s ok to warn about the
 other . Likewise , it ' s ok for strings to extend across multiple
 lines , as long as a line continuation character ( backslash )
 terminates each line . Although not currently prohibited by the C + +
 style guide , it ' s ugly and unnecessary . We don ' t do well with either
 in this lint program , so we warn about both .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 error : The function to call with any errors found ."""

implementation:
line = clean_lines . elided [ linenum ]
# Remove all \ \ ( escaped backslashes ) from the line . They are OK , and the
# second ( escaped ) slash may trigger later \ " detection erroneously .
line = line . replace ( '\\\\' , '' )
if line . count ( '/*' ) > line . count ( '*/' ) :
    error ( filename , linenum , 'readability/multiline_comment' , 5 , 'Complex multi-line /*...*/-style comment found. ' 'Lint may give bogus warnings.  ' 'Consider replacing these with //-style comments, ' 'with #if 0...#endif, ' 'or with more clearly structured multi-line comments.' )
if ( line . count ( '"' ) - line . count ( '\\"' ) ) % 2 :
    error ( filename , linenum , 'readability/multiline_string' , 5 , 'Multi-line string ("...") found.  This lint script doesn\'t ' 'do well with such strings, and may give bogus warnings.  ' 'Use C++11 raw strings or concatenation instead.' )


signature:
def CheckPosixThreading ( filename , clean_lines , linenum , error ) :
    """Checks for calls to thread - unsafe functions .
 Much code has been originally written without consideration of
 multi - threading . Also , engineers are relying on their old experience ;
 they have learned posix before threading extensions were added . These
 tests guide the engineers to use thread - safe functions ( when using
 posix directly ) .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 error : The function to call with any errors found ."""

implementation:
line = clean_lines . elided [ linenum ]
for single_thread_func , multithread_safe_func , pattern in _THREADING_LIST : # Additional pattern matching check to confirm that this is the
# function we are looking for
    if Search ( pattern , line ) :
        error ( filename , linenum , 'runtime/threadsafe_fn' , 2 , 'Consider using ' + multithread_safe_func + '...) instead of ' + single_thread_func + '...) for improved thread safety.' )


signature:
def CheckVlogArguments ( filename , clean_lines , linenum , error ) :
    """Checks that VLOG ( ) is only used for defining a logging level .
 For example , VLOG ( 2 ) is correct . VLOG ( INFO ) , VLOG ( WARNING ) , VLOG ( ERROR ) , and
 VLOG ( FATAL ) are not .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 error : The function to call with any errors found ."""

implementation:
line = clean_lines . elided [ linenum ]
if Search ( r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)' , line ) :
    error ( filename , linenum , 'runtime/vlog' , 5 , 'VLOG() should be used with numeric verbosity level.  ' 'Use LOG() if you want symbolic severity levels.' )


signature:
def CheckInvalidIncrement ( filename , clean_lines , linenum , error ) :
    """Checks for invalid increment * count + + .
 For example following function :
 void increment _ counter ( int * count ) {
 * count + + ;
 is invalid , because it effectively does count + + , moving pointer , and should
 be replaced with + + * count , ( * count ) + + or * count + = 1.
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 error : The function to call with any errors found ."""

implementation:
line = clean_lines . elided [ linenum ]
if _RE_PATTERN_INVALID_INCREMENT . match ( line ) :
    error ( filename , linenum , 'runtime/invalid_increment' , 5 , 'Changing pointer instead of value (or unused value of operator*).' )


signature:
def CheckSpacingForFunctionCall ( filename , clean_lines , linenum , error ) :
    """Checks for the correctness of various spacing around function calls .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 error : The function to call with any errors found ."""

implementation:
line = clean_lines . elided [ linenum ]
# Since function calls often occur inside if / for / while / switch
# expressions - which have their own , more liberal conventions - we
# first see if we should be looking inside such an expression for a
# function call , to which we can apply more strict standards .
fncall = line
# if there ' s no control flow construct , look at whole line
for pattern in ( r'\bif\s*\((.*)\)\s*{' , r'\bfor\s*\((.*)\)\s*{' , r'\bwhile\s*\((.*)\)\s*[{;]' , r'\bswitch\s*\((.*)\)\s*{' ) :
    match = Search ( pattern , line )
    if match :
        fncall = match . group ( 1 )
        # look inside the parens for function calls
        break
# Except in if / for / while / switch , there should never be space
# immediately inside parens ( eg " f ( 3 , 4 ) " ) . We make an exception
# for nested parens ( ( a + b ) + c ) . Likewise , there should never be
# a space before a ( when it ' s a function argument . I assume it ' s a
# function argument when the char before the whitespace is legal in
# a function name ( alnum + _ ) and we ' re not starting a macro . Also ignore
# pointers and references to arrays and functions coz they ' re too tricky :
# we use a very simple way to recognize these :
# " ( something ) ( maybe - something ) " or
# " ( something ) ( maybe - something , " or
# " ( something ) [ something ] "
# Note that we assume the contents of [ ] to be short enough that
# they ' ll never need to wrap .
if ( # Ignore control structures .
not Search ( r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b' , fncall ) and # Ignore pointers / references to functions .
not Search ( r' \([^)]+\)\([^)]*(\)|,$)' , fncall ) and # Ignore pointers / references to arrays .
not Search ( r' \([^)]+\)\[[^\]]+\]' , fncall ) ) :
    if Search ( r'\w\s*\(\s(?!\s*\\$)' , fncall ) : # a ( used for a fn call
        error ( filename , linenum , 'whitespace/parens' , 4 , 'Extra space after ( in function call' )
    elif Search ( r'\(\s+(?!(\s*\\)|\()' , fncall ) :
        error ( filename , linenum , 'whitespace/parens' , 2 , 'Extra space after (' )
    if ( Search ( r'\w\s+\(' , fncall ) and not Search ( r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(' , fncall ) and not Search ( r'#\s*define|typedef|using\s+\w+\s*=' , fncall ) and not Search ( r'\w\s+\((\w+::)*\*\w+\)\(' , fncall ) and not Search ( r'\bcase\s+\(' , fncall ) ) : # TODO ( unknown ) : Space after an operator function seem to be a common
    # error , silence those for now by restricting them to highest verbosity .
        if Search ( r'\boperator_*\b' , line ) :
            error ( filename , linenum , 'whitespace/parens' , 0 , 'Extra space before ( in function call' )
        else :
            error ( filename , linenum , 'whitespace/parens' , 4 , 'Extra space before ( in function call' )
    # If the ) is followed only by a newline or a { + newline , assume it ' s
    # part of a control statement ( if / while / etc ) , and don ' t complain
    if Search ( r'[^)]\s+\)\s*[^{\s]' , fncall ) : # If the closing parenthesis is preceded by only whitespaces ,
    # try to give a more descriptive error message .
        if Search ( r'^\s+\)' , fncall ) :
            error ( filename , linenum , 'whitespace/parens' , 2 , 'Closing ) should be moved to the previous line' )
        else :
            error ( filename , linenum , 'whitespace/parens' , 2 , 'Extra space before )' )


signature:
def CheckForFunctionLengths ( filename , clean_lines , linenum , function_state , error ) :
    """Reports for long function bodies .
 For an overview why this is done , see :
 https : / / google - styleguide . googlecode . com / svn / trunk / cppguide . xml # Write _ Short _ Functions
 Uses a simplistic algorithm assuming other style guidelines
 ( especially spacing ) are followed .
 Only checks unindented functions , so class members are unchecked .
 Trivial bodies are unchecked , so constructors with huge initializer lists
 may be missed .
 Blank / comment lines are not counted so as to avoid encouraging the removal
 of vertical space and comments just to get through a lint check .
 NOLINT * on the last line of a function * disables this check .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 function _ state : Current function name and lines in body so far .
 error : The function to call with any errors found ."""

implementation:
lines = clean_lines . lines
line = lines [ linenum ]
joined_line = ''
starting_func = False
regexp = r'(\w(\w|::|\*|\&|\s)*)\('
# decls * & space : : name ( . . .
match_result = Match ( regexp , line )
if match_result : # If the name is all caps and underscores , figure it ' s a macro and
# ignore it , unless it ' s TEST or TEST _ F .
    function_name = match_result . group ( 1 ) . split ( ) [ - 1 ]
    if function_name == 'TEST' or function_name == 'TEST_F' or ( not Match ( r'[A-Z_]+$' , function_name ) ) :
        starting_func = True
if starting_func :
    body_found = False
    for start_linenum in range ( linenum , clean_lines . NumLines ( ) ) :
        start_line = lines [ start_linenum ]
        joined_line += ' ' + start_line . lstrip ( )
        if Search ( r'(;|})' , start_line ) : # Declarations and trivial functions
            body_found = True
            break
            # . . . ignore
        elif Search ( r'{' , start_line ) :
            body_found = True
            function = Search ( r'((\w|:)*)\(' , line ) . group ( 1 )
            if Match ( r'TEST' , function ) : # Handle TEST . . . macros
                parameter_regexp = Search ( r'(\(.*\))' , joined_line )
                if parameter_regexp : # Ignore bad syntax
                    function += parameter_regexp . group ( 1 )
            else :
                function += '()'
            function_state . Begin ( function )
            break
    if not body_found : # No body for the function ( or evidence of a non - function ) was found .
        error ( filename , linenum , 'readability/fn_size' , 5 , 'Lint failed to find start of function body.' )
elif Match ( r'^\}\s*$' , line ) : # function end
    function_state . Check ( error , filename , linenum )
    function_state . End ( )
elif not Match ( r'^\s*$' , line ) :
    function_state . Count ( )


signature:
def CheckComment ( line , filename , linenum , next_line_start , error ) :
    """Checks for common mistakes in comments .
 Args :
 line : The line in question .
 filename : The name of the current file .
 linenum : The number of the line to check .
 next _ line _ start : The first non - whitespace column of the next line .
 error : The function to call with any errors found ."""

implementation:
commentpos = line . find ( '//' )
if commentpos != - 1 : # Check if the / / may be in quotes . If so , ignore it
    if re . sub ( r'\\.' , '' , line [ 0 : commentpos ] ) . count ( '"' ) % 2 == 0 : # Allow one space for new scopes , two spaces otherwise :
        if ( not ( Match ( r'^.*{ *//' , line ) and next_line_start == commentpos ) and ( ( commentpos >= 1 and line [ commentpos - 1 ] not in string . whitespace ) or ( commentpos >= 2 and line [ commentpos - 2 ] not in string . whitespace ) ) ) :
            error ( filename , linenum , 'whitespace/comments' , 2 , 'At least two spaces is best between code and comments' )
        # Checks for common mistakes in TODO comments .
        comment = line [ commentpos : ]
        match = _RE_PATTERN_TODO . match ( comment )
        if match : # One whitespace is correct ; zero whitespace is handled elsewhere .
            leading_whitespace = match . group ( 1 )
            if len ( leading_whitespace ) > 1 :
                error ( filename , linenum , 'whitespace/todo' , 2 , 'Too many spaces before TODO' )
            username = match . group ( 2 )
            if not username :
                error ( filename , linenum , 'readability/todo' , 2 , 'Missing username in TODO; it should look like ' '"// TODO(my_username): Stuff."' )
            middle_whitespace = match . group ( 3 )
            # Comparisons made explicit for correctness - - pylint : disable = g - explicit - bool - comparison
            if middle_whitespace != ' ' and middle_whitespace != '' :
                error ( filename , linenum , 'whitespace/todo' , 2 , 'TODO(my_username) should be followed by a space' )
        # If the comment contains an alphanumeric character , there
        # should be a space somewhere between it and the / / unless
        # it ' s a / / / or / / ! Doxygen comment .
        if ( Match ( r'//[^ ]*\w' , comment ) and not Match ( r'(///|//\!)(\s+|$)' , comment ) ) :
            error ( filename , linenum , 'whitespace/comments' , 4 , 'Should have a space between // and comment' )


signature:
def CheckAccess ( filename , clean_lines , linenum , nesting_state , error ) :
    """Checks for improper use of DISALLOW * macros .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 nesting _ state : A NestingState instance which maintains information about
 the current stack of nested blocks being parsed .
 error : The function to call with any errors found ."""

implementation:
line = clean_lines . elided [ linenum ]
# get rid of comments and strings
matched = Match ( ( r'\s*(DISALLOW_COPY_AND_ASSIGN|' r'DISALLOW_IMPLICIT_CONSTRUCTORS)' ) , line )
if not matched :
    return
if nesting_state . stack and isinstance ( nesting_state . stack [ - 1 ] , _ClassInfo ) :
    if nesting_state . stack [ - 1 ] . access != 'private' :
        error ( filename , linenum , 'readability/constructors' , 3 , '%s must be in the private: section' % matched . group ( 1 ) )
else : # Found DISALLOW * macro outside a class declaration , or perhaps it
# was used inside a function when it should have been part of the
# class declaration . We could issue a warning here , but it
# probably resulted in a compiler error already .
    pass


signature:
def CheckSpacing ( filename , clean_lines , linenum , nesting_state , error ) :
    """Checks for the correctness of various spacing issues in the code .
 Things we check for : spaces around operators , spaces after
 if / for / while / switch , no spaces around parens in function calls , two
 spaces between code and comment , don ' t start a block with a blank
 line , don ' t end a function with a blank line , don ' t add a blank line
 after public / protected / private , don ' t have too many blank lines in a row .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 nesting _ state : A NestingState instance which maintains information about
 the current stack of nested blocks being parsed .
 error : The function to call with any errors found ."""

implementation:
# Don ' t use " elided " lines here , otherwise we can ' t check commented lines .
# Don ' t want to use " raw " either , because we don ' t want to check inside C + + 11
# raw strings ,
raw = clean_lines . lines_without_raw_strings
line = raw [ linenum ]
# Before nixing comments , check if the line is blank for no good
# reason . This includes the first line after a block is opened , and
# blank lines at the end of a function ( ie , right before a line like ' } '
# Skip all the blank line checks if we are immediately inside a
# namespace body . In other words , don ' t issue blank line warnings
# for this block :
# namespace {
# A warning about missing end of namespace comments will be issued instead .
# Also skip blank line checks for ' extern " C " ' blocks , which are formatted
# like namespaces .
if ( IsBlankLine ( line ) and not nesting_state . InNamespaceBody ( ) and not nesting_state . InExternC ( ) ) :
    elided = clean_lines . elided
    prev_line = elided [ linenum - 1 ]
    prevbrace = prev_line . rfind ( '{' )
    # TODO ( unknown ) : Don ' t complain if line before blank line , and line after ,
    # both start with alnums and are indented the same amount .
    # This ignores whitespace at the start of a namespace block
    # because those are not usually indented .
    if prevbrace != - 1 and prev_line [ prevbrace : ] . find ( '}' ) == - 1 : # OK , we have a blank line at the start of a code block . Before we
    # complain , we check if it is an exception to the rule : The previous
    # non - empty line has the parameters of a function header that are indented
    # 4 spaces ( because they did not fit in a 80 column line when placed on
    # the same line as the function name ) . We also check for the case where
    # the previous line is indented 6 spaces , which may happen when the
    # initializers of a constructor do not fit into a 80 column line .
        exception = False
        if Match ( r' {6}\w' , prev_line ) : # Initializer list ?
        # We are looking for the opening column of initializer list , which
        # should be indented 4 spaces to cause 6 space indentation afterwards .
            search_position = linenum - 2
            while ( search_position >= 0 and Match ( r' {6}\w' , elided [ search_position ] ) ) :
                search_position -= 1
            exception = ( search_position >= 0 and elided [ search_position ] [ : 5 ] == '    :' )
        else : # Search for the function arguments or an initializer list . We use a
        # simple heuristic here : If the line is indented 4 spaces ; and we have a
        # closing paren , without the opening paren , followed by an opening brace
        # or colon ( for initializer lists ) we assume that it is the last line of
        # a function header . If we have a colon indented 4 spaces , it is an
        # initializer list .
            exception = ( Match ( r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)' , prev_line ) or Match ( r' {4}:' , prev_line ) )
        if not exception :
            error ( filename , linenum , 'whitespace/blank_line' , 2 , 'Redundant blank line at the start of a code block ' 'should be deleted.' )
    # Ignore blank lines at the end of a block in a long if - else
    # chain , like this :
    # if ( condition1 ) {
    # / / Something followed by a blank line
    # } else if ( condition2 ) {
    # / / Something else
    if linenum + 1 < clean_lines . NumLines ( ) :
        next_line = raw [ linenum + 1 ]
        if ( next_line and Match ( r'\s*}' , next_line ) and next_line . find ( '} else ' ) == - 1 ) :
            error ( filename , linenum , 'whitespace/blank_line' , 3 , 'Redundant blank line at the end of a code block ' 'should be deleted.' )
    matched = Match ( r'\s*(public|protected|private):' , prev_line )
    if matched :
        error ( filename , linenum , 'whitespace/blank_line' , 3 , 'Do not leave a blank line after "%s:"' % matched . group ( 1 ) )
# Next , check comments
next_line_start = 0
if linenum + 1 < clean_lines . NumLines ( ) :
    next_line = raw [ linenum + 1 ]
    next_line_start = len ( next_line ) - len ( next_line . lstrip ( ) )
CheckComment ( line , filename , linenum , next_line_start , error )
# get rid of comments and strings
line = clean_lines . elided [ linenum ]
# You shouldn ' t have spaces before your brackets , except maybe after
# ' delete [ ] ' or ' return [ ] ( ) { } ; '
if Search ( r'\w\s+\[' , line ) and not Search ( r'(?:delete|return)\s+\[' , line ) :
    error ( filename , linenum , 'whitespace/braces' , 5 , 'Extra space before [' )
# In range - based for , we wanted spaces before and after the colon , but
# not around " : : " tokens that might appear .
if ( Search ( r'for *\(.*[^:]:[^: ]' , line ) or Search ( r'for *\(.*[^: ]:[^:]' , line ) ) :
    error ( filename , linenum , 'whitespace/forcolon' , 2 , 'Missing space around colon in range-based for loop' )


signature:
def CheckParenthesisSpacing ( filename , clean_lines , linenum , error ) :
    """Checks for horizontal spacing around parentheses .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 error : The function to call with any errors found ."""

implementation:
line = clean_lines . elided [ linenum ]
# No spaces after an if , while , switch , or for
match = Search ( r' (if\(|for\(|while\(|switch\()' , line )
if match :
    error ( filename , linenum , 'whitespace/parens' , 5 , 'Missing space before ( in %s' % match . group ( 1 ) )
# For if / for / while / switch , the left and right parens should be
# consistent about how many spaces are inside the parens , and
# there should either be zero or one spaces inside the parens .
# We don ' t want : " if ( foo ) " or " if ( foo ) " .
# Exception : " for ( ; foo ; bar ) " and " for ( foo ; bar ; ) " are allowed .
match = Search ( r'\b(if|for|while|switch)\s*' r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$' , line )
if match :
    if len ( match . group ( 2 ) ) != len ( match . group ( 4 ) ) :
        if not ( match . group ( 3 ) == ';' and len ( match . group ( 2 ) ) == 1 + len ( match . group ( 4 ) ) or not match . group ( 2 ) and Search ( r'\bfor\s*\(.*; \)' , line ) ) :
            error ( filename , linenum , 'whitespace/parens' , 5 , 'Mismatching spaces inside () in %s' % match . group ( 1 ) )
    if len ( match . group ( 2 ) ) not in [ 0 , 1 ] :
        error ( filename , linenum , 'whitespace/parens' , 5 , 'Should have zero or one spaces inside ( and ) in %s' % match . group ( 1 ) )


signature:
def CheckCommaSpacing ( filename , clean_lines , linenum , error ) :
    """Checks for horizontal spacing near commas and semicolons .
 Args :
 filename : The name of the current file .
 clean _ lines : A CleansedLines instance containing the file .
 linenum : The number of the line to check .
 error : The function to call with any errors found ."""

implementation:
raw = clean_lines . lines_without_raw_strings
line = clean_lines . elided [ linenum ]
# You should always have a space after a comma ( either as fn arg or operator )
# This does not apply when the non - space character following the
# comma is another comma , since the only time when that happens is
# for empty macro arguments .
# We run this check in two passes : first pass on elided lines to
# verify that lines contain missing whitespaces , second pass on raw
# lines to confirm that those missing whitespaces are not due to
# elided comments .
if ( Search ( r',[^,\s]' , ReplaceAll ( r'\boperator\s*,\s*\(' , 'F(' , line ) ) and Search ( r',[^,\s]' , raw [ linenum ] ) ) :
    error ( filename , linenum , 'whitespace/comma' , 3 , 'Missing space after ,' )
# You should always have a space after a semicolon
# except for few corner cases
# TODO ( unknown ) : clarify if ' if ( 1 ) { return 1 ; } ' is requires one more
# space after ;
if Search ( r';[^\s};\\)/]' , line ) :
    error ( filename , linenum , 'whitespace/semicolon' , 3 , 'Missing space after ;' )


signature:
def _IsType ( clean_lines , nesting_state , expr ) :
    """Check if expression looks like a type name , returns true if so .
 Args :
 clean _ lines : A CleansedLines instance containing the file .
 nesting _ state : A NestingState instance which maintains information about
 the current stack of nested blocks being parsed .
 expr : The expression to check .
 Returns :
 True , if token looks like a type ."""

implementation:
# Keep only the last token in the expression
last_word = Match ( r'^.*(\b\S+)$' , expr )
if last_word :
    token = last_word . group ( 1 )
else :
    token = expr
# Match native types and stdint types
if _TYPES . match ( token ) :
    return True
# Try a bit harder to match templated types . Walk up the nesting
# stack until we find something that resembles a typename
# declaration for what we are looking for .
typename_pattern = ( r'\b(?:typename|class|struct)\s+' + re . escape ( token ) + r'\b' )
block_index = len ( nesting_state . stack ) - 1
while block_index >= 0 :
    if isinstance ( nesting_state . stack [ block_index ] , _NamespaceInfo ) :
        return False
    # Found where the opening brace is . We want to scan from this
    # line up to the beginning of the function , minus a few lines .
    # template < typename Type1 , / / stop scanning here
    # class C
    # : public . . . { / / start scanning here
    last_line = nesting_state . stack [ block_index ] . starting_linenum
    next_block_start = 0
    if block_index > 0 :
        next_block_start = nesting_state . stack [ block_index - 1 ] . starting_linenum
    first_line = last_line
    while first_line >= next_block_start :
        if clean_lines . elided [ first_line ] . find ( 'template' ) >= 0 :
            break
        first_line -= 1
    if first_line < next_block_start : # Didn ' t find any " template " keyword before reaching the next block ,
    # there are probably no template things to check for this block
        block_index -= 1
        continue
    # Look for typename in the specified range
    for i in xrange ( first_line , last_line + 1 , 1 ) :
        if Search ( typename_pattern , clean_lines . elided [ i ] ) :
            return True
    block_index -= 1
return False