signature:
def connect ( self ) :
    """Connects and subscribes to the WebSocket Feed ."""

implementation:
if not self . connected ( ) :
    self . _ws = create_connection ( self . WS_URI )
    message = { 'type' : self . WS_TYPE , 'product_id' : self . WS_PRODUCT_ID }
    self . _ws . send ( dumps ( message ) )
    # There will be only one keep alive thread per client instance
    with self . _lock :
        if not self . _thread :
            thread = Thread ( target = self . _keep_alive_thread , args = [ ] )
            thread . start ( )


signature:
def cached_httpbl_exempt ( view_func ) :
    """Marks a view function as being exempt from the cached httpbl view protection ."""

implementation:
# We could just do view _ func . cached _ httpbl _ exempt = True , but decorators
# are nicer if they don ' t have side - effects , so we return a new
# function .
def wrapped_view ( * args , ** kwargs ) :
    return view_func ( * args , ** kwargs )
wrapped_view . cached_httpbl_exempt = True
return wraps ( view_func , assigned = available_attrs ( view_func ) ) ( wrapped_view )


signature:
def get_conn ( self , aws_access_key = None , aws_secret_key = None ) :
    '''Hook point for overriding how the CounterPool gets its connection to
 AWS .'''

implementation: return boto . connect_dynamodb ( aws_access_key_id = aws_access_key , aws_secret_access_key = aws_secret_key , )


signature:
def get_schema ( self ) :
    '''Hook point for overriding how the CounterPool determines the schema
 to be used when creating a missing table .'''

implementation:
if not self . schema :
    raise NotImplementedError ( 'You must provide a schema value or override the get_schema method' )
return self . conn . create_schema ( ** self . schema )


signature:
def create_table ( self ) :
    '''Hook point for overriding how the CounterPool creates a new table
 in DynamooDB'''

implementation:
table = self . conn . create_table ( name = self . get_table_name ( ) , schema = self . get_schema ( ) , read_units = self . get_read_units ( ) , write_units = self . get_write_units ( ) , )
if table . status != 'ACTIVE' :
    table . refresh ( wait_for_active = True , retry_seconds = 1 )
return table


signature:
def get_table ( self ) :
    '''Hook point for overriding how the CounterPool transforms table _ name
 into a boto DynamoDB Table object .'''

implementation:
if hasattr ( self , '_table' ) :
    table = self . _table
else :
    try :
        table = self . conn . get_table ( self . get_table_name ( ) )
    except boto . exception . DynamoDBResponseError :
        if self . auto_create_table :
            table = self . create_table ( )
        else :
            raise
    self . _table = table
return table


signature:
def create_item ( self , hash_key , start = 0 , extra_attrs = None ) :
    '''Hook point for overriding how the CouterPool creates a DynamoDB item
 for a given counter when an existing item can ' t be found .'''

implementation:
table = self . get_table ( )
now = datetime . utcnow ( ) . replace ( microsecond = 0 ) . isoformat ( )
attrs = { 'created_on' : now , 'modified_on' : now , 'count' : start , }
if extra_attrs :
    attrs . update ( extra_attrs )
item = table . new_item ( hash_key = hash_key , attrs = attrs , )
return item


signature:
def get_item ( self , hash_key , start = 0 , extra_attrs = None ) :
    '''Hook point for overriding how the CouterPool fetches a DynamoDB item
 for a given counter .'''

implementation:
table = self . get_table ( )
try :
    item = table . get_item ( hash_key = hash_key )
except DynamoDBKeyNotFoundError :
    item = None
if item is None :
    item = self . create_item ( hash_key = hash_key , start = start , extra_attrs = extra_attrs , )
return item


signature:
def get_counter ( self , name , start = 0 ) :
    '''Gets the DynamoDB item behind a counter and ties it to a Counter
 instace .'''

implementation:
item = self . get_item ( hash_key = name , start = start )
counter = Counter ( dynamo_item = item , pool = self )
return counter


signature:
def exact_match ( self , descriptor ) :
    """Matches this descriptor to another descriptor exactly .
 Args :
 descriptor : another descriptor to match this one .
 Returns : True if descriptors match or False otherwise ."""

implementation: return self . _exact_match_field ( self . _group , descriptor . get_group ( ) ) and self . _exact_atch_field ( self . _type , descriptor . get_type ( ) ) and self . _exact_match_field ( self . _kind , descriptor . get_kind ( ) ) and self . _exact_match_field ( self . _name , descriptor . get_name ( ) ) and self . _exact_match_field ( self . _version , descriptor . get_version ( ) )


signature:
def many_to_one ( clsname , ** kw ) :
    """Use an event to build a many - to - one relationship on a class .
 This makes use of the : meth : ` . References . _ reference _ table ` method
 to generate a full foreign key relationship to the remote table ."""

implementation:
@ declared_attr
def m2o ( cls ) :
    cls . _references ( ( cls . __name__ , clsname ) )
    return relationship ( clsname , ** kw )
return m2o


signature:
def one_to_many ( clsname , ** kw ) :
    """Use an event to build a one - to - many relationship on a class .
 This makes use of the : meth : ` . References . _ reference _ table ` method
 to generate a full foreign key relationship from the remote table ."""

implementation:
@ declared_attr
def o2m ( cls ) :
    cls . _references ( ( clsname , cls . __name__ ) )
    return relationship ( clsname , ** kw )
return o2m


signature:
def djeffify_string ( string_to_djeff ) :
    """Djeffifies string _ to _ djeff"""

implementation:
string_to_djeff = re . sub ( r'^(?=[jg])' , 'd' , string_to_djeff , flags = re . IGNORECASE )
# first
string_to_djeff = re . sub ( r'[ ](?=[jg])' , ' d' , string_to_djeff , flags = re . IGNORECASE )
# spaces
string_to_djeff = re . sub ( r'[\n](?=[jg])' , '\nd' , string_to_djeff , flags = re . IGNORECASE )
return string_to_djeff


signature:
def handle_data ( self , data ) :
    """Djeffify data between tags"""

implementation:
if data . strip ( ) :
    data = djeffify_string ( data )
self . djhtml += data


signature:
def _reference_table ( cls , ref_table ) :
    """Create a foreign key reference from the local class to the given remote
 table .
 Adds column references to the declarative class and adds a
 ForeignKeyConstraint ."""

implementation:
# create pairs of ( Foreign key column , primary key column )
cols = [ ( sa . Column ( ) , refcol ) for refcol in ref_table . primary_key ]
# set " tablename _ colname = Foreign key Column " on the local class
for col , refcol in cols :
    setattr ( cls , "%s_%s" % ( ref_table . name , refcol . name ) , col )
# add a ForeignKeyConstraint ( [ local columns ] , [ remote columns ] )
cls . __table__ . append_constraint ( sa . ForeignKeyConstraint ( * zip ( * cols ) ) )


signature:
def __try_to_json ( self , request , attr ) :
    """Try to run _ _ json _ _ on the given object .
 Raise TypeError is _ _ json _ _ is missing
 : param request : Pyramid Request object
 : type request : < Request >
 : param obj : Object to JSONify
 : type obj : any object that has _ _ json _ _ method
 : exception : TypeError"""

implementation:
# check for _ _ json _ _ method and try to JSONify
if hasattr ( attr , '__json__' ) :
    return attr . __json__ ( request )
# raise error otherwise
raise TypeError ( '__json__ method missing on %s' % str ( attr ) )


signature:
def prepare_path ( path ) :
    """Path join helper method
 Join paths if list passed
 : type path : str | unicode | list
 : rtype : str | unicode"""

implementation:
if type ( path ) == list :
    return os . path . join ( * path )
return path


signature:
def read_from_file ( file_path , encoding = "utf-8" ) :
    """Read helper method
 : type file _ path : str | unicode
 : type encoding : str | unicode
 : rtype : str | unicode"""

implementation:
with codecs . open ( file_path , "r" , encoding ) as f :
    return f . read ( )


signature:
def write_to_file ( file_path , contents , encoding = "utf-8" ) :
    """Write helper method
 : type file _ path : str | unicode
 : type contents : str | unicode
 : type encoding : str | unicode"""

implementation:
with codecs . open ( file_path , "w" , encoding ) as f :
    f . write ( contents )


signature:
def copy_file ( src , dest ) :
    """Copy file helper method
 : type src : str | unicode
 : type dest : str | unicode"""

implementation:
dir_path = os . path . dirname ( dest )
if not os . path . exists ( dir_path ) :
    os . makedirs ( dir_path )
shutil . copy2 ( src , dest )


signature:
def get_path_extension ( path ) :
    """Split file name and extension
 : type path : str | unicode
 : rtype : one str | unicode"""

implementation:
file_path , file_ext = os . path . splitext ( path )
return file_ext . lstrip ( '.' )


signature:
def split_path ( path ) :
    """Helper method for absolute and relative paths resolution
 Split passed path and return each directory parts
 example : " / usr / share / dir "
 return : [ " usr " , " share " , " dir " ]
 @ type path : one of ( unicode , str )
 @ rtype : list"""

implementation:
result_parts = [ ]
# todo : check loops
while path != "/" :
    parts = os . path . split ( path )
    if parts [ 1 ] == path :
        result_parts . insert ( 0 , parts [ 1 ] )
        break
    elif parts [ 0 ] == path :
        result_parts . insert ( 0 , parts [ 0 ] )
        break
    else :
        path = parts [ 0 ]
        result_parts . insert ( 0 , parts [ 1 ] )
return result_parts


signature:
def _create_api_uri ( self , * parts ) :
    """Creates fully qualified endpoint URIs .
 : param parts : the string parts that form the request URI"""

implementation: return urljoin ( self . API_URI , '/' . join ( map ( quote , parts ) ) )


signature:
def _format_iso_time ( self , time ) :
    """Makes sure we have proper ISO 8601 time .
 : param time : either already ISO 8601 a string or datetime . datetime
 : returns : ISO 8601 time
 : rtype : str"""

implementation:
if isinstance ( time , str ) :
    return time
elif isinstance ( time , datetime ) :
    return time . strftime ( '%Y-%m-%dT%H:%M:%S.%fZ' )
else :
    return None


signature:
def _handle_response ( self , response ) :
    """Returns the given response or raises an APIError for non - 2xx responses .
 : param requests . Response response : HTTP response
 : returns : requested data
 : rtype : requests . Response
 : raises APIError : for non - 2xx responses"""

implementation:
if not str ( response . status_code ) . startswith ( '2' ) :
    raise get_api_error ( response )
return response


signature:
def _check_next ( self ) :
    """Checks if a next message is possible .
 : returns : True if a next message is possible , otherwise False
 : rtype : bool"""

implementation:
if self . is_initial :
    return True
if self . before :
    if self . before_cursor :
        return True
    else :
        return False
else :
    if self . after_cursor :
        return True
    else :
        return False


signature:
def _wrap_color ( self , code , text , format = None , style = None ) :
    """Colors text with code and given format"""

implementation:
color = None
if code [ : 3 ] == self . bg . PREFIX :
    color = self . bg . COLORS . get ( code , None )
if not color :
    color = self . fg . COLORS . get ( code , None )
if not color :
    raise Exception ( 'Color code not found' )
if format and format not in self . formats :
    raise Exception ( 'Color format not found' )
fmt = "0;"
if format == 'bold' :
    fmt = "1;"
elif format == 'underline' :
    fmt = "4;"
# Manage the format
parts = color . split ( '[' )
color = '{0}[{1}{2}' . format ( parts [ 0 ] , fmt , parts [ 1 ] )
if self . has_colors and self . colors_enabled : # Set brightness
    st = ''
    if style :
        st = self . st . COLORS . get ( style , '' )
    return "{0}{1}{2}{3}" . format ( st , color , text , self . st . COLORS [ 'reset_all' ] )
else :
    return text


signature:
def RegisterMessage ( self , message ) :
    """Registers the given message type in the local database .
 Args :
 message : a message . Message , to be registered .
 Returns :
 The provided message ."""

implementation:
desc = message . DESCRIPTOR
self . _symbols [ desc . full_name ] = message
if desc . file . name not in self . _symbols_by_file :
    self . _symbols_by_file [ desc . file . name ] = { }
self . _symbols_by_file [ desc . file . name ] [ desc . full_name ] = message
self . pool . AddDescriptor ( desc )
return message


signature:
def GetMessages ( self , files ) :
    """Gets all the messages from a specified file .
 This will find and resolve dependencies , failing if they are not registered
 in the symbol database .
 Args :
 files : The file names to extract messages from .
 Returns :
 A dictionary mapping proto names to the message classes . This will include
 any dependent messages as well as any messages defined in the same file as
 a specified message .
 Raises :
 KeyError : if a file could not be found ."""

implementation:
result = { }
for f in files :
    result . update ( self . _symbols_by_file [ f ] )
return result


signature:
def insert ( self , index , value ) :
    """Insert object before index .
 : param int index : index to insert in
 : param string value : path to insert"""

implementation:
self . _list . insert ( index , value )
self . _sync ( )


signature:
def parse ( self , string ) :
    """Parse runtime path representation to list .
 : param string string : runtime path string
 : return : list of runtime paths
 : rtype : list of string"""

implementation:
var , eq , values = string . strip ( ) . partition ( '=' )
assert var == 'runtimepath'
assert eq == '='
return values . split ( ',' )


signature:
def add_bundle ( self , * args ) :
    """Add some bundle to build group
 : type bundle : static _ bundle . bundles . AbstractBundle
 @ rtype : BuildGroup"""

implementation:
for bundle in args :
    if not self . multitype and self . has_bundles ( ) :
        first_bundle = self . get_first_bundle ( )
        if first_bundle . get_type ( ) != bundle . get_type ( ) :
            raise Exception ( 'Different bundle types for one Asset: %s[%s -> %s]' 'check types or set multitype parameter to True' % ( self . name , first_bundle . get_type ( ) , bundle . get_type ( ) ) )
    self . bundles . append ( bundle )
return self


signature:
def collect_files ( self ) :
    """Return collected files links
 : rtype : list [ static _ bundle . files . StaticFileResult ]"""

implementation:
self . files = [ ]
for bundle in self . bundles :
    bundle . init_build ( self , self . builder )
    bundle_files = bundle . prepare ( )
    self . files . extend ( bundle_files )
return self


signature:
def get_minifier ( self ) :
    """Asset minifier
 Uses default minifier in bundle if it ' s not defined
 : rtype : static _ bundle . minifiers . DefaultMinifier | None"""

implementation:
if self . minifier is None :
    if not self . has_bundles ( ) :
        raise Exception ( "Unable to get default minifier, no bundles in build group" )
    minifier = self . get_first_bundle ( ) . get_default_minifier ( )
else :
    minifier = self . minifier
if minifier :
    minifier . init_asset ( self )
return minifier


signature:
def create_asset ( self , name , ** kwargs ) :
    """Create asset
 : type name : unicode | str
 : rtype : Asset"""

implementation:
asset = Asset ( self , name , ** kwargs )
self . assets [ name ] = asset
return asset


signature:
def render_asset ( self , name ) :
    """Render all includes in asset by names
 : type name : str | unicode
 : rtype : str | unicode"""

implementation:
result = ""
if self . has_asset ( name ) :
    asset = self . get_asset ( name )
    if asset . files :
        for f in asset . files :
            result += f . render_include ( ) + "\r\n"
return result


signature:
def collect_links ( self , env = None ) :
    """Return links without build files"""

implementation:
for asset in self . assets . values ( ) :
    if asset . has_bundles ( ) :
        asset . collect_files ( )
if env is None :
    env = self . config . env
if env == static_bundle . ENV_PRODUCTION :
    self . _minify ( emulate = True )
self . _add_url_prefix ( )


signature:
def make_build ( self ) :
    """Move files / make static build"""

implementation:
for asset in self . assets . values ( ) :
    if asset . has_bundles ( ) :
        asset . collect_files ( )
if not os . path . exists ( self . config . output_dir ) :
    os . makedirs ( self . config . output_dir )
if self . config . copy_only_bundles :
    for asset in self . assets . values ( ) :
        if not asset . minify and asset . files :
            for f in asset . files :
                copy_file ( f . abs_path , self . _get_output_path ( f . abs_path ) )
else :
    copy_excludes = { }
    for asset in self . assets . values ( ) :
        if asset . minify and asset . files :
            for f in asset . files :
                copy_excludes [ f . abs_path ] = f
    for root , dirs , files in os . walk ( self . config . input_dir ) :
        for fpath in files :
            current_file_path = os . path . join ( root , fpath )
            if current_file_path not in copy_excludes :
                copy_file ( current_file_path , self . _get_output_path ( current_file_path ) )
self . _minify ( )


signature:
def clear ( self , exclude = None ) :
    """Clear build output dir
 : type exclude : list | None"""

implementation:
exclude = exclude or [ ]
for root , dirs , files in os . walk ( self . config . output_dir ) :
    for f in files :
        if f not in exclude :
            os . unlink ( os . path . join ( root , f ) )
    for d in dirs :
        if d not in exclude :
            shutil . rmtree ( os . path . join ( root , d ) )


signature:
def _default_json_default ( obj ) :
    """Coerce everything to strings .
 All objects representing time get output according to default _ date _ fmt ."""

implementation:
if isinstance ( obj , ( datetime . datetime , datetime . date , datetime . time ) ) :
    return obj . strftime ( default_date_fmt )
else :
    return str ( obj )


signature:
def init_logs ( path = None , target = None , logger_name = 'root' , level = logging . DEBUG , maxBytes = 1 * 1024 * 1024 , backupCount = 5 , application_name = 'default' , server_hostname = None , fields = None ) :
    """Initialize the zlogger .
 Sets up a rotating file handler to the specified path and file with
 the given size and backup count limits , sets the default
 application _ name , server _ hostname , and default / whitelist fields .
 : param path : path to write the log file
 : param target : name of the log file
 : param logger _ name : name of the logger ( defaults to root )
 : param level : log level for this logger ( defaults to logging . DEBUG )
 : param maxBytes : size of the file before rotation ( default 1MB )
 : param application _ name : app name to add to each log entry
 : param server _ hostname : hostname to add to each log entry
 : param fields : default / whitelist fields .
 : type path : string
 : type target : string
 : type logger _ name : string
 : type level : int
 : type maxBytes : int
 : type backupCount : int
 : type application _ name : string
 : type server _ hostname : string
 : type fields : dict"""

implementation:
log_file = os . path . abspath ( os . path . join ( path , target ) )
logger = logging . getLogger ( logger_name )
logger . setLevel ( level )
handler = logging . handlers . RotatingFileHandler ( log_file , maxBytes = maxBytes , backupCount = backupCount )
handler . setLevel ( level )
handler . setFormatter ( JsonFormatter ( application_name = application_name , server_hostname = server_hostname , fields = fields ) )
logger . addHandler ( handler )


signature:
def format ( self , record ) :
    """formats a logging . Record into a standard json log entry
 : param record : record to be formatted
 : type record : logging . Record
 : return : the formatted json string
 : rtype : string"""

implementation:
record_fields = record . __dict__ . copy ( )
self . _set_exc_info ( record_fields )
event_name = 'default'
if record_fields . get ( 'event_name' ) :
    event_name = record_fields . pop ( 'event_name' )
log_level = 'INFO'
if record_fields . get ( 'log_level' ) :
    log_level = record_fields . pop ( 'log_level' )
[ record_fields . pop ( k ) for k in record_fields . keys ( ) if k not in self . fields ]
defaults = self . defaults . copy ( )
fields = self . fields . copy ( )
fields . update ( record_fields )
filtered_fields = { }
for k , v in fields . iteritems ( ) :
    if v is not None :
        filtered_fields [ k ] = v
defaults . update ( { 'event_timestamp' : self . _get_now ( ) , 'event_name' : event_name , 'log_level' : log_level , 'fields' : filtered_fields } )
return json . dumps ( defaults , default = self . json_default )


signature:
def includeme ( config ) :
    """Initialize the model for a Pyramid app .
 Activate this setup using ` ` config . include ( ' baka _ model ' ) ` ` ."""

implementation:
settings = config . get_settings ( )
should_create = asbool ( settings . get ( 'baka_model.should_create_all' , False ) )
should_drop = asbool ( settings . get ( 'baka_model.should_drop_all' , False ) )
# Configure the transaction manager to support retrying retryable
# exceptions . We also register the session factory with the thread - local
# transaction manager , so that all sessions it creates are registered .
# " tm . attempts " : 3,
config . add_settings ( { "retry.attempts" : 3 , "tm.activate_hook" : tm_activate_hook , "tm.annotate_user" : False , } )
# use pyramid _ retry couse pyramid _ tm disabled it
config . include ( 'pyramid_retry' )
# use pyramid _ tm to hook the transaction lifecycle to the request
config . include ( 'pyramid_tm' )
engine = get_engine ( settings )
session_factory = get_session_factory ( engine )
config . registry [ 'db_session_factory' ] = session_factory
# make request . db available for use in Pyramid
config . add_request_method ( # r . tm is the transaction manager used by pyramid _ tm
lambda r : get_tm_session ( session_factory , r . tm ) , 'db' , reify = True )
# service model factory
config . include ( '.service' )
# Register a deferred action to bind the engine when the configuration is
# committed . Deferring the action means that this module can be included
# before model modules without ill effect .
config . action ( None , bind_engine , ( engine , ) , { 'should_create' : should_create , 'should_drop' : should_drop } , order = 10 )


signature:
def store ( self , collection , ** kwargs ) :
    '''validate the passed values in kwargs based on the collection ,
 store them in the mongodb collection'''

implementation:
key = validate ( collection , ** kwargs )
if self . fetch ( collection , ** { key : kwargs [ key ] } ) :
    raise Proauth2Error ( 'duplicate_key' )
self . db [ collection ] . insert ( kwargs )


signature:
def get_abs_and_rel_paths ( self , root_path , file_name , input_dir ) :
    """Return absolute and relative path for file
 : type root _ path : str | unicode
 : type file _ name : str | unicode
 : type input _ dir : str | unicode
 : rtype : tuple"""

implementation:
# todo : change relative path resolving [ bug on duplicate dir names in path ]
relative_dir = root_path . replace ( input_dir , '' )
return os . path . join ( root_path , file_name ) , relative_dir + '/' + file_name


signature:
def get_files ( self ) :
    """: inheritdoc"""

implementation:
assert self . bundle , 'Cannot fetch file name with empty bundle'
abs_path , rel_path = self . get_abs_and_rel_paths ( self . bundle . path , self . file_path , self . bundle . input_dir )
file_cls = self . bundle . get_file_cls ( )
return [ file_cls ( rel_path , abs_path ) ]


implementation:
assert self . bundle , 'Cannot fetch directory name with empty bundle'
result_files = [ ]
bundle_ext = self . bundle . get_extension ( )
ext = "." + bundle_ext if bundle_ext else None
if self . directory_path == "" :
    root_path = self . bundle . path
else :
    root_path = os . path . join ( self . bundle . path , self . directory_path )
for root , dirs , files in os . walk ( root_path ) :
    for fpath in files :
        if ( not ext or fpath . endswith ( ext ) ) and ( not self . exclusions or all ( fpath != n for n in self . exclusions ) ) :
            abs_path , rel_path = self . get_abs_and_rel_paths ( root , fpath , self . bundle . input_dir )
            file_cls = self . bundle . get_file_cls ( )
            result_files . append ( file_cls ( rel_path , abs_path ) )
return result_files


signature:
def replicate_existing ( source_db , target_db ) :
    """Replicate an existing database to another existing database ."""

implementation:
# Get the server from which to manage the replication .
server = shortcuts . get_server ( )
logger = logging . getLogger ( 'relax.couchdb.replicate' )
logger . debug ( 'POST ' + urlparse . urljoin ( server . resource . uri , '/_replicate' ) )
source , target = specifier_to_db ( source_db ) , specifier_to_db ( target_db )
logger . debug ( 'Source DB: %s' % ( source , ) )
logger . debug ( 'Target DB: %s' % ( target , ) )
try :
    resp_headers , resp_body = server . resource . post ( path = '/_replicate' , content = json . dumps ( { 'source' : source , 'target' : target } ) )
except couchdb . client . ServerError , exc :
    logger . error ( 'Replication failed.' )
    raise ReplicationError ( exc . args )
result = resp_body [ 'history' ] [ 0 ]
if resp_body [ 'ok' ] :
    logger . info ( 'Replication %s... successful!' % ( resp_body [ 'session_id' ] [ : 6 ] , ) )
    logger . info ( 'Replication started: ' + result [ 'start_time' ] )
    logger . info ( 'Replication finished: ' + result [ 'end_time' ] )
    result [ 'start_time' ] = datetime . datetime . strptime ( result [ 'start_time' ] , '%a, %d %b %Y %H:%M:%S GMT' )
    result [ 'end_time' ] = datetime . datetime . strptime ( result [ 'end_time' ] , '%a, %d %b %Y %H:%M:%S GMT' )
    timedelta = result [ 'end_time' ] - result [ 'start_time' ]
    if timedelta . days :
        logger . info ( 'Replication took %d days and %.2f seconds.' % ( timedelta . days , timedelta . seconds + ( timedelta . microseconds * ( 1e-6 ) ) ) )
    else :
        logger . info ( 'Replication took %.2f seconds.' % ( timedelta . seconds + ( timedelta . microseconds * ( 1e-6 ) ) ) )
    # Prepare the ' result ' dictionary .
    result [ 'ok' ] = resp_body [ 'ok' ]
    result [ 'session_id' ] = resp_body [ 'session_id' ]
    result [ 'source_last_seq' ] = resp_body [ 'source_last_seq' ]
    # Info - log the number of docs read / written and checked / found .
    if result [ 'docs_read' ] == 1 :
        docs_read = '1 document read'
    else :
        docs_read = '%d documents read' % ( result [ 'docs_read' ] , )
    if result [ 'docs_written' ] == 1 :
        docs_written = '1 document written'
    else :
        docs_written = '%d documents written' % ( result [ 'docs_written' ] , )
    if result [ 'missing_checked' ] == 1 :
        missing_checked = 'Checked for 1 missing document, found %d.' % ( result [ 'missing_found' ] , )
    else :
        missing_checked = 'Checked for %d missing documents, found %d.' % ( result [ 'missing_checked' ] , result [ 'missing_found' ] , )
    logging . info ( '%s, %s' % ( docs_read , docs_written ) )
    logging . info ( missing_checked )
    return result
else :
    logger . error ( 'Replication %s... failed.' % ( resp_body [ 'session_id' ] [ : 6 ] , ) )
    result [ 'ok' ] = resp_body [ 'ok' ]
    result [ 'session_id' ] = resp_body [ 'session_id' ]
    result [ 'source_last_seq' ] = resp_body [ 'source_last_seq' ]
    raise ReplicationFailure ( resp_headers , result )


signature:
def mcmc_advance ( start , stdevs , logp , nsteps = 1e300 , adapt = True , callback = None ) :
    """Generic Metropolis MCMC . Advances the chain by nsteps .
 Called by : func : ` mcmc `
 : param adapt : enables adaptive stepwidth alteration ( converges ) ."""

implementation:
import scipy
from numpy import log
import progressbar
prob = logp ( start )
chain = [ start ]
accepts = [ True ]
probs = [ prob ]
assert not numpy . isnan ( start ) . any ( )
assert not numpy . isnan ( stdevs ) . any ( )
i = 0
widgets = [ 'AR' , progressbar . Percentage ( ) , progressbar . Counter ( '%5d' ) , progressbar . Bar ( ) , progressbar . ETA ( ) ]
pbar = progressbar . ProgressBar ( widgets = widgets , maxval = nsteps ) . start ( )
prev = start
prev_prob = prob
print 'MCMC: start at prob' , prob
stepchange = 0.1
while len ( chain ) < nsteps :
    i = i + 1
    next = scipy . random . normal ( prev , stdevs )
    next [ next > 1 ] = 1
    next [ next < 0 ] = 0
    next_prob = logp ( next )
    assert not numpy . isnan ( next ) . any ( )
    assert not numpy . isnan ( next_prob ) . any ( )
    delta = next_prob - prev_prob
    dice = log ( scipy . random . uniform ( 0 , 1 ) )
    accept = delta > dice
    if accept :
        prev = next
        prev_prob = next_prob
        if adapt :
            stdevs *= ( 1 + stepchange )
    else :
        if adapt :
            stdevs *= ( 1 + stepchange ) ** ( - 0.4 )
            # aiming for 40 % acceptance
    if callback :
        callback ( prev_prob , prev , accept )
    chain . append ( prev )
    accepts . append ( accept )
    probs . append ( prev_prob )
    if adapt :
        stepchange = min ( 0.1 , 10. / i )
    # print ' STDEV ' , stdevs [ : 5 ] , stepchange
    # compute stats
    widgets [ 0 ] = 'AR: %.03f' % numpy . mean ( numpy . array ( accepts [ len ( accepts ) / 3 : ] ) + 0 )
    pbar . update ( pbar . currval + 1 )
pbar . finish ( )
return chain , probs , accepts , stdevs


signature:
def mcmc ( transform , loglikelihood , parameter_names , nsteps = 40000 , nburn = 400 , stdevs = 0.1 , start = 0.5 , ** problem ) :
    """* * Metropolis Hastings MCMC * *
 with automatic step width adaption .
 Burnin period is also used to guess steps .
 : param nburn : number of burnin steps
 : param stdevs : step widths to start with"""

implementation:
if 'seed' in problem :
    numpy . random . seed ( problem [ 'seed' ] )
n_params = len ( parameter_names )
def like ( cube ) :
    cube = numpy . array ( cube )
    if ( cube <= 1e-10 ) . any ( ) or ( cube >= 1 - 1e-10 ) . any ( ) :
        return - 1e100
    params = transform ( cube )
    return loglikelihood ( params )
start = start + numpy . zeros ( n_params )
stdevs = stdevs + numpy . zeros ( n_params )
def compute_stepwidths ( chain ) :
    return numpy . std ( chain , axis = 0 ) / 3
import matplotlib . pyplot as plt
plt . figure ( figsize = ( 7 , 7 ) )
steps = numpy . array ( [ 0.1 ] * ( n_params ) )
print 'burn-in (1/2)...'
chain , prob , _ , steps_ = mcmc_advance ( start , steps , like , nsteps = nburn / 2 , adapt = True )
steps = compute_stepwidths ( chain )
print 'burn-in (2/2)...'
chain , prob , _ , steps_ = mcmc_advance ( chain [ - 1 ] , steps , like , nsteps = nburn / 2 , adapt = True )
steps = compute_stepwidths ( chain )
print 'recording chain ...'
chain , prob , _ , steps_ = mcmc_advance ( chain [ - 1 ] , steps , like , nsteps = nsteps )
chain = numpy . array ( chain )
i = numpy . argmax ( prob )
final = chain [ - 1 ]
print 'postprocessing...'
chain = numpy . array ( [ transform ( params ) for params in chain ] )
return dict ( start = chain [ - 1 ] , maximum = chain [ i ] , seeds = [ final , chain [ i ] ] , chain = chain , method = 'Metropolis MCMC' )


signature:
def ensemble ( transform , loglikelihood , parameter_names , nsteps = 40000 , nburn = 400 , start = 0.5 , ** problem ) :
    """* * Ensemble MCMC * *
 via ` emcee < http : / / dan . iel . fm / emcee / > ` _"""

implementation:
import emcee
import progressbar
if 'seed' in problem :
    numpy . random . seed ( problem [ 'seed' ] )
n_params = len ( parameter_names )
nwalkers = 50 + n_params * 2
if nwalkers > 200 :
    nwalkers = 200
p0 = [ numpy . random . rand ( n_params ) for i in xrange ( nwalkers ) ]
start = start + numpy . zeros ( n_params )
p0 [ 0 ] = start
def like ( cube ) :
    cube = numpy . array ( cube )
    if ( cube <= 1e-10 ) . any ( ) or ( cube >= 1 - 1e-10 ) . any ( ) :
        return - 1e100
    params = transform ( cube )
    return loglikelihood ( params )
sampler = emcee . EnsembleSampler ( nwalkers , n_params , like , live_dangerously = True )
print 'burn-in...'
pos , prob , state = sampler . run_mcmc ( p0 , nburn / nwalkers )
# Reset the chain to remove the burn - in samples .
sampler . reset ( )
print 'running ...'
# Starting from the final position in the burn - in chain , sample
pbar = progressbar . ProgressBar ( widgets = [ progressbar . Percentage ( ) , progressbar . Counter ( '%5d' ) , progressbar . Bar ( ) , progressbar . ETA ( ) ] , maxval = nsteps ) . start ( )
for results in sampler . sample ( pos , iterations = nsteps / nwalkers , rstate0 = state ) :
    pbar . update ( pbar . currval + 1 )
pbar . finish ( )
print "Mean acceptance fraction:" , numpy . mean ( sampler . acceptance_fraction )
chain = sampler . flatchain
final = chain [ - 1 ]
print 'postprocessing...'
chain_post = numpy . array ( [ transform ( params ) for params in chain ] )
chain_prob = sampler . flatlnprobability
return dict ( start = final , chain = chain_post , chain_prior = chain , chain_prob = chain_prob , method = 'Ensemble MCMC' )


signature:
def _get_classes ( package_name , base_class ) :
    """search monits or works classes . Class must have ' name ' attribute
 : param package _ name : ' monits ' or ' works '
 : param base _ class : Monit or Work
 : return : tuple of tuples monit / work - name and class"""

implementation:
classes = { }
base_dir = os . getcwd ( )
root_module_name = base_dir . split ( '/' ) [ - 1 ]
package_dir = base_dir + '/%s' % package_name
if os . path . isdir ( package_dir ) :
    for module_path in os . listdir ( package_dir ) :
        if not module_path . endswith ( '.py' ) :
            continue
        module_name = os . path . splitext ( module_path ) [ 0 ]
        module_full_name = '%s.%s.%s' % ( root_module_name , package_name , module_name )
        __import__ ( module_full_name )
        work_module = sys . modules [ module_full_name ]
        for module_item in work_module . __dict__ . values ( ) :
            if type ( module_item ) is type and issubclass ( module_item , base_class ) and module_item is not base_class and hasattr ( module_item , 'name' ) and module_item . name :
                classes . setdefault ( module_item . name , [ ] ) . append ( module_item )
# check no duplicated names
for work_name , work_modules in classes . items ( ) :
    if len ( work_modules ) > 1 :
        raise DuplicatedNameException ( 'Modules %s have same name "%s"' % ( ' and ' . join ( map ( str , work_modules ) ) , work_name ) )
# create immutable list of modules
return tuple ( [ ( work_name , work_modules [ 0 ] ) for work_name , work_modules in classes . items ( ) ] )


signature:
def AddEnumDescriptor ( self , enum_desc ) :
    """Adds an EnumDescriptor to the pool .
 This method also registers the FileDescriptor associated with the message .
 Args :
 enum _ desc : An EnumDescriptor ."""

implementation:
if not isinstance ( enum_desc , descriptor . EnumDescriptor ) :
    raise TypeError ( 'Expected instance of descriptor.EnumDescriptor.' )
self . _enum_descriptors [ enum_desc . full_name ] = enum_desc
self . AddFileDescriptor ( enum_desc . file )


signature:
def FindFileContainingSymbol ( self , symbol ) :
    """Gets the FileDescriptor for the file containing the specified symbol .
 Args :
 symbol : The name of the symbol to search for .
 Returns :
 A FileDescriptor that contains the specified symbol .
 Raises :
 KeyError : if the file can not be found in the pool ."""

implementation:
symbol = _NormalizeFullyQualifiedName ( symbol )
try :
    return self . _descriptors [ symbol ] . file
except KeyError :
    pass
try :
    return self . _enum_descriptors [ symbol ] . file
except KeyError :
    pass
try :
    file_proto = self . _internal_db . FindFileContainingSymbol ( symbol )
except KeyError as error :
    if self . _descriptor_db :
        file_proto = self . _descriptor_db . FindFileContainingSymbol ( symbol )
    else :
        raise error
if not file_proto :
    raise KeyError ( 'Cannot find a file containing %s' % symbol )
return self . _ConvertFileProtoToFileDescriptor ( file_proto )


signature:
def FindMessageTypeByName ( self , full_name ) :
    """Loads the named descriptor from the pool .
 Args :
 full _ name : The full name of the descriptor to load .
 Returns :
 The descriptor for the named type ."""

implementation:
full_name = _NormalizeFullyQualifiedName ( full_name )
if full_name not in self . _descriptors :
    self . FindFileContainingSymbol ( full_name )
return self . _descriptors [ full_name ]


signature:
def FindEnumTypeByName ( self , full_name ) :
    """Loads the named enum descriptor from the pool .
 Args :
 full _ name : The full name of the enum descriptor to load .
 Returns :
 The enum descriptor for the named type ."""

implementation:
full_name = _NormalizeFullyQualifiedName ( full_name )
if full_name not in self . _enum_descriptors :
    self . FindFileContainingSymbol ( full_name )
return self . _enum_descriptors [ full_name ]


signature:
def FindExtensionByName ( self , full_name ) :
    """Loads the named extension descriptor from the pool .
 Args :
 full _ name : The full name of the extension descriptor to load .
 Returns :
 A FieldDescriptor , describing the named extension ."""

implementation:
full_name = _NormalizeFullyQualifiedName ( full_name )
message_name , _ , extension_name = full_name . rpartition ( '.' )
try : # Most extensions are nested inside a message .
    scope = self . FindMessageTypeByName ( message_name )
except KeyError : # Some extensions are defined at file scope .
    scope = self . FindFileContainingSymbol ( full_name )
return scope . extensions_by_name [ extension_name ]


signature:
def _ConvertEnumDescriptor ( self , enum_proto , package = None , file_desc = None , containing_type = None , scope = None ) :
    """Make a protobuf EnumDescriptor given an EnumDescriptorProto protobuf .
 Args :
 enum _ proto : The descriptor _ pb2 . EnumDescriptorProto protobuf message .
 package : Optional package name for the new message EnumDescriptor .
 file _ desc : The file containing the enum descriptor .
 containing _ type : The type containing this enum .
 scope : Scope containing available types .
 Returns :
 The added descriptor"""

implementation:
if package :
    enum_name = '.' . join ( ( package , enum_proto . name ) )
else :
    enum_name = enum_proto . name
if file_desc is None :
    file_name = None
else :
    file_name = file_desc . name
values = [ self . _MakeEnumValueDescriptor ( value , index ) for index , value in enumerate ( enum_proto . value ) ]
desc = descriptor . EnumDescriptor ( name = enum_proto . name , full_name = enum_name , filename = file_name , file = file_desc , values = values , containing_type = containing_type , options = enum_proto . options )
scope [ '.%s' % enum_name ] = desc
self . _enum_descriptors [ enum_name ] = desc
return desc


signature:
def _MakeFieldDescriptor ( self , field_proto , message_name , index , is_extension = False ) :
    """Creates a field descriptor from a FieldDescriptorProto .
 For message and enum type fields , this method will do a look up
 in the pool for the appropriate descriptor for that type . If it
 is unavailable , it will fall back to the _ source function to
 create it . If this type is still unavailable , construction will
 fail .
 Args :
 field _ proto : The proto describing the field .
 message _ name : The name of the containing message .
 index : Index of the field
 is _ extension : Indication that this field is for an extension .
 Returns :
 An initialized FieldDescriptor object"""

implementation:
if message_name :
    full_name = '.' . join ( ( message_name , field_proto . name ) )
else :
    full_name = field_proto . name
return descriptor . FieldDescriptor ( name = field_proto . name , full_name = full_name , index = index , number = field_proto . number , type = field_proto . type , cpp_type = None , message_type = None , enum_type = None , containing_type = None , label = field_proto . label , has_default_value = False , default_value = None , is_extension = is_extension , extension_scope = None , options = field_proto . options )


signature:
def validate ( table , ** data ) :
    '''theoretically , any data store can be implemented to work with this package ,
 which means basic data validation must be done in - package , so that weird
 stuff can ' t be stored in the data store .
 this function raises an exception if an invalid table name is passed , not
 all of the required fields are in the data kwargs , or if a field that was
 passed is not expected .
 it also returns the key field name , for ensuring uniqueness ( again , that may
 not be built into whatever data store is impelemented . )'''

implementation:
if table not in good . keys ( ) :
    raise Proauth2Error ( 'invalid_request' , 'invalid name: %s' % table )
for req in good [ table ] [ 'required' ] :
    if not data . get ( req , None ) :
        raise Proauth2Error ( 'invalid_request' , 'missing required field: %s' % req )
for key in data . keys ( ) :
    if key not in good [ table ] [ 'required' ] and key not in good [ table ] [ 'optional' ] :
        raise Proauth2Error ( 'invalid_request' , 'invalid field: %s' % key )
return good [ table ] [ 'key' ]


signature:
def record_results ( self , results ) :
    """Record the results of this experiment , by updating the tag .
 : param results : A dictionary containing the results of the experiment .
 : type results : dict"""

implementation:
repository = Repo ( self . __repository_directory , search_parent_directories = True )
for tag in repository . tags :
    if tag . name == self . __tag_name :
        tag_object = tag
        break
else :
    raise Exception ( "Experiment tag has been deleted since experiment started" )
data = json . loads ( tag_object . tag . message )
data [ "results" ] = results
TagReference . create ( repository , self . __tag_name , message = json . dumps ( data ) , ref = tag_object . tag . object , force = True )
self . __results_recorded = True


signature:
def __tag_repo ( self , data , repository ) :
    """Tag the current repository .
 : param data : a dictionary containing the data about the experiment
 : type data : dict"""

implementation:
assert self . __tag_name not in [ t . name for t in repository . tags ]
return TagReference . create ( repository , self . __tag_name , message = json . dumps ( data ) )


signature:
def __get_files_to_be_added ( self , repository ) :
    """: return : the files that have been modified and can be added"""

implementation:
for root , dirs , files in os . walk ( repository . working_dir ) :
    for f in files :
        relative_path = os . path . join ( root , f ) [ len ( repository . working_dir ) + 1 : ]
        try :
            repository . head . commit . tree [ relative_path ]
            # will fail if not tracked
            yield relative_path
        except :
            pass


signature:
def __start_experiment ( self , parameters ) :
    """Start an experiment by capturing the state of the code
 : param parameters : a dictionary containing the parameters of the experiment
 : type parameters : dict
 : return : the tag representing this experiment
 : rtype : TagReference"""

implementation:
repository = Repo ( self . __repository_directory , search_parent_directories = True )
if len ( repository . untracked_files ) > 0 :
    logging . warning ( "Untracked files will not be recorded: %s" , repository . untracked_files )
current_commit = repository . head . commit
started_state_is_dirty = repository . is_dirty ( )
if started_state_is_dirty :
    repository . index . add ( [ p for p in self . __get_files_to_be_added ( repository ) ] )
    commit_obj = repository . index . commit ( "Temporary commit for experiment " + self . __experiment_name )
    sha = commit_obj . hexsha
else :
    sha = repository . head . object . hexsha
data = { "parameters" : parameters , "started" : time . time ( ) , "description" : self . __description , "commit_sha" : sha }
tag_object = self . __tag_repo ( data , repository )
if started_state_is_dirty :
    repository . head . reset ( current_commit , working_tree = False , index = True )
return tag_object


signature:
def get_tm_session ( session_factory , transaction_manager ) :
    """Get a ` ` sqlalchemy . orm . Session ` ` instance backed by a transaction .
 This function will hook the session to the transaction manager which
 will take care of committing any changes .
 - When using pyramid _ tm it will automatically be committed or aborted
 depending on whether an exception is raised .
 - When using scripts you should wrap the session in a manager yourself .
 For example : :
 import transaction
 engine = get _ engine ( settings )
 session _ factory = get _ session _ factory ( engine )
 with transaction . manager :
 dbsession = get _ tm _ session ( session _ factory , transaction . manager )"""

implementation:
dbsession = session_factory ( )
zope . sqlalchemy . register ( dbsession , transaction_manager = transaction_manager )
return dbsession


signature:
def run ( self ) :
    """run your main spider here , and get a list / tuple of url as result
 then make the instance of branch thread
 : return : None"""

implementation:
global existed_urls_list
config = config_creator ( )
debug = config . debug
main_thread_sleep = config . main_thread_sleep
branch_thread_num = config . branch_thread_num
while 1 :
    url = self . main_queue . get ( )
    if debug :
        print ( 'main thread-{} start' . format ( url ) )
    main_spider = self . main_spider ( url )
    sleep ( random . randrange ( * main_thread_sleep ) )
    links = main_spider . request_urls ( )
    try :
        assert type ( links ) in VALIDATE_URLS
    except AssertionError :
        error_message ( 'except to return a list or tuple which contains url' )
        links = list ( )
    branch_queue = queue . Queue ( branch_thread_num )
    for i in range ( branch_thread_num ) :
        branch_thread = BranchThread ( branch_queue = branch_queue , branch_spider = self . branch_spider )
        branch_thread . daemon = True
        branch_thread . start ( )
    for link in links :
        if link not in existed_urls_list :
            existed_urls_list . append ( link )
            branch_queue . put ( link )
    branch_queue . join ( )
    if debug :
        print ( 'main thread-{}\'s child threads is all finish' . format ( url ) )
    self . main_queue . task_done ( )


signature:
def pypi_render ( source ) :
    """Copied ( and slightly adapted ) from pypi . description _ tools"""

implementation:
ALLOWED_SCHEMES = '''file ftp gopher hdl http https imap mailto mms news
        nntp prospero rsync rtsp rtspu sftp shttp sip sips snews svn svn+ssh
        telnet wais irc''' . split ( )
settings_overrides = { "raw_enabled" : 0 , # no raw HTML code
"file_insertion_enabled" : 0 , # no file / URL access
"halt_level" : 2 , # at warnings or errors , raise an exception
"report_level" : 5 , # never report problems with the reST code
}
# capture publishing errors , they go to stderr
old_stderr = sys . stderr
sys . stderr = s = StringIO . StringIO ( )
parts = None
try : # Convert reStructuredText to HTML using Docutils .
    document = publish_doctree ( source = source , settings_overrides = settings_overrides )
    for node in document . traverse ( ) :
        if node . tagname == '#text' :
            continue
        if node . hasattr ( 'refuri' ) :
            uri = node [ 'refuri' ]
        elif node . hasattr ( 'uri' ) :
            uri = node [ 'uri' ]
        else :
            continue
        o = urlparse . urlparse ( uri )
        if o . scheme not in ALLOWED_SCHEMES :
            raise TransformError ( 'link scheme not allowed' )
    # now turn the transformed document into HTML
    reader = readers . doctree . Reader ( parser_name = 'null' )
    pub = Publisher ( reader , source = io . DocTreeInput ( document ) , destination_class = io . StringOutput )
    pub . set_writer ( 'html' )
    pub . process_programmatic_settings ( None , settings_overrides , None )
    pub . set_destination ( None , None )
    pub . publish ( )
    parts = pub . writer . parts
except :
    pass
sys . stderr = old_stderr
# original text if publishing errors occur
if parts is None or len ( s . getvalue ( ) ) > 0 :
    return None
else :
    return parts [ 'body' ]


signature:
def generate ( length = DEFAULT_LENGTH ) :
    """Generate a random string of the specified length .
 The returned string is composed of an alphabet that shouldn ' t include any
 characters that are easily mistakeable for one another ( I , 1 , O , 0 ) , and
 hopefully won ' t accidentally contain any English - language curse words ."""

implementation: return '' . join ( random . SystemRandom ( ) . choice ( ALPHABET ) for _ in range ( length ) )


signature:
def require ( name , field , data_type ) :
    """Require that the named ` field ` has the right ` data _ type `"""

implementation:
if not isinstance ( field , data_type ) :
    msg = '{0} must have {1}, got: {2}' . format ( name , data_type , field )
    raise AssertionError ( msg )


signature:
def _enqueue ( self , msg ) :
    """Push a new ` msg ` onto the queue , return ` ( success , msg ) `"""

implementation:
self . log . debug ( 'queueing: %s' , msg )
if self . queue . full ( ) :
    self . log . warn ( 'librato_bg queue is full' )
    return False , msg
self . queue . put ( msg )
self . log . debug ( 'enqueued %s.' , msg )
return True , msg


signature:
def flush ( self ) :
    """Forces a flush from the internal queue to the server"""

implementation:
queue = self . queue
size = queue . qsize ( )
queue . join ( )
self . log . debug ( 'successfully flushed %s items.' , size )


signature:
def open ( name = None , fileobj = None , closefd = True ) :
    """Use all decompressor possible to make the stream"""

implementation: return Guesser ( ) . open ( name = name , fileobj = fileobj , closefd = closefd )


signature:
def marv ( ctx , config , loglevel , logfilter , verbosity ) :
    """Manage a Marv site"""

implementation:
if config is None :
    cwd = os . path . abspath ( os . path . curdir )
    while cwd != os . path . sep :
        config = os . path . join ( cwd , 'marv.conf' )
        if os . path . exists ( config ) :
            break
        cwd = os . path . dirname ( cwd )
    else :
        config = '/etc/marv/marv.conf'
        if not os . path . exists ( config ) :
            config = None
ctx . obj = config
setup_logging ( loglevel , verbosity , logfilter )


signature:
def main ( global_config , ** settings ) :
    """This function returns a Pyramid WSGI application ."""

implementation:
set_cache_regions_from_settings ( settings )
config = Configurator ( settings = settings )
config . include ( 'cms' )
config . configure_celery ( global_config [ '__file__' ] )
return config . make_wsgi_app ( )


signature:
def _SignedVarintDecoder ( mask , result_type ) :
    """Like _ VarintDecoder ( ) but decodes signed values ."""

implementation:
def DecodeVarint ( buffer , pos ) :
    result = 0
    shift = 0
    while 1 :
        b = six . indexbytes ( buffer , pos )
        result |= ( ( b & 0x7f ) << shift )
        pos += 1
        if not ( b & 0x80 ) :
            if result > 0x7fffffffffffffff :
                result -= ( 1 << 64 )
                result |= ~ mask
            else :
                result &= mask
            result = result_type ( result )
            return ( result , pos )
        shift += 7
        if shift >= 64 :
            raise _DecodeError ( 'Too many bytes when decoding varint.' )
return DecodeVarint


signature:
def MessageSetItemDecoder ( extensions_by_number ) :
    """Returns a decoder for a MessageSet item .
 The parameter is the _ extensions _ by _ number map for the message class .
 The message set message looks like this :
 message MessageSet {
 repeated group Item = 1 {
 required int32 type _ id = 2;
 required string message = 3;"""

implementation:
type_id_tag_bytes = encoder . TagBytes ( 2 , wire_format . WIRETYPE_VARINT )
message_tag_bytes = encoder . TagBytes ( 3 , wire_format . WIRETYPE_LENGTH_DELIMITED )
item_end_tag_bytes = encoder . TagBytes ( 1 , wire_format . WIRETYPE_END_GROUP )
local_ReadTag = ReadTag
local_DecodeVarint = _DecodeVarint
local_SkipField = SkipField
def DecodeItem ( buffer , pos , end , message , field_dict ) :
    message_set_item_start = pos
    type_id = - 1
    message_start = - 1
    message_end = - 1
    # Technically , type _ id and message can appear in any order , so we need
    # a little loop here .
    while 1 :
        ( tag_bytes , pos ) = local_ReadTag ( buffer , pos )
        if tag_bytes == type_id_tag_bytes :
            ( type_id , pos ) = local_DecodeVarint ( buffer , pos )
        elif tag_bytes == message_tag_bytes :
            ( size , message_start ) = local_DecodeVarint ( buffer , pos )
            pos = message_end = message_start + size
        elif tag_bytes == item_end_tag_bytes :
            break
        else :
            pos = SkipField ( buffer , pos , end , tag_bytes )
            if pos == - 1 :
                raise _DecodeError ( 'Missing group end tag.' )
    if pos > end :
        raise _DecodeError ( 'Truncated message.' )
    if type_id == - 1 :
        raise _DecodeError ( 'MessageSet item missing type_id.' )
    if message_start == - 1 :
        raise _DecodeError ( 'MessageSet item missing message.' )
    extension = extensions_by_number . get ( type_id )
    if extension is not None :
        value = field_dict . get ( extension )
        if value is None :
            value = field_dict . setdefault ( extension , extension . message_type . _concrete_class ( ) )
        if value . _InternalParse ( buffer , message_start , message_end ) != message_end : # The only reason _ InternalParse would return early is if it encountered
        # an end - group tag .
            raise _DecodeError ( 'Unexpected end-group tag.' )
    else :
        if not message . _unknown_fields :
            message . _unknown_fields = [ ]
        message . _unknown_fields . append ( ( MESSAGE_SET_ITEM_TAG , buffer [ message_set_item_start : pos ] ) )
    return pos
return DecodeItem


signature:
def get_app_name ( ) :
    """Flask like implementation of getting the applicaiton name via
 the filename of the including file"""

implementation:
fn = getattr ( sys . modules [ '__main__' ] , '__file__' , None )
if fn is None :
    return '__main__'
return os . path . splitext ( os . path . basename ( fn ) ) [ 0 ]


signature:
def get_function ( function_name ) :
    """Given a Python function name , return the function it refers to ."""

implementation:
module , basename = str ( function_name ) . rsplit ( '.' , 1 )
try :
    return getattr ( __import__ ( module , fromlist = [ basename ] ) , basename )
except ( ImportError , AttributeError ) :
    raise FunctionNotFound ( function_name )


signature:
def handle_add_fun ( self , function_name ) :
    """Add a function to the function list , in order ."""

implementation:
function_name = function_name . strip ( )
try :
    function = get_function ( function_name )
except Exception , exc :
    self . wfile . write ( js_error ( exc ) + NEWLINE )
    return
# This tests to see if the function has been decorated with the view
# server synchronisation decorator ( ` ` decorate _ view ` ` ) .
if not getattr ( function , 'view_decorated' , None ) :
    self . functions [ function_name ] = ( self . function_counter , function )
# The decorator gets called with the logger function .
else :
    self . functions [ function_name ] = ( self . function_counter , function ( self . log ) )
self . function_counter += 1
return True


signature:
def handle_map_doc ( self , document ) :
    """Return the mapping of a document according to the function list ."""

implementation:
# This uses the stored set of functions , sorted by order of addition .
for function in sorted ( self . functions . values ( ) , key = lambda x : x [ 0 ] ) :
    try : # It has to be run through ` ` list ` ` , because it may be a
    # generator function .
        yield [ list ( function ( document ) ) ]
    except Exception , exc : # Otherwise , return an empty list and log the event .
        yield [ ]
        self . log ( repr ( exc ) )


signature:
def handle_reduce ( self , reduce_function_names , mapped_docs ) :
    """Reduce several mapped documents by several reduction functions ."""

implementation:
reduce_functions = [ ]
# This gets a large list of reduction functions , given their names .
for reduce_function_name in reduce_function_names :
    try :
        reduce_function = get_function ( reduce_function_name )
        if getattr ( reduce_function , 'view_decorated' , None ) :
            reduce_function = reduce_function ( self . log )
        reduce_functions . append ( reduce_function )
    except Exception , exc :
        self . log ( repr ( exc ) )
        reduce_functions . append ( lambda * args , ** kwargs : None )
# Transform lots of ( key , value ) pairs into one ( keys , values ) pair .
keys , values = zip ( ( key , value ) for ( ( key , doc_id ) , value ) in mapped_docs )
# This gets the list of results from the reduction functions .
results = [ ]
for reduce_function in reduce_functions :
    try :
        results . append ( reduce_function ( keys , values , rereduce = False ) )
    except Exception , exc :
        self . log ( repr ( exc ) )
        results . append ( None )
return [ True , results ]


signature:
def handle_rereduce ( self , reduce_function_names , values ) :
    """Re - reduce a set of values , with a list of rereduction functions ."""

implementation:
# This gets a large list of reduction functions , given their names .
reduce_functions = [ ]
for reduce_function_name in reduce_function_names :
    try :
        reduce_function = get_function ( reduce_function_name )
        if getattr ( reduce_function , 'view_decorated' , None ) :
            reduce_function = reduce_function ( self . log )
        reduce_functions . append ( reduce_function )
    except Exception , exc :
        self . log ( repr ( exc ) )
        reduce_functions . append ( lambda * args , ** kwargs : None )
# This gets the list of results from those functions .
results = [ ]
for reduce_function in reduce_functions :
    try :
        results . append ( reduce_function ( None , values , rereduce = True ) )
    except Exception , exc :
        self . log ( repr ( exc ) )
        results . append ( None )
return [ True , results ]


signature:
def handle_validate ( self , function_name , new_doc , old_doc , user_ctx ) :
    """Validate . . . this function is undocumented , but still in CouchDB ."""

implementation:
try :
    function = get_function ( function_name )
except Exception , exc :
    self . log ( repr ( exc ) )
    return False
try :
    return function ( new_doc , old_doc , user_ctx )
except Exception , exc :
    self . log ( repr ( exc ) )
    return repr ( exc )


signature:
def handle ( self ) :
    """The main function called to handle a request ."""

implementation:
while True :
    try :
        line = self . rfile . readline ( )
        try : # All input data are lines of JSON like the following :
        # [ " < cmd _ name > " " < cmd _ arg1 > " " < cmd _ arg2 > " . . . ]
        # So I handle this by dispatching to various methods .
            cmd = json . loads ( line )
        except Exception , exc : # Sometimes errors come up . Once again , I can ' t predict
        # anything , but can at least tell CouchDB about the error .
            self . wfile . write ( repr ( exc ) + NEWLINE )
            continue
        else : # Automagically get the command handler .
            handler = getattr ( self , 'handle_' + cmd [ 0 ] , None )
            if not handler : # We are ready to not find commands . It probably won ' t
            # happen , but fortune favours the prepared .
                self . wfile . write ( repr ( CommandNotFound ( cmd [ 0 ] ) ) + NEWLINE )
                continue
            return_value = handler ( * cmd [ 1 : ] )
            if not return_value :
                continue
            # We write the output back to CouchDB .
            self . wfile . write ( one_lineify ( json . dumps ( return_value ) ) + NEWLINE )
    except Exception , exc :
        self . wfile . write ( repr ( exc ) + NEWLINE )
        continue


signature:
def log ( self , string ) :
    """Log an event on the CouchDB server ."""

implementation: self . wfile . write ( json . dumps ( { 'log' : string } ) + NEWLINE )


signature:
def guid ( * args ) :
    """Generates a universally unique ID .
 Any arguments only create more randomness ."""

implementation:
t = float ( time . time ( ) * 1000 )
r = float ( random . random ( ) * 10000000000000 )
a = random . random ( ) * 10000000000000
data = str ( t ) + ' ' + str ( r ) + ' ' + str ( a ) + ' ' + str ( args )
data = hashlib . md5 ( data . encode ( ) ) . hexdigest ( ) [ : 10 ]
return data


signature:
def get_pages ( self , limit = 5 , order_by = ( 'position' , '-modified_at' ) ) :
    """Return pages the GitModel knows about .
 : param int limit :
 The number of pages to return , defaults to 5.
 : param tuple order _ by :
 The attributes to order on ,
 defaults to ( ' position ' , ' - modified _ at ' )"""

implementation: return to_eg_objects ( self . workspace . S ( Page ) . filter ( language = self . locale ) . order_by ( * order_by ) [ : limit ] )


signature:
def get_featured_pages ( self , limit = 5 , order_by = ( 'position' , '-modified_at' ) ) :
    """Return featured pages the GitModel knows about .
 : param str locale :
 The locale string , like ` eng _ UK ` .
 : param int limit :
 The number of pages to return , defaults to 5.
 : param tuple order _ by :
 The attributes to order on ,
 defaults to ( ' position ' , ' - modified _ at ' ) ."""

implementation: return self . _get_featured_pages ( self . locale , limit , order_by )


signature:
def register_app ( self , name , redirect_uri , callback ) :
    '''register _ app takes an application name and redirect _ uri
 It generates client _ id ( client _ key ) and client _ secret ,
 then stores all of the above in the data _ store ,
 and returns a dictionary containing the client _ id and client _ secret .'''

implementation:
client_id = self . _generate_token ( )
client_secret = self . _generate_token ( 64 )
yield Task ( self . data_store . store , 'applications' , client_id = client_id , client_secret = client_secret , name = name , redirect_uri = redirect_uri )
callback ( { 'client_id' : client_id , 'client_secret' : client_secret } )


signature:
def request_authorization ( self , client_id , user_id , response_type , redirect_uri = None , scope = None , state = None , expires = 600 , callback = None ) :
    '''request _ authorization generates a nonce , and stores it in the data _ store along with the
 client _ id , user _ id , and expiration timestamp .
 It then returns a dictionary containing the nonce as " code , " and the passed
 state .
 response _ type MUST be " code . " this is directly from the OAuth2 spec .
 this probably doesn ' t need to be checked here , but if it ' s in the spec I
 guess it should be verified somewhere .
 scope has not been implemented here . it will be stored , but there is no
 scope - checking built in here at this time .
 if a redirect _ uri is passed , it must match the registered redirect _ uri .
 again , this is per spec .'''

implementation:
if response_type != 'code' :
    raise Proauth2Error ( 'invalid_request' , 'response_type must be "code"' , state = state )
client = yield Task ( self . data_store . fetch , 'applications' , client_id = client_id )
if not client :
    raise Proauth2Error ( 'access_denied' )
if redirect_uri and client [ 'redirect_uri' ] != redirect_uri :
    raise Proauth2Error ( 'invalid_request' , "redirect_uris don't match" )
nonce_code = self . _generate_token ( )
expires = time ( ) + expires
try :
    yield Task ( self . data_store . store , 'nonce_codes' , code = nonce_code , client_id = client_id , expires = expires , user_id = user_id , scope = scope )
except Proauth2Error as e :
    e . state = state
    raise e
callback ( { 'code' : nonce_code , 'state' : state } )


signature:
def request_access_token ( self , client_id , key , code , grant_type , redirect_uri = None , method = 'direct_auth' , callback = None ) :
    '''request _ access _ token validates the client _ id and client _ secret , using the
 provided method , then generates an access _ token , stores it with the user _ id
 from the nonce , and returns a dictionary containing an access _ token and
 bearer token .
 from the spec , it looks like there are different types of
 tokens , but i don ' t understand the disctintions , so someone else can fix
 this if need be .
 regarding the method : it appears that it is intended for there to be
 multiple ways to verify the client _ id . my assumption is that you use the
 secret as the salt and pass the hashed of the client _ id or something , and
 then compare hashes on the server end . currently the only implemented method
 is direct comparison of the client _ ids and client _ secrets .
 additional methods can be added to proauth2 . auth _ methods'''

implementation:
if grant_type != 'authorization_code' :
    raise Proauth2Error ( 'invalid_request' , 'grant_type must be "authorization_code"' )
yield Task ( self . _auth , client_id , key , method )
user_id = yield Task ( self . _validate_request_code , code , client_id )
access_token = self . _generate_token ( 64 )
yield Task ( self . data_store . store , 'tokens' , token = access_token , user_id = user_id , client_id = client_id )
callback ( { 'access_token' : access_token , 'token_type' : 'bearer' } )


signature:
def authenticate_token ( self , token , callback ) :
    '''authenticate _ token checks the passed token and returns the user _ id it is
 associated with . it is assumed that this method won ' t be directly exposed to
 the oauth client , but some kind of framework or wrapper . this allows the
 framework to have the user _ id without doing additional DB calls .'''

implementation:
token_data = yield Task ( self . data_store . fetch , 'tokens' , token = token )
if not token_data :
    raise Proauth2Error ( 'access_denied' , 'token does not exist or has been revoked' )
callback ( token_data [ 'user_id' ] )


signature:
def revoke_token ( self , token , callback ) :
    '''revoke _ token removes the access token from the data _ store'''

implementation:
yield Task ( self . data_store . remove , 'tokens' , token = token )
callback ( )


signature:
def _auth ( self , client_id , key , method , callback ) :
    '''_ auth - internal method to ensure the client _ id and client _ secret passed with
 the nonce match'''

implementation:
available = auth_methods . keys ( )
if method not in available :
    raise Proauth2Error ( 'invalid_request' , 'unsupported authentication method: %s' 'available methods: %s' % ( method , '\n' . join ( available ) ) )
client = yield Task ( self . data_store . fetch , 'applications' , client_id = client_id )
if not client :
    raise Proauth2Error ( 'access_denied' )
if not auth_methods [ method ] ( key , client [ 'client_secret' ] ) :
    raise Proauth2Error ( 'access_denied' )
callback ( )


signature:
def _validate_request_code ( self , code , client_id , callback ) :
    '''_ validate _ request _ code - internal method for verifying the the given nonce .
 also removes the nonce from the data _ store , as they are intended for
 one - time use .'''

implementation:
nonce = yield Task ( self . data_store . fetch , 'nonce_codes' , code = code )
if not nonce :
    raise Proauth2Error ( 'access_denied' , 'invalid request code: %s' % code )
if client_id != nonce [ 'client_id' ] :
    raise Proauth2Error ( 'access_denied' , 'invalid request code: %s' % code )
user_id = nonce [ 'user_id' ]
expires = nonce [ 'expires' ]
yield Task ( self . data_store . remove , 'nonce_codes' , code = code , client_id = client_id , user_id = user_id )
if time ( ) > expires :
    raise Proauth2Error ( 'access_denied' , 'request code %s expired' % code )
callback ( user_id )


signature:
def _generate_token ( self , length = 32 ) :
    '''_ generate _ token - internal function for generating randomized alphanumberic
 strings of a given length'''

implementation: return '' . join ( choice ( ascii_letters + digits ) for x in range ( length ) )


signature:
def merge_ordered ( ordereds : typing . Iterable [ typing . Any ] ) -> typing . Iterable [ typing . Any ] :
    """Merge multiple ordered so that within - ordered order is preserved"""

implementation:
seen_set = set ( )
add_seen = seen_set . add
return reversed ( tuple ( map ( lambda obj : add_seen ( obj ) or obj , filterfalse ( seen_set . __contains__ , chain . from_iterable ( map ( reversed , reversed ( ordereds ) ) ) , ) , ) ) )


signature:
def validate_params ( required , optional , params ) :
    """Helps us validate the parameters for the request
 : param valid _ options : a list of strings of valid options for the
 api request
 : param params : a dict , the key - value store which we really only care about
 the key which has tells us what the user is using for the
 API request
 : returns : None or throws an exception if the validation fails"""

implementation:
missing_fields = [ x for x in required if x not in params ]
if missing_fields :
    field_strings = ", " . join ( missing_fields )
    raise Exception ( "Missing fields: %s" % field_strings )
disallowed_fields = [ x for x in params if x not in optional and x not in required ]
if disallowed_fields :
    field_strings = ", " . join ( disallowed_fields )
    raise Exception ( "Disallowed fields: %s" % field_strings )


signature:
def authenticate_token ( self , token ) :
    '''authenticate _ token checks the passed token and returns the user _ id it is
 associated with . it is assumed that this method won ' t be directly exposed to
 the oauth client , but some kind of framework or wrapper . this allows the
 framework to have the user _ id without doing additional DB calls .'''

implementation:
token_data = self . data_store . fetch ( 'tokens' , token = token )
if not token_data :
    raise Proauth2Error ( 'access_denied' , 'token does not exist or has been revoked' )
return token_data [ 'user_id' ]


signature:
def main ( ) :
    """Register your own mode and handle method here ."""

implementation:
plugin = Register ( )
if plugin . args . option == 'filenumber' :
    plugin . filenumber_handle ( )
elif plugin . args . option == 'fileage' :
    plugin . fileage_handle ( )
elif plugin . args . option == 'sqlserverlocks' :
    plugin . sqlserverlocks_handle ( )
else :
    plugin . unknown ( "Unknown actions." )


signature:
def filenumber_handle ( self ) :
    """Get the number of file in the folder ."""

implementation:
self . file_list = [ ]
self . count = 0
status = self . ok
if self . args . recursion :
    self . __result , self . __file_list = self . __get_folder ( self . args . path )
else :
    self . __result , self . __file_list = self . __get_file ( self . args . path )
# Compare the vlaue .
if self . __result > self . args . critical :
    status = self . critical
elif self . __result > self . args . warning :
    status = self . warning
else :
    status = self . ok
# Output
self . shortoutput = "Found {0} files in {1}." . format ( self . __result , self . args . path )
self . logger . debug ( "file_list: {}" . format ( self . __file_list ) )
[ self . longoutput . append ( file_data . get ( 'Name' ) ) for file_data in self . __file_list ]
self . perfdata . append ( "{path}={result};{warn};{crit};0;" . format ( crit = self . args . critical , warn = self . args . warning , result = self . __result , path = self . args . path ) )
# Return status with message to Nagios .
status ( self . output ( long_output_limit = None ) )
self . logger . debug ( "Return status and exit to Nagios." )


signature:
def __get_current_datetime ( self ) :
    """Get current datetime for every file ."""

implementation:
self . wql_time = "SELECT LocalDateTime FROM Win32_OperatingSystem"
self . current_time = self . query ( self . wql_time )
# [ { ' LocalDateTime ' : ' 20160824161431.977000 + 480 ' } ] '
self . current_time_string = str ( self . current_time [ 0 ] . get ( 'LocalDateTime' ) . split ( '.' ) [ 0 ] )
# '20160824161431'
self . current_time_format = datetime . datetime . strptime ( self . current_time_string , '%Y%m%d%H%M%S' )
# param : datetime . datetime ( 2016 , 8 , 24 , 16 , 14 , 31 ) - > type :
# datetime . datetime
return self . current_time_format


signature:
def fileage_handle ( self ) :
    """Get the number of file in the folder ."""

implementation:
self . file_list = [ ]
self . ok_file = [ ]
self . warn_file = [ ]
self . crit_file = [ ]
status = self . ok
if self . args . recursion :
    self . __file_list = self . __get_folder ( self . args . path )
else :
    self . __file_list = self . __get_file ( self . args . path )
self . logger . debug ( "file_list: {}" . format ( self . __file_list ) )
# [ { ' LastModified ' : ' 20160824142017.737101 + 480 ' , ' Name ' : ' d : \ \ test \ \ 1 . txt ' } ,
# { ' LastModified ' : ' 20160824142021.392101 + 480 ' , ' Name ' : ' d : \ \ test \ \ 2 . txt ' } ,
# { ' LastModified ' : ' 20160824142106.460101 + 480 ' , ' Name ' : ' d : \ \ test \ \ test1 \ \ 21 . txt ' } ]
for file_dict in self . __file_list :
    self . filename = file_dict . get ( 'Name' )
    if self . filename and self . filename != 'Name' :
        self . logger . debug ( "===== start to compare {} =====" . format ( self . filename ) )
        self . file_datetime_string = file_dict . get ( 'LastModified' ) . split ( '.' ) [ 0 ]
        self . file_datetime = datetime . datetime . strptime ( self . file_datetime_string , '%Y%m%d%H%M%S' )
        self . logger . debug ( "file_datetime: {}" . format ( self . file_datetime ) )
        self . current_datetime = self . __get_current_datetime ( )
        self . logger . debug ( "current_datetime: {}" . format ( self . current_datetime ) )
        self . __delta_datetime = self . current_datetime - self . file_datetime
        self . logger . debug ( "delta_datetime: {}" . format ( self . __delta_datetime ) )
        self . logger . debug ( "warn_datetime: {}" . format ( datetime . timedelta ( minutes = self . args . warning ) ) )
        self . logger . debug ( "crit_datetime: {}" . format ( datetime . timedelta ( minutes = self . args . critical ) ) )
        if self . __delta_datetime > datetime . timedelta ( minutes = self . args . critical ) :
            self . crit_file . append ( self . filename )
        elif self . __delta_datetime > datetime . timedelta ( minutes = self . args . warning ) :
            self . warn_file . append ( self . filename )
        else :
            self . ok_file . append ( self . filename )
# Compare the vlaue .
if self . crit_file :
    status = self . critical
elif self . warn_file :
    status = self . warning
else :
    status = self . ok
# Output
self . shortoutput = "Found {0} files out of date." . format ( len ( self . crit_file ) )
if self . crit_file :
    self . longoutput . append ( "===== Critical File out of date ====" )
[ self . longoutput . append ( filename ) for filename in self . crit_file if self . crit_file ]
if self . warn_file :
    self . longoutput . append ( "===== Warning File out of date ====" )
[ self . longoutput . append ( filename ) for filename in self . warn_file if self . warn_file ]
if self . ok_file :
    self . longoutput . append ( "===== OK File out of date ====" )
[ self . longoutput . append ( filename ) for filename in self . ok_file if self . ok_file ]
self . perfdata . append ( "{path}={result};{warn};{crit};0;" . format ( crit = self . args . critical , warn = self . args . warning , result = len ( self . crit_file ) , path = self . args . drive + self . args . path ) )
# Return status with message to Nagios .
status ( self . output ( long_output_limit = None ) )
self . logger . debug ( "Return status and exit to Nagios." )


signature:
def run ( self ) :
    """run your main spider here
 as for branch spider result data , you can return everything or do whatever with it
 in your own code
 : return : None"""

implementation:
config = config_creator ( )
debug = config . debug
branch_thread_sleep = config . branch_thread_sleep
while 1 :
    url = self . branch_queue . get ( )
    if debug :
        print ( 'branch thread-{} start' . format ( url ) )
    branch_spider = self . branch_spider ( url )
    sleep ( random . randrange ( * branch_thread_sleep ) )
    branch_spider . request_page ( )
    if debug :
        print ( 'branch thread-{} end' . format ( url ) )
    self . branch_queue . task_done ( )


signature:
def get_version ( relpath ) :
    """Read version info from a file without importing it"""

implementation:
from os . path import dirname , join
if '__file__' not in globals ( ) : # Allow to use function interactively
    root = '.'
else :
    root = dirname ( __file__ )
# The code below reads text file with unknown encoding in
# in Python2/3 compatible way . Reading this text file
# without specifying encoding will fail in Python 3 on some
# systems ( see http : / / goo . gl / 5XmOH ) . Specifying encoding as
# open ( ) parameter is incompatible with Python 2
# cp437 is the encoding without missing points , safe against :
# UnicodeDecodeError : ' charmap ' codec can ' t decode byte . . .
for line in open ( join ( root , relpath ) , 'rb' ) :
    line = line . decode ( 'cp437' )
    if '__version__' in line :
        if '"' in line : # _ _ version _ _ = " 0.9"
            return line . split ( '"' ) [ 1 ]
        elif "'" in line :
            return line . split ( "'" ) [ 1 ]


signature:
def MakeDescriptor ( desc_proto , package = '' , build_file_if_cpp = True , syntax = None ) :
    """Make a protobuf Descriptor given a DescriptorProto protobuf .
 Handles nested descriptors . Note that this is limited to the scope of defining
 a message inside of another message . Composite fields can currently only be
 resolved if the message is defined in the same scope as the field .
 Args :
 desc _ proto : The descriptor _ pb2 . DescriptorProto protobuf message .
 package : Optional package name for the new message Descriptor ( string ) .
 build _ file _ if _ cpp : Update the C + + descriptor pool if api matches .
 Set to False on recursion , so no duplicates are created .
 syntax : The syntax / semantics that should be used . Set to " proto3 " to get
 proto3 field presence semantics .
 Returns :
 A Descriptor for protobuf messages ."""

implementation:
if api_implementation . Type ( ) == 'cpp' and build_file_if_cpp : # The C + + implementation requires all descriptors to be backed by the same
# definition in the C + + descriptor pool . To do this , we build a
# FileDescriptorProto with the same definition as this descriptor and build
# it into the pool .
    from typy . google . protobuf import descriptor_pb2
    file_descriptor_proto = descriptor_pb2 . FileDescriptorProto ( )
    file_descriptor_proto . message_type . add ( ) . MergeFrom ( desc_proto )
    # Generate a random name for this proto file to prevent conflicts with any
    # imported ones . We need to specify a file name so the descriptor pool
    # accepts our FileDescriptorProto , but it is not important what that file
    # name is actually set to .
    proto_name = str ( uuid . uuid4 ( ) )
    if package :
        file_descriptor_proto . name = os . path . join ( package . replace ( '.' , '/' ) , proto_name + '.proto' )
        file_descriptor_proto . package = package
    else :
        file_descriptor_proto . name = proto_name + '.proto'
    _message . default_pool . Add ( file_descriptor_proto )
    result = _message . default_pool . FindFileByName ( file_descriptor_proto . name )
    if _USE_C_DESCRIPTORS :
        return result . message_types_by_name [ desc_proto . name ]
full_message_name = [ desc_proto . name ]
if package :
    full_message_name . insert ( 0 , package )
# Create Descriptors for enum types
enum_types = { }
for enum_proto in desc_proto . enum_type :
    full_name = '.' . join ( full_message_name + [ enum_proto . name ] )
    enum_desc = EnumDescriptor ( enum_proto . name , full_name , None , [ EnumValueDescriptor ( enum_val . name , ii , enum_val . number ) for ii , enum_val in enumerate ( enum_proto . value ) ] )
    enum_types [ full_name ] = enum_desc
# Create Descriptors for nested types
nested_types = { }
for nested_proto in desc_proto . nested_type :
    full_name = '.' . join ( full_message_name + [ nested_proto . name ] )
    # Nested types are just those defined inside of the message , not all types
    # used by fields in the message , so no loops are possible here .
    nested_desc = MakeDescriptor ( nested_proto , package = '.' . join ( full_message_name ) , build_file_if_cpp = False , syntax = syntax )
    nested_types [ full_name ] = nested_desc
fields = [ ]
for field_proto in desc_proto . field :
    full_name = '.' . join ( full_message_name + [ field_proto . name ] )
    enum_desc = None
    nested_desc = None
    if field_proto . HasField ( 'type_name' ) :
        type_name = field_proto . type_name
        full_type_name = '.' . join ( full_message_name + [ type_name [ type_name . rfind ( '.' ) + 1 : ] ] )
        if full_type_name in nested_types :
            nested_desc = nested_types [ full_type_name ]
        elif full_type_name in enum_types :
            enum_desc = enum_types [ full_type_name ]
        # Else type _ name references a non - local type , which isn ' t implemented
    field = FieldDescriptor ( field_proto . name , full_name , field_proto . number - 1 , field_proto . number , field_proto . type , FieldDescriptor . ProtoTypeToCppProtoType ( field_proto . type ) , field_proto . label , None , nested_desc , enum_desc , None , False , None , options = field_proto . options , has_default_value = False )
    fields . append ( field )
desc_name = '.' . join ( full_message_name )
return Descriptor ( desc_proto . name , desc_name , None , None , fields , list ( nested_types . values ( ) ) , list ( enum_types . values ( ) ) , [ ] , options = desc_proto . options )


signature:
def GetTopLevelContainingType ( self ) :
    """Returns the root if this is a nested type , or itself if its the root ."""

implementation:
desc = self
while desc . containing_type is not None :
    desc = desc . containing_type
return desc


signature:
def FindMethodByName ( self , name ) :
    """Searches for the specified method , and returns its descriptor ."""

implementation:
for method in self . methods :
    if name == method . name :
        return method
return None


implementation:
plugin = Register ( )
if plugin . args . option == 'sqlserverlocks' :
    plugin . sqlserverlocks_handle ( )
else :
    plugin . unknown ( "Unknown actions." )


signature:
def MessageToJson ( message , including_default_value_fields = False ) :
    """Converts protobuf message to JSON format .
 Args :
 message : The protocol buffers message instance to serialize .
 including _ default _ value _ fields : If True , singular primitive fields ,
 repeated fields , and map fields will always be serialized . If
 False , only serialize non - empty fields . Singular message fields
 and oneof fields are not affected by this option .
 Returns :
 A string containing the JSON formatted protocol buffer message ."""

implementation:
js = _MessageToJsonObject ( message , including_default_value_fields )
return json . dumps ( js , indent = 2 )


signature:
def _MessageToJsonObject ( message , including_default_value_fields ) :
    """Converts message to an object according to Proto3 JSON Specification ."""

implementation:
message_descriptor = message . DESCRIPTOR
full_name = message_descriptor . full_name
if _IsWrapperMessage ( message_descriptor ) :
    return _WrapperMessageToJsonObject ( message )
if full_name in _WKTJSONMETHODS :
    return _WKTJSONMETHODS [ full_name ] [ 0 ] ( message , including_default_value_fields )
js = { }
return _RegularMessageToJsonObject ( message , js , including_default_value_fields )


signature:
def _StructMessageToJsonObject ( message , unused_including_default = False ) :
    """Converts Struct message according to Proto3 JSON Specification ."""

implementation:
fields = message . fields
ret = { }
for key in fields :
    ret [ key ] = _ValueMessageToJsonObject ( fields [ key ] )
return ret


signature:
def Parse ( text , message ) :
    """Parses a JSON representation of a protocol message into a message .
 Args :
 text : Message JSON representation .
 message : A protocol beffer message to merge into .
 Returns :
 The same message passed as argument .
 Raises : :
 ParseError : On JSON parsing problems ."""

implementation:
if not isinstance ( text , six . text_type ) :
    text = text . decode ( 'utf-8' )
try :
    if sys . version_info < ( 2 , 7 ) : # object _ pair _ hook is not supported before python2.7
        js = json . loads ( text )
    else :
        js = json . loads ( text , object_pairs_hook = _DuplicateChecker )
except ValueError as e :
    raise ParseError ( 'Failed to load JSON: {0}.' . format ( str ( e ) ) )
_ConvertMessage ( js , message )
return message


signature:
def _ConvertFieldValuePair ( js , message ) :
    """Convert field value pairs into regular message .
 Args :
 js : A JSON object to convert the field value pairs .
 message : A regular protocol message to record the data .
 Raises :
 ParseError : In case of problems converting ."""

implementation:
names = [ ]
message_descriptor = message . DESCRIPTOR
for name in js :
    try :
        field = message_descriptor . fields_by_camelcase_name . get ( name , None )
        if not field :
            raise ParseError ( 'Message type "{0}" has no field named "{1}".' . format ( message_descriptor . full_name , name ) )
        if name in names :
            raise ParseError ( 'Message type "{0}" should not have multiple "{1}" fields.' . format ( message . DESCRIPTOR . full_name , name ) )
        names . append ( name )
        # Check no other oneof field is parsed .
        if field . containing_oneof is not None :
            oneof_name = field . containing_oneof . name
            if oneof_name in names :
                raise ParseError ( 'Message type "{0}" should not have multiple "{1}" ' 'oneof fields.' . format ( message . DESCRIPTOR . full_name , oneof_name ) )
            names . append ( oneof_name )
        value = js [ name ]
        if value is None :
            message . ClearField ( field . name )
            continue
        # Parse field value .
        if _IsMapEntry ( field ) :
            message . ClearField ( field . name )
            _ConvertMapFieldValue ( value , message , field )
        elif field . label == descriptor . FieldDescriptor . LABEL_REPEATED :
            message . ClearField ( field . name )
            if not isinstance ( value , list ) :
                raise ParseError ( 'repeated field {0} must be in [] which is ' '{1}.' . format ( name , value ) )
            if field . cpp_type == descriptor . FieldDescriptor . CPPTYPE_MESSAGE : # Repeated message field .
                for item in value :
                    sub_message = getattr ( message , field . name ) . add ( )
                    # None is a null _ value in Value .
                    if ( item is None and sub_message . DESCRIPTOR . full_name != 'google.protobuf.Value' ) :
                        raise ParseError ( 'null is not allowed to be used as an element' ' in a repeated field.' )
                    _ConvertMessage ( item , sub_message )
            else : # Repeated scalar field .
                for item in value :
                    if item is None :
                        raise ParseError ( 'null is not allowed to be used as an element' ' in a repeated field.' )
                    getattr ( message , field . name ) . append ( _ConvertScalarFieldValue ( item , field ) )
        elif field . cpp_type == descriptor . FieldDescriptor . CPPTYPE_MESSAGE :
            sub_message = getattr ( message , field . name )
            _ConvertMessage ( value , sub_message )
        else :
            setattr ( message , field . name , _ConvertScalarFieldValue ( value , field ) )
    except ParseError as e :
        if field and field . containing_oneof is None :
            raise ParseError ( 'Failed to parse {0} field: {1}' . format ( name , e ) )
        else :
            raise ParseError ( str ( e ) )
    except ValueError as e :
        raise ParseError ( 'Failed to parse {0} field: {1}.' . format ( name , e ) )
    except TypeError as e :
        raise ParseError ( 'Failed to parse {0} field: {1}.' . format ( name , e ) )


signature:
def _ConvertMessage ( value , message ) :
    """Convert a JSON object into a message .
 Args :
 value : A JSON object .
 message : A WKT or regular protocol message to record the data .
 Raises :
 ParseError : In case of convert problems ."""

implementation:
message_descriptor = message . DESCRIPTOR
full_name = message_descriptor . full_name
if _IsWrapperMessage ( message_descriptor ) :
    _ConvertWrapperMessage ( value , message )
elif full_name in _WKTJSONMETHODS :
    _WKTJSONMETHODS [ full_name ] [ 1 ] ( value , message )
else :
    _ConvertFieldValuePair ( value , message )


signature:
def _ConvertValueMessage ( value , message ) :
    """Convert a JSON representation into Value message ."""

implementation:
if isinstance ( value , dict ) :
    _ConvertStructMessage ( value , message . struct_value )
elif isinstance ( value , list ) :
    _ConvertListValueMessage ( value , message . list_value )
elif value is None :
    message . null_value = 0
elif isinstance ( value , bool ) :
    message . bool_value = value
elif isinstance ( value , six . string_types ) :
    message . string_value = value
elif isinstance ( value , _INT_OR_FLOAT ) :
    message . number_value = value
else :
    raise ParseError ( 'Unexpected type for Value message.' )


signature:
def _ConvertListValueMessage ( value , message ) :
    """Convert a JSON representation into ListValue message ."""

implementation:
if not isinstance ( value , list ) :
    raise ParseError ( 'ListValue must be in [] which is {0}.' . format ( value ) )
message . ClearField ( 'values' )
for item in value :
    _ConvertValueMessage ( item , message . values . add ( ) )


signature:
def _ConvertStructMessage ( value , message ) :
    """Convert a JSON representation into Struct message ."""

implementation:
if not isinstance ( value , dict ) :
    raise ParseError ( 'Struct must be in a dict which is {0}.' . format ( value ) )
for key in value :
    _ConvertValueMessage ( value [ key ] , message . fields [ key ] )
return


signature:
def update_config ( new_config ) :
    """Update config options with the provided dictionary of options ."""

implementation:
flask_app . base_config . update ( new_config )
# Check for changed working directory .
if new_config . has_key ( 'working_directory' ) :
    wd = os . path . abspath ( new_config [ 'working_directory' ] )
    if nbmanager . notebook_dir != wd :
        if not os . path . exists ( wd ) :
            raise IOError ( 'Path not found: %s' % wd )
        nbmanager . notebook_dir = wd


signature:
def set_config ( new_config = { } ) :
    """Reset config options to defaults , and then update ( optionally )
 with the provided dictionary of options ."""

implementation:
# The default base configuration .
flask_app . base_config = dict ( working_directory = '.' , template = 'collapse-input' , debug = False , port = None )
update_config ( new_config )


signature:
def execute ( self , correlation_id , args ) :
    """Executes the command given specific arguments as an input .
 Args :
 correlation _ id : a unique correlation / transaction id
 args : command arguments
 Returns : an execution result .
 Raises :
 ApplicationException : when execution fails for whatever reason ."""

implementation:
# Validate arguments
if self . _schema != None :
    self . validate_and_throw_exception ( correlation_id , args )
# Call the function
try :
    return self . _function ( correlation_id , args )
# Intercept unhandled errors
except Exception as ex :
    raise InvocationException ( correlation_id , "EXEC_FAILED" , "Execution " + self . _name + " failed: " + str ( ex ) ) . with_details ( "command" , self . _name ) . wrap ( ex )


signature:
def optimize ( function , x0 , cons = [ ] , ftol = 0.2 , disp = 0 , plot = False ) :
    """* * Optimization method based on Brent ' s method * *
 First , a bracket ( a b c ) is sought that contains the minimum ( b value is
 smaller than both a or c ) .
 The bracket is then recursively halfed . Here we apply some modifications
 to ensure our suggested point is not too close to either a or c ,
 because that could be problematic with the local approximation .
 Also , if the bracket does not seem to include the minimum ,
 it is expanded generously in the right direction until it covers it .
 Thus , this function is fail safe , and will always find a local minimum ."""

implementation:
if disp > 0 :
    print
    print '  ===== custom 1d optimization routine ==== '
    print
    print 'initial suggestion on' , function , ':' , x0
points = [ ]
values = [ ]
def recordfunction ( x ) :
    v = function ( x )
    points . append ( x )
    values . append ( v )
    return v
( a , b , c ) , ( va , vb , vc ) = seek_minimum_bracket ( recordfunction , x0 , cons = cons , ftol = ftol , disp = disp , plot = plot )
if disp > 0 :
    print '---------------------------------------------------'
    print 'found useable minimum bracker after %d evaluations:' % len ( points ) , ( a , b , c ) , ( va , vb , vc )
if disp > 2 :
    if plot :
        plot_values ( values , points , lastpoint = - 1 , ftol = ftol )
    pause ( )
result = brent ( recordfunction , a , b , c , va , vb , vc , cons = cons , ftol = ftol , disp = disp , plot = plot )
if disp > 0 :
    print '---------------------------------------------------'
    print 'found minimum after %d evaluations:' % len ( points ) , result
if disp > 1 or len ( points ) > 20 :
    if plot :
        plot_values ( values , points , lastpoint = - 1 , ftol = ftol )
    if disp > 2 :
        pause ( )
if disp > 0 :
    print '---------------------------------------------------'
    print
    print '  ===== end of custom 1d optimization routine ==== '
    print
global neval
neval += len ( points )
return result


signature:
def cache2errors ( function , cache , disp = 0 , ftol = 0.05 ) :
    """This function will attempt to identify 1 sigma errors , assuming your
 function is a chi ^ 2 . For this , the 1 - sigma is bracketed .
 If you were smart enough to build a cache list of [ x , y ] into your function ,
 you can pass it here . The values bracketing 1 sigma will be used as
 starting values .
 If no such values exist , e . g . because all values were very close to the
 optimum ( good starting values ) , the bracket is expanded ."""

implementation:
vals = numpy . array ( sorted ( cache , key = lambda x : x [ 0 ] ) )
if disp > 0 :
    print ' --- cache2errors --- ' , vals
vi = vals [ : , 1 ] . min ( )
def renormedfunc ( x ) :
    y = function ( x )
    cache . append ( [ x , y ] )
    if disp > 1 :
        print '    renormed:' , x , y , y - ( vi + 1 )
    return y - ( vi + 1 )
vals [ : , 1 ] -= vi + 1
lowmask = vals [ : , 1 ] < 0
highmask = vals [ : , 1 ] > 0
indices = numpy . arange ( len ( vals ) )
b , vb = vals [ indices [ lowmask ] [ 0 ] , : ]
c , vc = vals [ indices [ lowmask ] [ - 1 ] , : ]
if any ( vals [ : , 0 ] [ highmask ] < b ) :
    if disp > 0 :
        print 'already have bracket'
    a , va = vals [ indices [ highmask ] [ vals [ : , 0 ] [ highmask ] < b ] [ - 1 ] , : ]
else :
    a = b
    va = vb
    while b > - 50 :
        a = b - max ( vals [ - 1 , 0 ] - vals [ 0 , 0 ] , 1 )
        va = renormedfunc ( a )
        if disp > 0 :
            print 'going further left: %.1f [%.1f] --> %.1f [%.1f]' % ( b , vb , a , va )
        if va > 0 :
            if disp > 0 :
                print 'found outer part'
            break
        else : # need to go further
            b = a
            vb = va
if disp > 0 :
    print 'left  bracket' , a , b , va , vb
if va > 0 and vb < 0 :
    leftroot = scipy . optimize . brentq ( renormedfunc , a , b , rtol = ftol )
else :
    if disp > 0 :
        print 'WARNING: border problem found.'
    leftroot = a
if disp > 0 :
    print 'left  root' , leftroot
if any ( vals [ : , 0 ] [ highmask ] > c ) :
    if disp > 0 :
        print 'already have bracket'
    d , vd = vals [ indices [ highmask ] [ vals [ : , 0 ] [ highmask ] > c ] [ 0 ] , : ]
else :
    d = c
    vd = vc
    while c < 50 :
        d = c + max ( vals [ - 1 , 0 ] - vals [ 0 , 0 ] , 1 )
        vd = renormedfunc ( d )
        if disp > 0 :
            print 'going further right: %.1f [%.1f] --> %.1f [%.1f]' % ( c , vc , d , vd )
        if vd > 0 :
            if disp > 0 :
                print 'found outer part'
            break
        else : # need to go further
            c = d
            vc = vd
if disp > 0 :
    print 'right bracket' , c , d , vc , vd
if vd > 0 and vc < 0 :
    rightroot = scipy . optimize . brentq ( renormedfunc , c , d , rtol = ftol )
else :
    if disp > 0 :
        print 'WARNING: border problem found.'
    rightroot = d
if disp > 0 :
    print 'right root' , rightroot
assert leftroot < rightroot
if disp > 2 :
    fullvals = numpy . array ( sorted ( cache , key = lambda x : x [ 0 ] ) )
    fullvals [ : , 1 ] -= vi + 1
    plt . figure ( )
    plt . plot ( fullvals [ : , 0 ] , fullvals [ : , 1 ] , 's' )
    plt . plot ( vals [ : , 0 ] , vals [ : , 1 ] , 'o' )
    plt . xlim ( a , d )
    plt . ylim ( min ( va , vb , vc , vd ) , max ( va , vb , vc , vd ) )
    ymin , ymax = plt . ylim ( )
    plt . vlines ( [ leftroot , rightroot ] , ymin , ymax , linestyles = 'dotted' )
    plt . savefig ( 'cache_brent.pdf' )
return leftroot , rightroot


signature:
def end_timing ( self ) :
    """Completes measuring time interval and updates counter ."""

implementation:
if self . _callback != None :
    elapsed = time . clock ( ) * 1000 - self . _start
    self . _callback . end_timing ( self . _counter , elapsed )