signature:
def run ( app = None , server = 'wsgiref' , host = '127.0.0.1' , port = 8080 , interval = 1 , reloader = False , quiet = False , plugins = None , debug = None , ** kargs ) :
    """Start a server instance . This method blocks until the server terminates .
 : param app : WSGI application or target string supported by
 : func : ` load _ app ` . ( default : : func : ` default _ app ` )
 : param server : Server adapter to use . See : data : ` server _ names ` keys
 for valid names or pass a : class : ` ServerAdapter ` subclass .
 ( default : ` wsgiref ` )
 : param host : Server address to bind to . Pass ` ` 0.0.0.0 ` ` to listens on
 all interfaces including the external one . ( default : 127.0.0.1)
 : param port : Server port to bind to . Values below 1024 require root
 privileges . ( default : 8080)
 : param reloader : Start auto - reloading server ? ( default : False )
 : param interval : Auto - reloader interval in seconds ( default : 1)
 : param quiet : Suppress output to stdout and stderr ? ( default : False )
 : param options : Options passed to the server adapter ."""

implementation:
if NORUN :
    return
if reloader and not os . environ . get ( 'BOTTLE_CHILD' ) :
    import subprocess
    lockfile = None
    try :
        fd , lockfile = tempfile . mkstemp ( prefix = 'bottle.' , suffix = '.lock' )
        os . close ( fd )
        # We only need this file to exist . We never write to it
        while os . path . exists ( lockfile ) :
            args = [ sys . executable ] + sys . argv
            environ = os . environ . copy ( )
            environ [ 'BOTTLE_CHILD' ] = 'true'
            environ [ 'BOTTLE_LOCKFILE' ] = lockfile
            p = subprocess . Popen ( args , env = environ )
            while p . poll ( ) is None : # Busy wait . . .
                os . utime ( lockfile , None )
                # I am alive !
                time . sleep ( interval )
            if p . poll ( ) != 3 :
                if os . path . exists ( lockfile ) :
                    os . unlink ( lockfile )
                sys . exit ( p . poll ( ) )
    except KeyboardInterrupt :
        pass
    finally :
        if os . path . exists ( lockfile ) :
            os . unlink ( lockfile )
    return
try :
    if debug is not None :
        _debug ( debug )
    app = app or default_app ( )
    if isinstance ( app , basestring ) :
        app = load_app ( app )
    if not callable ( app ) :
        raise ValueError ( "Application is not callable: %r" % app )
    for plugin in plugins or [ ] :
        if isinstance ( plugin , basestring ) :
            plugin = load ( plugin )
        app . install ( plugin )
    if server in server_names :
        server = server_names . get ( server )
    if isinstance ( server , basestring ) :
        server = load ( server )
    if isinstance ( server , type ) :
        server = server ( host = host , port = port , ** kargs )
    if not isinstance ( server , ServerAdapter ) :
        raise ValueError ( "Unknown or unsupported server: %r" % server )
    server . quiet = server . quiet or quiet
    if not server . quiet :
        _stderr ( "Bottle v%s server starting up (using %s)...\n" % ( __version__ , repr ( server ) ) )
        _stderr ( "Listening on http://%s:%d/\n" % ( server . host , server . port ) )
        _stderr ( "Hit Ctrl-C to quit.\n\n" )
    if reloader :
        lockfile = os . environ . get ( 'BOTTLE_LOCKFILE' )
        bgcheck = FileCheckerThread ( lockfile , interval )
        with bgcheck :
            server . run ( app )
        if bgcheck . status == 'reload' :
            sys . exit ( 3 )
    else :
        server . run ( app )
except KeyboardInterrupt :
    pass
except ( SystemExit , MemoryError ) :
    raise
except :
    if not reloader :
        raise
    if not getattr ( server , 'quiet' , quiet ) :
        print_exc ( )
    time . sleep ( interval )
    sys . exit ( 3 )


signature:
def load_dict ( self , source , namespace = '' ) :
    """Load values from a dictionary structure . Nesting can be used to
 represent namespaces .
 > > > c = ConfigDict ( )
 > > > c . load _ dict ( { ' some ' : { ' namespace ' : { ' key ' : ' value ' } } } )
 { ' some . namespace . key ' : ' value ' }"""

implementation:
for key , value in source . items ( ) :
    if isinstance ( key , str ) :
        nskey = ( namespace + '.' + key ) . strip ( '.' )
        if isinstance ( value , dict ) :
            self . load_dict ( value , namespace = nskey )
        else :
            self [ nskey ] = value
    else :
        raise TypeError ( 'Key has type %r (not a string)' % type ( key ) )
return self


signature:
def json ( request , * args , ** kwargs ) :
    """The oembed endpoint , or the url to which requests for metadata are passed .
 Third parties will want to access this view with URLs for your site ' s
 content and be returned OEmbed metadata ."""

implementation:
# coerce to dictionary
params = dict ( request . GET . items ( ) )
callback = params . pop ( 'callback' , None )
url = params . pop ( 'url' , None )
if not url :
    return HttpResponseBadRequest ( 'Required parameter missing: URL' )
try :
    provider = oembed . site . provider_for_url ( url )
    if not provider . provides :
        raise OEmbedMissingEndpoint ( )
except OEmbedMissingEndpoint :
    raise Http404 ( 'No provider found for %s' % url )
query = dict ( [ ( smart_str ( k ) , smart_str ( v ) ) for k , v in params . items ( ) if v ] )
try :
    resource = oembed . site . embed ( url , ** query )
except OEmbedException , e :
    raise Http404 ( 'Error embedding %s: %s' % ( url , str ( e ) ) )
response = HttpResponse ( mimetype = 'application/json' )
json = resource . json
if callback :
    response . write ( '%s(%s)' % ( defaultfilters . force_escape ( callback ) , json ) )
else :
    response . write ( json )
return response


signature:
def consume_json ( request ) :
    """Extract and return oembed content for given urls .
 Required GET params :
 urls - list of urls to consume
 Optional GET params :
 width - maxwidth attribute for oembed content
 height - maxheight attribute for oembed content
 template _ dir - template _ dir to use when rendering oembed
 Returns :
 list of dictionaries with oembed metadata and renderings , json encoded"""

implementation:
client = OEmbedConsumer ( )
urls = request . GET . getlist ( 'urls' )
width = request . GET . get ( 'width' )
height = request . GET . get ( 'height' )
template_dir = request . GET . get ( 'template_dir' )
output = { }
ctx = RequestContext ( request )
for url in urls :
    try :
        provider = oembed . site . provider_for_url ( url )
    except OEmbedMissingEndpoint :
        oembeds = None
        rendered = None
    else :
        oembeds = url
        rendered = client . parse_text ( url , width , height , context = ctx , template_dir = template_dir )
    output [ url ] = { 'oembeds' : oembeds , 'rendered' : rendered , }
return HttpResponse ( simplejson . dumps ( output ) , mimetype = 'application/json' )


signature:
def oembed_schema ( request ) :
    """A site profile detailing valid endpoints for a given domain . Allows for
 better auto - discovery of embeddable content .
 OEmbed - able content lives at a URL that maps to a provider ."""

implementation:
current_domain = Site . objects . get_current ( ) . domain
url_schemes = [ ]
# a list of dictionaries for all the urls we can match
endpoint = reverse ( 'oembed_json' )
# the public endpoint for our oembeds
providers = oembed . site . get_providers ( )
for provider in providers : # first make sure this provider class is exposed at the public endpoint
    if not provider . provides :
        continue
    match = None
    if isinstance ( provider , DjangoProvider ) : # django providers define their regex _ list by using urlreversing
        url_pattern = resolver . reverse_dict . get ( provider . _meta . named_view )
        # this regex replacement is set to be non - greedy , which results
        # in things like / news / * / * / * / * / - - this is more explicit
        if url_pattern :
            regex = re . sub ( r'%\(.+?\)s' , '*' , url_pattern [ 0 ] [ 0 ] [ 0 ] )
            match = 'http://%s/%s' % ( current_domain , regex )
    elif isinstance ( provider , HTTPProvider ) :
        match = provider . url_scheme
    else :
        match = provider . regex
    if match :
        url_schemes . append ( { 'type' : provider . resource_type , 'matches' : match , 'endpoint' : endpoint } )
url_schemes . sort ( key = lambda item : item [ 'matches' ] )
response = HttpResponse ( mimetype = 'application/json' )
response . write ( simplejson . dumps ( url_schemes ) )
return response


signature:
def find_meta ( * meta_file_parts , meta_key ) :
    """Extract _ _ * meta * _ _ from meta _ file ."""

implementation:
meta_file = read ( * meta_file_parts )
meta_match = re . search ( r"^__{}__ = ['\"]([^'\"]*)['\"]" . format ( meta_key ) , meta_file , re . M )
if meta_match :
    return meta_match . group ( 1 )
raise RuntimeError ( "Unable to find __{}__ string." . format ( meta_key ) )


signature:
def main ( path ) :
    '''scan path directory and any subdirectories for valid captain scripts'''

implementation:
basepath = os . path . abspath ( os . path . expanduser ( str ( path ) ) )
echo . h2 ( "Available scripts in {}" . format ( basepath ) )
echo . br ( )
for root_dir , dirs , files in os . walk ( basepath , topdown = True ) :
    for f in fnmatch . filter ( files , '*.py' ) :
        try :
            filepath = os . path . join ( root_dir , f )
            # super edge case , this makes sure the python script won ' t start
            # an interactive console session which would cause the session
            # to start and not allow the for loop to complete
            with open ( filepath , encoding = "UTF-8" ) as fp :
                body = fp . read ( )
                is_console = "InteractiveConsole" in body
                is_console = is_console or "code" in body
                is_console = is_console and "interact(" in body
                if is_console :
                    continue
            s = captain . Script ( filepath )
            if s . can_run_from_cli ( ) :
                rel_filepath = s . call_path ( basepath )
                p = s . parser
                echo . h3 ( rel_filepath )
                desc = p . description
                if desc :
                    echo . indent ( desc , indent = ( " " * 4 ) )
                subcommands = s . subcommands
                if subcommands :
                    echo . br ( )
                    echo . indent ( "Subcommands:" , indent = ( " " * 4 ) )
                    for sc in subcommands . keys ( ) :
                        echo . indent ( sc , indent = ( " " * 6 ) )
                echo . br ( )
        except captain . ParseError :
            pass
        except Exception as e : # echo . exception ( e )
        # echo . err ( " Failed to parse { } because { } " , f , e . message )
            echo . err ( "Failed to parse {}" , f )
            echo . verbose ( e . message )
            echo . br ( )


signature:
def get_rate ( self , zipcode , city = None , state = None , multiple_rates = False ) :
    """Finds sales tax for given info .
 Returns Decimal of the tax rate , e . g . 8.750."""

implementation:
data = self . make_request_data ( zipcode , city , state )
r = requests . get ( self . url , params = data )
resp = r . json ( )
return self . process_response ( resp , multiple_rates )


signature:
def make_request_data ( self , zipcode , city , state ) :
    """Make the request params given location data"""

implementation:
data = { 'key' : self . api_key , 'postalcode' : str ( zipcode ) , 'city' : city , 'state' : state }
data = ZipTaxClient . _clean_request_data ( data )
return data


signature:
def process_response ( self , resp , multiple_rates ) :
    """Get the tax rate from the ZipTax response"""

implementation:
self . _check_for_exceptions ( resp , multiple_rates )
rates = { }
for result in resp [ 'results' ] :
    rate = ZipTaxClient . _cast_tax_rate ( result [ 'taxSales' ] )
    rates [ result [ 'geoCity' ] ] = rate
if not multiple_rates :
    return rates [ list ( rates . keys ( ) ) [ 0 ] ]
return rates


signature:
def _check_for_exceptions ( self , resp , multiple_rates ) :
    """Check if there are exceptions that should be raised"""

implementation:
if resp [ 'rCode' ] != 100 :
    raise exceptions . get_exception_for_code ( resp [ 'rCode' ] ) ( resp )
results = resp [ 'results' ]
if len ( results ) == 0 :
    raise exceptions . ZipTaxNoResults ( 'No results found' )
if len ( results ) > 1 and not multiple_rates : # It ' s fine if all the taxes are the same
    rates = [ result [ 'taxSales' ] for result in results ]
    if len ( set ( rates ) ) != 1 :
        raise exceptions . ZipTaxMultipleResults ( 'Multiple results found but requested only one' )


signature:
def get_all_text ( node ) :
    """Recursively extract all text from node ."""

implementation:
if node . nodeType == node . TEXT_NODE :
    return node . data
else :
    text_string = ""
    for child_node in node . childNodes :
        text_string += get_all_text ( child_node )
    return text_string


signature:
def _extract_packages ( self ) :
    """Extract a package in a new temporary directory ."""

implementation:
self . path_unpacked = mkdtemp ( prefix = "scoap3_package_" , dir = CFG_TMPSHAREDDIR )
for path in self . retrieved_packages_unpacked :
    scoap3utils_extract_package ( path , self . path_unpacked , self . logger )
return self . path_unpacked


signature:
def register ( self , provider_class ) :
    """Registers a provider with the site ."""

implementation:
if not issubclass ( provider_class , BaseProvider ) :
    raise TypeError ( '%s is not a subclass of BaseProvider' % provider_class . __name__ )
if provider_class in self . _registered_providers :
    raise AlreadyRegistered ( '%s is already registered' % provider_class . __name__ )
if issubclass ( provider_class , DjangoProvider ) : # set up signal handler for cache invalidation
    signals . post_save . connect ( self . invalidate_stored_oembeds , sender = provider_class . _meta . model )
# don ' t build the regex yet - if not all urlconfs have been loaded
# and processed at this point , the DjangoProvider instances will fail
# when attempting to reverse urlpatterns that haven ' t been created .
# Rather , the regex - list will be populated once , on - demand .
self . _registered_providers . append ( provider_class )
# flag for re - population
self . invalidate_providers ( )


signature:
def unregister ( self , provider_class ) :
    """Unregisters a provider from the site ."""

implementation:
if not issubclass ( provider_class , BaseProvider ) :
    raise TypeError ( '%s must be a subclass of BaseProvider' % provider_class . __name__ )
if provider_class not in self . _registered_providers :
    raise NotRegistered ( '%s is not registered' % provider_class . __name__ )
self . _registered_providers . remove ( provider_class )
# flag for repopulation
self . invalidate_providers ( )


signature:
def populate ( self ) :
    """Populate the internal registry ' s dictionary with the regexes for each
 provider instance"""

implementation:
self . _registry = { }
for provider_class in self . _registered_providers :
    instance = provider_class ( )
    self . _registry [ instance ] = instance . regex
for stored_provider in StoredProvider . objects . active ( ) :
    self . _registry [ stored_provider ] = stored_provider . regex
self . _populated = True


signature:
def provider_for_url ( self , url ) :
    """Find the right provider for a URL"""

implementation:
for provider , regex in self . get_registry ( ) . items ( ) :
    if re . match ( regex , url ) is not None :
        return provider
raise OEmbedMissingEndpoint ( 'No endpoint matches URL: %s' % url )


signature:
def invalidate_stored_oembeds ( self , sender , instance , created , ** kwargs ) :
    """A hook for django - based oembed providers to delete any stored oembeds"""

implementation:
ctype = ContentType . objects . get_for_model ( instance )
StoredOEmbed . objects . filter ( object_id = instance . pk , content_type = ctype ) . delete ( )


signature:
def embed ( self , url , ** kwargs ) :
    """The heart of the matter"""

implementation:
try : # first figure out the provider
    provider = self . provider_for_url ( url )
except OEmbedMissingEndpoint :
    raise
else :
    try : # check the database for a cached response , because of certain
    # race conditions that exist with get _ or _ create ( ) , do a filter
    # lookup and just grab the first item
        stored_match = StoredOEmbed . objects . filter ( match = url , maxwidth = kwargs . get ( 'maxwidth' , None ) , maxheight = kwargs . get ( 'maxheight' , None ) , date_expires__gte = datetime . datetime . now ( ) ) [ 0 ]
        return OEmbedResource . create_json ( stored_match . response_json )
    except IndexError : # query the endpoint and cache response in db
    # prevent None from being passed in as a GET param
        params = dict ( [ ( k , v ) for k , v in kwargs . items ( ) if v ] )
        # request an oembed resource for the url
        resource = provider . request_resource ( url , ** params )
        try :
            cache_age = int ( resource . cache_age )
            if cache_age < MIN_OEMBED_TTL :
                cache_age = MIN_OEMBED_TTL
        except :
            cache_age = DEFAULT_OEMBED_TTL
        date_expires = datetime . datetime . now ( ) + datetime . timedelta ( seconds = cache_age )
        stored_oembed , created = StoredOEmbed . objects . get_or_create ( match = url , maxwidth = kwargs . get ( 'maxwidth' , None ) , maxheight = kwargs . get ( 'maxheight' , None ) )
        stored_oembed . response_json = resource . json
        stored_oembed . resource_type = resource . type
        stored_oembed . date_expires = date_expires
        if resource . content_object :
            stored_oembed . content_object = resource . content_object
        stored_oembed . save ( )
        return resource


signature:
def autodiscover ( self , url ) :
    """Load up StoredProviders from url if it is an oembed scheme"""

implementation:
headers , response = fetch_url ( url )
if headers [ 'content-type' ] . split ( ';' ) [ 0 ] in ( 'application/json' , 'text/javascript' ) :
    provider_data = json . loads ( response )
    return self . store_providers ( provider_data )


signature:
def store_providers ( self , provider_data ) :
    """Iterate over the returned json and try to sort out any new providers"""

implementation:
if not hasattr ( provider_data , '__iter__' ) :
    raise OEmbedException ( 'Autodiscovered response not iterable' )
provider_pks = [ ]
for provider in provider_data :
    if 'endpoint' not in provider or 'matches' not in provider :
        continue
    resource_type = provider . get ( 'type' )
    if resource_type not in RESOURCE_TYPES :
        continue
    stored_provider , created = StoredProvider . objects . get_or_create ( wildcard_regex = provider [ 'matches' ] )
    if created :
        stored_provider . endpoint_url = relative_to_full ( provider [ 'endpoint' ] , provider [ 'matches' ] )
        stored_provider . resource_type = resource_type
        stored_provider . save ( )
    provider_pks . append ( stored_provider . pk )
return StoredProvider . objects . filter ( pk__in = provider_pks )


signature:
def request_resource ( self , url , ** kwargs ) :
    """Request an OEmbedResource for a given url . Some valid keyword args :
 - format
 - maxwidth
 - maxheight"""

implementation:
params = kwargs
params [ 'url' ] = url
params [ 'format' ] = 'json'
if '?' in self . endpoint_url :
    url_with_qs = '%s&%s' % ( self . endpoint_url . rstrip ( '&' ) , urlencode ( params ) )
else :
    url_with_qs = "%s?%s" % ( self . endpoint_url , urlencode ( params ) )
headers , raw_response = self . _fetch ( url_with_qs )
resource = self . convert_to_resource ( headers , raw_response , params )
return resource


signature:
def _image_field ( self ) :
    """Try to automatically detect an image field"""

implementation:
for field in self . model . _meta . fields :
    if isinstance ( field , ImageField ) :
        return field . name


signature:
def _date_field ( self ) :
    """Try to automatically detect an image field"""

implementation:
for field in self . model . _meta . fields :
    if isinstance ( field , ( DateTimeField , DateField ) ) :
        return field . name


signature:
def _build_regex ( self ) :
    """Performs a reverse lookup on a named view and generates
 a list of regexes that match that object . It generates
 regexes with the domain name included , using sites provided
 by the get _ sites ( ) method .
 > > > regex = provider . regex
 > > > regex . pattern
 ' http : / / ( www2 . kusports . com | www2 . ljworld . com | www . lawrence . com ) / photos / ( ? P < year > \\ d { 4 } ) / ( ? P < month > \\ w { 3 } ) / ( ? P < day > \\ d { 1,2 } ) / ( ? P < object _ id > \\ d + ) / $ '"""

implementation:
# get the regexes from the urlconf
url_patterns = resolver . reverse_dict . get ( self . _meta . named_view )
try :
    regex = url_patterns [ 1 ]
except TypeError :
    raise OEmbedException ( 'Error looking up %s' % self . _meta . named_view )
# get a list of normalized domains
cleaned_sites = self . get_cleaned_sites ( )
site_regexes = [ ]
for site in self . get_sites ( ) :
    site_regexes . append ( cleaned_sites [ site . pk ] [ 0 ] )
# join the sites together with the regex ' or '
sites = '|' . join ( site_regexes )
# create URL - matching regexes for sites
regex = re . compile ( '(%s)/%s' % ( sites , regex ) )
return regex


signature:
def provider_from_url ( self , url ) :
    """Given a URL for any of our sites , try and match it to one , returning
 the domain & name of the match . If no match is found , return current .
 Returns a tuple of domain , site name - - used to determine ' provider '"""

implementation:
domain = get_domain ( url )
site_tuples = self . get_cleaned_sites ( ) . values ( )
for domain_re , name , normalized_domain in site_tuples :
    if re . match ( domain_re , domain ) :
        return normalized_domain , name
site = Site . objects . get_current ( )
return site . domain , site . name


signature:
def get_params ( self , url ) :
    """Extract the named parameters from a url regex . If the url regex does not contain
 named parameters , they will be keyed _ 0 , _ 1 , . . .
 * Named parameters
 Regex :
 / photos / ^ ( ? P < year > \ d { 4 } ) / ( ? P < month > \ w { 3 } ) / ( ? P < day > \ d { 1,2 } ) / ( ? P < object _ id > \ d + ) /
 URL :
 http : / / www2 . ljworld . com / photos / 2009 / oct / 11/12345/
 Return Value :
 { u ' day ' : ' 11 ' , u ' month ' : ' oct ' , u ' object _ id ' : ' 12345 ' , u ' year ' : ' 2009 ' }
 * Unnamed parameters
 Regex :
 / blah / ( [ \ w - ] + ) / ( \ d + ) /
 URL :
 http : / / www . example . com / blah / hello / 123/
 Return Value :
 { u ' _ 0 ' : ' hello ' , u ' _ 1 ' : ' 123 ' }"""

implementation:
match = re . match ( self . regex , url )
if match is not None :
    params = match . groupdict ( )
    if not params :
        params = { }
        for i , group in enumerate ( match . groups ( ) [ 1 : ] ) :
            params [ '_%s' % i ] = group
    return params
raise OEmbedException ( 'No regex matched the url %s' % ( url ) )


signature:
def get_object ( self , url ) :
    """Fields to match is a mapping of url params to fields , so for
 the photos example above , it would be :
 fields _ to _ match = { ' object _ id ' : ' id ' }
 This procedure parses out named params from a URL and then uses
 the fields _ to _ match dictionary to generate a query ."""

implementation:
params = self . get_params ( url )
query = { }
for key , value in self . _meta . fields_to_match . iteritems ( ) :
    try :
        query [ value ] = params [ key ]
    except KeyError :
        raise OEmbedException ( '%s was not found in the urlpattern parameters.  Valid names are: %s' % ( key , ', ' . join ( params . keys ( ) ) ) )
try :
    obj = self . get_queryset ( ) . get ( ** query )
except self . _meta . model . DoesNotExist :
    raise OEmbedException ( 'Requested object not found' )
return obj


signature:
def render_html ( self , obj , context = None ) :
    """Generate the ' html ' attribute of an oembed resource using a template .
 Sort of a corollary to the parser ' s render _ oembed method . By default ,
 the current mapping will be passed in as the context .
 OEmbed templates are stored in :
 oembed / provider / [ app _ label ] _ [ model ] . html
 - - or - -
 oembed / provider / media _ video . html"""

implementation:
provided_context = context or Context ( )
context = RequestContext ( mock_request ( ) )
context . update ( provided_context )
context . push ( )
context [ self . _meta . context_varname ] = obj
rendered = render_to_string ( self . _meta . template_name , context )
context . pop ( )
return rendered


signature:
def map_attr ( self , mapping , attr , obj ) :
    """A kind of cheesy method that allows for callables or attributes to
 be used interchangably"""

implementation:
if attr not in mapping and hasattr ( self , attr ) :
    if not callable ( getattr ( self , attr ) ) :
        mapping [ attr ] = getattr ( self , attr )
    else :
        mapping [ attr ] = getattr ( self , attr ) ( obj )


signature:
def get_image ( self , obj ) :
    """Return an ImageFileField instance"""

implementation:
if self . _meta . image_field :
    return getattr ( obj , self . _meta . image_field )


signature:
def resize ( self , image_field , new_width = None , new_height = None ) :
    """Resize an image to the ' best fit ' width & height , maintaining
 the scale of the image , so a 500x500 image sized to 300x400
 will actually be scaled to 300x300.
 Params :
 image : ImageFieldFile to be resized ( i . e . model . image _ field )
 new _ width & new _ height : desired maximums for resizing
 Returns :
 the url to the new image and the new width & height
 ( http : / / path - to - new - image , 300 , 300)"""

implementation:
if isinstance ( image_field , ImageFieldFile ) and image_field . field . width_field and image_field . field . height_field : # use model fields
    current_width = getattr ( image_field . instance , image_field . field . width_field )
    current_height = getattr ( image_field . instance , image_field . field . height_field )
else : # use PIL
    try :
        file_obj = storage . default_storage . open ( image_field . name , 'rb' )
        img_obj = Image . open ( file_obj )
        current_width , current_height = img_obj . size
    except IOError :
        return ( image_field . url , 0 , 0 )
# determine if resizing needs to be done ( will not scale up )
if current_width < new_width :
    if not new_height or current_height < new_height :
        return ( image_field . url , current_width , current_height )
# calculate ratios
new_width , new_height = scale ( current_width , current_height , new_width , new_height )
# use the image _ processor defined in the settings , or PIL by default
return self . _meta . image_processor . resize ( image_field , new_width , new_height )


signature:
def map_to_dictionary ( self , url , obj , ** kwargs ) :
    """Build a dictionary of metadata for the requested object ."""

implementation:
maxwidth = kwargs . get ( 'maxwidth' , None )
maxheight = kwargs . get ( 'maxheight' , None )
provider_url , provider_name = self . provider_from_url ( url )
mapping = { 'version' : '1.0' , 'url' : url , 'provider_name' : provider_name , 'provider_url' : provider_url , 'type' : self . resource_type }
# a hook
self . preprocess ( obj , mapping , ** kwargs )
# resize image if we have a photo , otherwise use the given maximums
if self . resource_type == 'photo' and self . get_image ( obj ) :
    self . resize_photo ( obj , mapping , maxwidth , maxheight )
elif self . resource_type in ( 'video' , 'rich' , 'photo' ) :
    width , height = size_to_nearest ( maxwidth , maxheight , self . _meta . valid_sizes , self . _meta . force_fit )
    mapping . update ( width = width , height = height )
# create a thumbnail
if self . get_image ( obj ) :
    self . thumbnail ( obj , mapping )
# map attributes to the mapping dictionary . if the attribute is
# a callable , it must have an argument signature of
# ( self , obj )
for attr in ( 'title' , 'author_name' , 'author_url' , 'html' ) :
    self . map_attr ( mapping , attr , obj )
# fix any urls
if 'url' in mapping :
    mapping [ 'url' ] = relative_to_full ( mapping [ 'url' ] , url )
if 'thumbnail_url' in mapping :
    mapping [ 'thumbnail_url' ] = relative_to_full ( mapping [ 'thumbnail_url' ] , url )
if 'html' not in mapping and mapping [ 'type' ] in ( 'video' , 'rich' ) :
    mapping [ 'html' ] = self . render_html ( obj , context = Context ( mapping ) )
# a hook
self . postprocess ( obj , mapping , ** kwargs )
return mapping


implementation:
obj = self . get_object ( url )
mapping = self . map_to_dictionary ( url , obj , ** kwargs )
resource = OEmbedResource . create ( mapping )
resource . content_object = obj
return resource


signature:
def get_object ( self , url , month_format = '%b' , day_format = '%d' ) :
    """Parses the date from a url and uses it in the query . For objects which
 are unique for date ."""

implementation:
params = self . get_params ( url )
try :
    year = params [ self . _meta . year_part ]
    month = params [ self . _meta . month_part ]
    day = params [ self . _meta . day_part ]
except KeyError :
    try : # named lookups failed , so try to get the date using the first
    # three parameters
        year , month , day = params [ '_0' ] , params [ '_1' ] , params [ '_2' ]
    except KeyError :
        raise OEmbedException ( 'Error extracting date from url parameters' )
try :
    tt = time . strptime ( '%s-%s-%s' % ( year , month , day ) , '%s-%s-%s' % ( '%Y' , month_format , day_format ) )
    date = datetime . date ( * tt [ : 3 ] )
except ValueError :
    raise OEmbedException ( 'Error parsing date from: %s' % url )
# apply the date - specific lookups
if isinstance ( self . _meta . model . _meta . get_field ( self . _meta . date_field ) , DateTimeField ) :
    min_date = datetime . datetime . combine ( date , datetime . time . min )
    max_date = datetime . datetime . combine ( date , datetime . time . max )
    query = { '%s__range' % self . _meta . date_field : ( min_date , max_date ) }
else :
    query = { self . _meta . date_field : date }
# apply the regular search lookups
for key , value in self . _meta . fields_to_match . iteritems ( ) :
    try :
        query [ value ] = params [ key ]
    except KeyError :
        raise OEmbedException ( '%s was not found in the urlpattern parameters.  Valid names are: %s' % ( key , ', ' . join ( params . keys ( ) ) ) )
try :
    obj = self . get_queryset ( ) . get ( ** query )
except self . _meta . model . DoesNotExist :
    raise OEmbedException ( 'Requested object not found' )
return obj


signature:
def get_record ( self ) :
    """Override the base ."""

implementation:
self . recid = self . get_recid ( )
self . remove_controlfields ( )
self . update_system_numbers ( )
self . add_systemnumber ( "Inspire" , recid = self . recid )
self . add_control_number ( "003" , "SzGeCERN" )
self . update_collections ( )
self . update_languages ( )
self . update_reportnumbers ( )
self . update_authors ( )
self . update_journals ( )
self . update_subject_categories ( "INSPIRE" , "SzGeCERN" , "categories_cds" )
self . update_pagenumber ( )
self . update_notes ( )
self . update_experiments ( )
self . update_isbn ( )
self . update_dois ( )
self . update_links_and_ffts ( )
self . update_date ( )
self . update_date_year ( )
self . update_hidden_notes ( )
self . update_oai_info ( )
self . update_cnum ( )
self . update_conference_info ( )
self . fields_list = [ "909" , "541" , "961" , "970" , "690" , "695" , "981" , ]
self . strip_fields ( )
if "ANNOUNCEMENT" in self . collections :
    self . update_conference_111 ( )
    self . update_conference_links ( )
    record_add_field ( self . record , "690" , ind1 = "C" , subfields = [ ( "a" , "CONFERENCE" ) ] )
if "THESIS" in self . collections :
    self . update_thesis_information ( )
    self . update_thesis_supervisors ( )
if "PROCEEDINGS" in self . collections : # Special proceeding syntax
    self . update_title_to_proceeding ( )
    self . update_author_to_proceeding ( )
    record_add_field ( self . record , "690" , ind1 = "C" , subfields = [ ( "a" , "CONFERENCE" ) ] )
# 690 tags
if self . tag_as_cern :
    record_add_field ( self . record , "690" , ind1 = "C" , subfields = [ ( "a" , "CERN" ) ] )
return self . record


signature:
def update_oai_info ( self ) :
    """Add the 909 OAI info to 035."""

implementation:
for field in record_get_field_instances ( self . record , '909' , ind1 = "C" , ind2 = "O" ) :
    new_subs = [ ]
    for tag , value in field [ 0 ] :
        if tag == "o" :
            new_subs . append ( ( "a" , value ) )
        else :
            new_subs . append ( ( tag , value ) )
        if value in [ "CERN" , "CDS" , "ForCDS" ] :
            self . tag_as_cern = True
    record_add_field ( self . record , '024' , ind1 = "8" , subfields = new_subs )
record_delete_fields ( self . record , '909' )


signature:
def update_cnum ( self ) :
    """Check if we shall add cnum in 035."""

implementation:
if "ConferencePaper" not in self . collections :
    cnums = record_get_field_values ( self . record , '773' , code = "w" )
    for cnum in cnums :
        cnum_subs = [ ( "9" , "INSPIRE-CNUM" ) , ( "a" , cnum ) ]
        record_add_field ( self . record , "035" , subfields = cnum_subs )


signature:
def update_hidden_notes ( self ) :
    """Remove hidden notes and tag a CERN if detected ."""

implementation:
if not self . tag_as_cern :
    notes = record_get_field_instances ( self . record , tag = "595" )
    for field in notes :
        for dummy , value in field [ 0 ] :
            if value == "CDS" :
                self . tag_as_cern = True
record_delete_fields ( self . record , tag = "595" )


signature:
def update_system_numbers ( self ) :
    """035 Externals ."""

implementation:
scn_035_fields = record_get_field_instances ( self . record , '035' )
new_fields = [ ]
for field in scn_035_fields :
    subs = field_get_subfields ( field )
    if '9' in subs :
        if subs [ '9' ] [ 0 ] . lower ( ) == "cds" and subs . get ( 'a' ) :
            self . add_control_number ( "001" , subs . get ( 'a' ) [ 0 ] )
        if subs [ '9' ] [ 0 ] . lower ( ) in [ "inspire" , "spirestex" , "inspiretex" , "desy" , "cds" ] :
            continue
    new_fields . append ( field_get_subfield_instances ( field ) )
record_delete_fields ( self . record , tag = "035" )
for field in new_fields :
    record_add_field ( self . record , tag = "035" , subfields = field )


signature:
def update_collections ( self ) :
    """Try to determine which collections this record should belong to ."""

implementation:
for value in record_get_field_values ( self . record , '980' , code = 'a' ) :
    if 'NOTE' in value . upper ( ) :
        self . collections . add ( 'NOTE' )
    if 'THESIS' in value . upper ( ) :
        self . collections . add ( 'THESIS' )
    if 'PUBLISHED' in value . upper ( ) :
        self . collections . add ( 'ARTICLE' )
    if 'CONFERENCES' in value . upper ( ) :
        self . collections . add ( 'ANNOUNCEMENT' )
    if 'PROCEEDINGS' in value . upper ( ) :
        self . collections . add ( 'PROCEEDINGS' )
    elif 'CONFERENCEPAPER' in value . upper ( ) and "ConferencePaper" not in self . collections :
        self . collections . add ( 'ConferencePaper' )
        if self . is_published ( ) and "ARTICLE" not in self . collections :
            self . collections . add ( 'ARTICLE' )
        else :
            self . collections . add ( 'PREPRINT' )
    if "HIDDEN" in value . upper ( ) :
        self . hidden = True
# Clear out any existing ones .
record_delete_fields ( self . record , "980" )
if not self . collections :
    self . collections . add ( 'PREPRINT' )
for collection in self . collections :
    record_add_field ( self . record , tag = '980' , subfields = [ ( 'a' , collection ) ] )
    if collection in self . collection_base :
        subs = [ ( 'a' , self . collection_base [ collection ] ) ]
        record_add_field ( self . record , tag = '960' , subfields = subs )


signature:
def update_notes ( self ) :
    """Remove INSPIRE specific notes ."""

implementation:
fields = record_get_field_instances ( self . record , '500' )
for field in fields :
    subs = field_get_subfields ( field )
    for sub in subs . get ( 'a' , [ ] ) :
        sub = sub . strip ( )
        # remove any spaces before / after
        if sub . startswith ( "*" ) and sub . endswith ( "*" ) :
            record_delete_field ( self . record , tag = "500" , field_position_global = field [ 4 ] )


signature:
def update_title_to_proceeding ( self ) :
    """Move title info from 245 to 111 proceeding style ."""

implementation:
titles = record_get_field_instances ( self . record , tag = "245" )
for title in titles :
    subs = field_get_subfields ( title )
    new_subs = [ ]
    if "a" in subs :
        new_subs . append ( ( "a" , subs [ 'a' ] [ 0 ] ) )
    if "b" in subs :
        new_subs . append ( ( "c" , subs [ 'b' ] [ 0 ] ) )
    record_add_field ( self . record , tag = "111" , subfields = new_subs )
record_delete_fields ( self . record , tag = "245" )
record_delete_fields ( self . record , tag = "246" )


signature:
def update_experiments ( self ) :
    """Experiment mapping ."""

implementation:
# 693 Remove if ' not applicable '
for field in record_get_field_instances ( self . record , '693' ) :
    subs = field_get_subfields ( field )
    acc_experiment = subs . get ( "e" , [ ] )
    if not acc_experiment :
        acc_experiment = subs . get ( "a" , [ ] )
        if not acc_experiment :
            continue
    experiment = acc_experiment [ - 1 ]
    # Handle special case of leading experiments numbers NA - 050 - > NA 50
    e_suffix = ""
    if "-NA-" in experiment or "-RD-" in experiment or "-WA-" in experiment :
        splitted_experiment = experiment . split ( "-" )
        e_suffix = "-" . join ( splitted_experiment [ 2 : ] )
        if e_suffix . startswith ( "0" ) :
            e_suffix = e_suffix [ 1 : ]
        experiment = "-" . join ( splitted_experiment [ : 2 ] )
    # only CERN - NA
    translated_experiment = self . get_config_item ( experiment , "experiments" )
    if not translated_experiment :
        continue
    new_subs = [ ]
    if "---" in translated_experiment :
        experiment_a , experiment_e = translated_experiment . split ( "---" )
        new_subs . append ( ( "a" , experiment_a . replace ( "-" , " " ) ) )
    else :
        experiment_e = translated_experiment
    new_subs . append ( ( "e" , experiment_e . replace ( "-" , " " ) + e_suffix ) )
    record_delete_field ( self . record , tag = "693" , field_position_global = field [ 4 ] )
    record_add_field ( self . record , "693" , subfields = new_subs )


signature:
def update_reportnumbers ( self ) :
    """Update reportnumbers ."""

implementation:
report_037_fields = record_get_field_instances ( self . record , '037' )
for field in report_037_fields :
    subs = field_get_subfields ( field )
    for val in subs . get ( "a" , [ ] ) :
        if "arXiv" not in val :
            record_delete_field ( self . record , tag = "037" , field_position_global = field [ 4 ] )
            new_subs = [ ( code , val [ 0 ] ) for code , val in subs . items ( ) ]
            record_add_field ( self . record , "088" , subfields = new_subs )
            break


signature:
def update_authors ( self ) :
    """100 & 700 punctuate author names ."""

implementation:
author_names = record_get_field_instances ( self . record , '100' )
author_names . extend ( record_get_field_instances ( self . record , '700' ) )
for field in author_names :
    subs = field_get_subfields ( field )
    for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
        if key == 'a' :
            field [ 0 ] [ idx ] = ( 'a' , value . replace ( "." , " " ) . strip ( ) )
        elif key == 'v' :
            del field [ 0 ] [ idx ]
    if subs . get ( "u" , None ) == "CERN" :
        self . tag_as_cern = True


signature:
def update_isbn ( self ) :
    """Remove dashes from ISBN ."""

implementation:
isbns = record_get_field_instances ( self . record , '020' )
for field in isbns :
    for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
        if key == 'a' :
            field [ 0 ] [ idx ] = ( 'a' , value . replace ( "-" , "" ) . strip ( ) )


signature:
def update_dois ( self ) :
    """Remove duplicate BibMatch DOIs ."""

implementation:
dois = record_get_field_instances ( self . record , '024' , ind1 = "7" )
all_dois = { }
for field in dois :
    subs = field_get_subfield_instances ( field )
    subs_dict = dict ( subs )
    if subs_dict . get ( 'a' ) :
        if subs_dict [ 'a' ] in all_dois :
            record_delete_field ( self . record , tag = '024' , ind1 = '7' , field_position_global = field [ 4 ] )
            continue
        all_dois [ subs_dict [ 'a' ] ] = field


signature:
def update_journals ( self ) :
    """773 journal translations ."""

implementation:
for field in record_get_field_instances ( self . record , '773' ) :
    subs = field_get_subfield_instances ( field )
    new_subs = [ ]
    volume_letter = ""
    journal_name = ""
    for idx , ( key , value ) in enumerate ( subs ) :
        if key == 'p' :
            journal_name = self . get_config_item ( value , "journals" , allow_substring = False )
            # Make sure journal names have the form ( dot ) ( space ) ( I know it ' s horrible )
            journal_name = journal_name . replace ( '. ' , '.' ) . replace ( '.' , '. ' ) . replace ( '. ,' , '.,' ) . strip ( )
        elif key == 'v' :
            volume_letter = value
        else :
            new_subs . append ( ( key , value ) )
    if not journal_name == "PoS" : # Special handling of journal name and volumes , except PoS
        letter = return_letters_from_string ( volume_letter )
        if letter :
            journal_name = "{0} {1}" . format ( journal_name , letter )
            volume_letter = volume_letter . strip ( letter )
    if journal_name :
        new_subs . append ( ( "p" , journal_name ) )
    if volume_letter :
        new_subs . append ( ( "v" , volume_letter ) )
    record_delete_field ( self . record , tag = "773" , field_position_global = field [ 4 ] )
    record_add_field ( self . record , "773" , subfields = new_subs )


signature:
def update_thesis_supervisors ( self ) :
    """700 - > 701 Thesis supervisors ."""

implementation:
for field in record_get_field_instances ( self . record , '701' ) :
    subs = list ( field [ 0 ] )
    subs . append ( ( "e" , "dir." ) )
    record_add_field ( self . record , '700' , subfields = subs )
record_delete_fields ( self . record , '701' )


signature:
def update_thesis_information ( self ) :
    """501 degree info - move subfields ."""

implementation:
fields_501 = record_get_field_instances ( self . record , '502' )
for field in fields_501 :
    new_subs = [ ]
    for key , value in field [ 0 ] :
        if key == 'b' :
            new_subs . append ( ( 'a' , value ) )
        elif key == 'c' :
            new_subs . append ( ( 'b' , value ) )
        elif key == 'd' :
            new_subs . append ( ( 'c' , value ) )
        else :
            new_subs . append ( ( key , value ) )
    record_delete_field ( self . record , tag = "502" , field_position_global = field [ 4 ] )
    record_add_field ( self . record , "502" , subfields = new_subs )


signature:
def update_pagenumber ( self ) :
    """300 page number ."""

implementation:
pages = record_get_field_instances ( self . record , '300' )
for field in pages :
    for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
        if key == 'a' :
            field [ 0 ] [ idx ] = ( 'a' , "{0} p" . format ( value ) )


signature:
def update_date ( self ) :
    """269 Date normalization ."""

implementation:
dates_269 = record_get_field_instances ( self . record , '269' )
for idx , field in enumerate ( dates_269 ) :
    new_subs = [ ]
    old_subs = field [ 0 ]
    for code , value in old_subs :
        if code == "c" :
            new_subs . append ( ( "c" , convert_date_from_iso_to_human ( value ) ) )
        else :
            new_subs . append ( ( code , value ) )
    dates_269 [ idx ] = field_swap_subfields ( field , new_subs )


signature:
def update_date_year ( self ) :
    """260 Date normalization ."""

implementation:
dates = record_get_field_instances ( self . record , '260' )
for field in dates :
    for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
        if key == 'c' :
            field [ 0 ] [ idx ] = ( 'c' , value [ : 4 ] )
        elif key == 't' :
            del field [ 0 ] [ idx ]
if not dates :
    published_years = record_get_field_values ( self . record , "773" , code = "y" )
    if published_years :
        record_add_field ( self . record , "260" , subfields = [ ( "c" , published_years [ 0 ] [ : 4 ] ) ] )
    else :
        other_years = record_get_field_values ( self . record , "269" , code = "c" )
        if other_years :
            record_add_field ( self . record , "260" , subfields = [ ( "c" , other_years [ 0 ] [ : 4 ] ) ] )


signature:
def is_published ( self ) :
    """Check fields 980 and 773 to see if the record has already been published .
 : return : True is published , else False"""

implementation:
field773 = record_get_field_instances ( self . record , '773' )
for f773 in field773 :
    if 'c' in field_get_subfields ( f773 ) :
        return True
return False


signature:
def update_links_and_ffts ( self ) :
    """FFT ( 856 ) Dealing with files ."""

implementation:
for field in record_get_field_instances ( self . record , tag = '856' , ind1 = '4' ) :
    subs = field_get_subfields ( field )
    newsubs = [ ]
    url = subs . get ( "u" , [ ] )
    if not url :
        record_delete_field ( self . record , '856' , ind1 = '4' , field_position_global = field [ 4 ] )
        continue
    url = url [ 0 ]
    if "inspirehep.net/record" in url and url . endswith ( "pdf" ) : # We have an FFT from INSPIRE
        newsubs . append ( ( 'a' , url ) )
        description = subs . get ( "y" , [ ] )
        if description :
            newsubs . append ( ( 'd' , description [ 0 ] ) )
        if newsubs :
            record_add_field ( self . record , 'FFT' , subfields = newsubs )
            record_delete_field ( self . record , '856' , ind1 = '4' , field_position_global = field [ 4 ] )
    else : # Remove $ w
        for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
            if key == 'w' :
                del field [ 0 ] [ idx ]


signature:
def update_languages ( self ) :
    """041 Language ."""

implementation:
language_fields = record_get_field_instances ( self . record , '041' )
language = "eng"
record_delete_fields ( self . record , "041" )
for field in language_fields :
    subs = field_get_subfields ( field )
    if 'a' in subs :
        language = self . get_config_item ( subs [ 'a' ] [ 0 ] , "languages" )
        break
new_subs = [ ( 'a' , language ) ]
record_add_field ( self . record , "041" , subfields = new_subs )


signature:
def pathjoin ( * args , ** kwargs ) :
    """Arguments :
 args ( list ) : * args list of paths
 if len ( args ) = = 1 , args [ 0 ] is not a string , and args [ 0 ] is iterable ,
 set args to args [ 0 ] .
 Basically : :
 joined _ path = u ' / ' . join (
 [ args [ 0 ] . rstrip ( ' / ' ) ] +
 [ a . strip ( ' / ' ) for a in args [ 1 : - 1 ] ] +
 [ args [ - 1 ] . lstrip ( ' / ' ) ] )"""

implementation:
log . debug ( 'pathjoin: %r' % list ( args ) )
def _pathjoin ( * args , ** kwargs ) :
    len_ = len ( args ) - 1
    if len_ < 0 :
        raise Exception ( 'no args specified' )
    elif len_ == 0 :
        if not isinstance ( args , basestring ) :
            if hasattr ( args , '__iter__' ) :
                _args = args
                _args
                args = args [ 0 ]
    for i , arg in enumerate ( args ) :
        if not i :
            yield arg . rstrip ( '/' )
        elif i == len_ :
            yield arg . lstrip ( '/' )
        else :
            yield arg . strip ( '/' )
joined_path = u'/' . join ( _pathjoin ( * args ) )
return sanitize_path ( joined_path )


signature:
def generate_dirlist_html ( FS , filepath ) :
    """Generate directory listing HTML
 Arguments :
 FS ( FS ) : filesystem object to read files from
 filepath ( str ) : path to generate directory listings for
 Keyword Arguments :
 list _ dir ( callable : list [ str ] ) : list file names in a directory
 isdir ( callable : bool ) : os . path . isdir
 Yields :
 str : lines of an HTML table"""

implementation:
yield '
'
if filepath == '/' :
    filepath = ''
for name in FS . listdir ( filepath ) :
    full_path = pathjoin ( filepath , name )
    if FS . isdir ( full_path ) :
        full_path = full_path + '/'
    yield u'
{0}
' . format ( cgi . escape ( full_path ) )
    # TODO XXX
yield '
'


signature:
def git_static_file ( filename , mimetype = 'auto' , download = False , charset = 'UTF-8' ) :
    """This method is derived from bottle . static _ file :
 Open [ a file ] and return : exc : ` HTTPResponse ` with status
 code 200 , 305 , 403 or 404 . The ` ` Content - Type ` ` , ` ` Content - Encoding ` ` ,
 ` ` Content - Length ` ` and ` ` Last - Modified ` ` headers are set if possible .
 Special support for ` ` If - Modified - Since ` ` [ . . . ] .
 : param filename : Name or path of the file to send .
 : param mimetype : Defines the content - type header ( default : guess from
 file extension )
 : param download : If True , ask the browser to open a ` Save as . . . ` dialog
 instead of opening the file with the associated program . You can
 specify a custom filename as a string . If not specified , the
 original filename is used ( default : False ) .
 : param charset : The charset to use for files with a ` ` text / * ` `
 mime - type . ( default : UTF - 8)"""

implementation:
# root = os . path . abspath ( root ) + os . sep
# filename = os . path . abspath ( pathjoin ( root , filename . strip ( ' / \ \ ' ) ) )
filename = filename . strip ( '/\\' )
headers = dict ( )
FS = request . app . config [ 'pgs.FS' ]
# if not filename . startswith ( root ) :
# return HTTPError ( 403 , " Access denied . " )
if not FS . exists ( filename ) :
    return HTTPError ( 404 , "Not found." )
# if not os . access ( filename , os . R _ OK ) :
# return HTTPError ( 403 , " You do not have permission to access this file . " )
if mimetype == 'auto' :
    if download and download is not True :
        mimetype , encoding = mimetypes . guess_type ( download )
    else :
        mimetype , encoding = mimetypes . guess_type ( filename )
    if encoding :
        headers [ 'Content-Encoding' ] = encoding
if mimetype :
    if mimetype [ : 5 ] == 'text/' and charset and 'charset' not in mimetype :
        mimetype += '; charset=%s' % charset
    headers [ 'Content-Type' ] = mimetype
if download :
    download = os . path . basename ( filename if download else download )
    headers [ 'Content-Disposition' ] = 'attachment; filename="%s"' % download
# stats = os . stat ( filename )
info = FS . getinfo ( filename )
headers [ 'Content-Length' ] = clen = info [ 'size' ]
lm = time . strftime ( "%a, %d %b %Y %H:%M:%S GMT" , time . gmtime ( info [ 'modified_time' ] ) )
headers [ 'Last-Modified' ] = lm
ims = request . environ . get ( 'HTTP_IF_MODIFIED_SINCE' )
if ims :
    ims = parse_date ( ims . split ( ";" ) [ 0 ] . strip ( ) )
mtime = info [ 'modified_time' ]
if mtime and ims is not None and ims >= int ( mtime ) :
    headers [ 'Date' ] = time . strftime ( "%a, %d %b %Y %H:%M:%S GMT" , time . gmtime ( ) )
    return HTTPResponse ( status = 304 , ** headers )
body = '' if request . method == 'HEAD' else FS . get_fileobj ( filename )
clen
# headers [ " Accept - Ranges " ] = " bytes "
# ranges = request . environ . get ( ' HTTP _ RANGE ' )
# if ' HTTP _ RANGE ' in request . environ :
# ranges = list ( parse _ range _ header ( request . environ [ ' HTTP _ RANGE ' ] , clen ) )
# if not ranges :
# return HTTPError ( 416 , " Requested Range Not Satisfiable " )
# offset , end = ranges [ 0]
# headers [ " Content - Range " ] = " bytes % d - % d / % d " % ( offset , end - 1 , clen )
# headers [ " Content - Length " ] = str ( end - offset )
# if body : body = _ file _ iter _ range ( body , offset , end - offset )
# return HTTPResponse ( body , status = 206 , * * headers )
return HTTPResponse ( body , ** headers )


signature:
def check_pkgs_integrity ( filelist , logger , ftp_connector , timeout = 120 , sleep_time = 10 ) :
    """Checks if files are not being uploaded to server .
 @ timeout - time after which the script will register an error ."""

implementation:
ref_1 = [ ]
ref_2 = [ ]
i = 1
print >> sys . stdout , "\nChecking packages integrity."
for filename in filelist : # ref _ 1 . append ( self . get _ remote _ file _ size ( filename ) )
    get_remote_file_size ( ftp_connector , filename , ref_1 )
print >> sys . stdout , "\nGoing to sleep for %i sec." % ( sleep_time , )
time . sleep ( sleep_time )
while sleep_time * i < timeout :
    for filename in filelist : # ref _ 2 . append ( self . get _ remote _ file _ size ( filename ) )
        get_remote_file_size ( ftp_connector , filename , ref_2 )
    if ref_1 == ref_2 :
        print >> sys . stdout , "\nIntegrity OK:)"
        logger . info ( "Packages integrity OK." )
        break
    else :
        print >> sys . stdout , "\nWaiting %d time for itegrity..." % ( i , )
        logger . info ( "\nWaiting %d time for itegrity..." % ( i , ) )
        i += 1
        ref_1 , ref_2 = ref_2 , [ ]
        time . sleep ( sleep_time )
else :
    not_finished_files = [ ]
    for count , val1 in enumerate ( ref_1 ) :
        if val1 != ref_2 [ count ] :
            not_finished_files . append ( filelist [ count ] )
    print >> sys . stdout , "\nOMG, OMG something wrong with integrity."
    logger . error ( "Integrity check faild for files %s" % ( not_finished_files , ) )


signature:
def normalize_query ( query_string , terms = TERMS , norm_space = NORM_SPACE ) :
    """Example :
 > > > normalize _ query ( ' some random words " with quotes " and spaces ' )
 [ ' some ' , ' random ' , ' words ' , ' with quotes ' , ' and ' , ' spaces ' ]"""

implementation: return [ norm_space ( ' ' , ( t [ 0 ] or t [ 1 ] ) . strip ( ) ) for t in terms ( query_string ) ]


signature:
def collapse_initials ( name ) :
    """Removes the space between initials .
 eg T . A . - - > T . A ."""

implementation:
if len ( name . split ( ) ) > 1 :
    name = re . sub ( r'([A-Z]\.) +(?=[A-Z]\.)' , r'\1' , name )
return name


signature:
def fix_name_capitalization ( lastname , givennames ) :
    """Converts capital letters to lower keeps first letter capital ."""

implementation:
lastnames = lastname . split ( )
if len ( lastnames ) == 1 :
    if '-' in lastname :
        names = lastname . split ( '-' )
        names = map ( lambda a : a [ 0 ] + a [ 1 : ] . lower ( ) , names )
        lastname = '-' . join ( names )
    else :
        lastname = lastname [ 0 ] + lastname [ 1 : ] . lower ( )
else :
    names = [ ]
    for name in lastnames :
        if re . search ( r'[A-Z]\.' , name ) :
            names . append ( name )
        else :
            names . append ( name [ 0 ] + name [ 1 : ] . lower ( ) )
    lastname = ' ' . join ( names )
    lastname = collapse_initials ( lastname )
names = [ ]
for name in givennames :
    if re . search ( r'[A-Z]\.' , name ) :
        names . append ( name )
    else :
        names . append ( name [ 0 ] + name [ 1 : ] . lower ( ) )
givennames = ' ' . join ( names )
return lastname , givennames


signature:
def extract_oembeds ( self , text , maxwidth = None , maxheight = None , resource_type = None ) :
    """Scans a block of text and extracts oembed data on any urls ,
 returning it in a list of dictionaries"""

implementation:
parser = text_parser ( )
urls = parser . extract_urls ( text )
return self . handle_extracted_urls ( urls , maxwidth , maxheight , resource_type )


signature:
def strip ( self , text , * args , ** kwargs ) :
    """Try to maintain parity with what is extracted by extract since strip
 will most likely be used in conjunction with extract"""

implementation:
if OEMBED_DEFAULT_PARSE_HTML :
    extracted = self . extract_oembeds_html ( text , * args , ** kwargs )
else :
    extracted = self . extract_oembeds ( text , * args , ** kwargs )
matches = [ r [ 'original_url' ] for r in extracted ]
match_handler = lambda m : m . group ( ) not in matches and m . group ( ) or ''
return re . sub ( URL_RE , match_handler , text )


signature:
def autodiscover ( ) :
    """Automatically build the provider index ."""

implementation:
import imp
from django . conf import settings
for app in settings . INSTALLED_APPS :
    try :
        app_path = __import__ ( app , { } , { } , [ app . split ( '.' ) [ - 1 ] ] ) . __path__
    except AttributeError :
        continue
    try :
        imp . find_module ( 'oembed_providers' , app_path )
    except ImportError :
        continue
    __import__ ( "%s.oembed_providers" % app )


signature:
def select ( options = None ) :
    """pass in a list of options , promt the user to select one , and return the selected option or None"""

implementation:
if not options :
    return None
width = len ( str ( len ( options ) ) )
for x , option in enumerate ( options ) :
    sys . stdout . write ( '{:{width}}) {}\n' . format ( x + 1 , option , width = width ) )
sys . stdout . write ( '{:>{width}} ' . format ( '#?' , width = width + 1 ) )
sys . stdout . flush ( )
if sys . stdin . isatty ( ) : # regular prompt
    try :
        response = raw_input ( ) . strip ( )
    except ( EOFError , KeyboardInterrupt ) : # handle ctrl - d , ctrl - c
        response = ''
else : # try connecting to current tty , when using pipes
    sys . stdin = open ( "/dev/tty" )
    try :
        response = ''
        while True :
            response += sys . stdin . read ( 1 )
            if response . endswith ( '\n' ) :
                break
    except ( EOFError , KeyboardInterrupt ) :
        sys . stdout . flush ( )
        pass
try :
    response = int ( response ) - 1
except ValueError :
    return None
if response < 0 or response >= len ( options ) :
    return None
return options [ response ]


signature:
def main ( ) :
    argparser = ArgumentParser ( )
    subparsers = argparser . add_subparsers ( dest = 'selected_subparser' )
    all_parser = subparsers . add_parser ( 'all' )
    elsevier_parser = subparsers . add_parser ( 'elsevier' )
    oxford_parser = subparsers . add_parser ( 'oxford' )
    springer_parser = subparsers . add_parser ( 'springer' )
    all_parser . add_argument ( '--update-credentials' , action = 'store_true' )
    elsevier_parser . add_argument ( '--run-locally' , action = 'store_true' )
    elsevier_parser . add_argument ( '--package-name' )
    elsevier_parser . add_argument ( '--path' )
    elsevier_parser . add_argument ( '--CONSYN' , action = 'store_true' )
    elsevier_parser . add_argument ( '--update-credentials' , action = 'store_true' )
    elsevier_parser . add_argument ( '--extract-nations' , action = 'store_true' )
    oxford_parser . add_argument ( '--dont-empty-ftp' , action = 'store_true' )
    oxford_parser . add_argument ( '--package-name' )
    oxford_parser . add_argument ( '--path' )
    oxford_parser . add_argument ( '--update-credentials' , action = 'store_true' )
    oxford_parser . add_argument ( '--extract-nations' , action = 'store_true' )
    springer_parser . add_argument ( '--package-name' )
    springer_parser . add_argument ( '--path' )
    springer_parser . add_argument ( '--update-credentials' , action = 'store_true' )
    springer_parser . add_argument ( '--extract-nations' , action = 'store_true' )
    '''Transforms the argparse arguments from Namespace to dict and then to Bunch
 Therefore it is not necessary to access the arguments using the dict syntax
 The settings can be called like regular vars on the settings object'''

implementation:
settings = Bunch ( vars ( argparser . parse_args ( ) ) )
call_package ( settings )


signature:
def get_record ( self , record ) :
    """Reads a dom xml element in oaidc format and
 returns the bibrecord object"""

implementation:
self . document = record
rec = create_record ( )
language = self . _get_language ( )
if language and language != 'en' :
    record_add_field ( rec , '041' , subfields = [ ( 'a' , language ) ] )
publisher = self . _get_publisher ( )
date = self . _get_date ( )
if publisher and date :
    record_add_field ( rec , '260' , subfields = [ ( 'b' , publisher ) , ( 'c' , date ) ] )
elif publisher :
    record_add_field ( rec , '260' , subfields = [ ( 'b' , publisher ) ] )
elif date :
    record_add_field ( rec , '260' , subfields = [ ( 'c' , date ) ] )
title = self . _get_title ( )
if title :
    record_add_field ( rec , '245' , subfields = [ ( 'a' , title ) ] )
record_copyright = self . _get_copyright ( )
if record_copyright :
    record_add_field ( rec , '540' , subfields = [ ( 'a' , record_copyright ) ] )
subject = self . _get_subject ( )
if subject :
    record_add_field ( rec , '650' , ind1 = '1' , ind2 = '7' , subfields = [ ( 'a' , subject ) , ( '2' , 'PoS' ) ] )
authors = self . _get_authors ( )
first_author = True
for author in authors :
    subfields = [ ( 'a' , author [ 0 ] ) ]
    for affiliation in author [ 1 ] :
        subfields . append ( ( 'v' , affiliation ) )
    if first_author :
        record_add_field ( rec , '100' , subfields = subfields )
        first_author = False
    else :
        record_add_field ( rec , '700' , subfields = subfields )
identifier = self . get_identifier ( )
conference = identifier . split ( ':' ) [ 2 ]
conference = conference . split ( '/' ) [ 0 ]
contribution = identifier . split ( ':' ) [ 2 ]
contribution = contribution . split ( '/' ) [ 1 ]
record_add_field ( rec , '773' , subfields = [ ( 'p' , 'PoS' ) , ( 'v' , conference . replace ( ' ' , '' ) ) , ( 'c' , contribution ) , ( 'y' , date [ : 4 ] ) ] )
record_add_field ( rec , '980' , subfields = [ ( 'a' , 'ConferencePaper' ) ] )
record_add_field ( rec , '980' , subfields = [ ( 'a' , 'HEP' ) ] )
return rec


signature:
def progress ( length , ** kwargs ) :
    """display a progress that can update in place
 example - -
 total _ length = 1000
 with echo . progress ( total _ length ) as p :
 for x in range ( total _ length ) :
 # do something crazy
 p . update ( x )
 length - - int - - the total size of what you will be updating progress on"""

implementation:
quiet = False
progress_class = kwargs . pop ( "progress_class" , Progress )
kwargs [ "write_method" ] = istdout . info
kwargs [ "width" ] = kwargs . get ( "width" , globals ( ) [ "WIDTH" ] )
kwargs [ "length" ] = length
pbar = progress_class ( ** kwargs )
pbar . update ( 0 )
yield pbar
pbar . update ( length )
br ( )


signature:
def increment ( itr , n = 1 , format_msg = "{}. " ) :
    """Similar to enumerate but will set format _ msg . format ( n ) into the prefix on
 each iteration
 : Example :
 for v in increment ( [ " foo " , " bar " ] ) :
 echo . out ( v ) # 1 . foo \n 2 . bar
 : param itr : iterator , any iterator you want to set a numeric prefix on on every
 iteration
 : param n : integer , the starting integer for the numeric prefix
 : param format _ msg : string , this will basically do : format _ msg . format ( n ) so there
 should only be one set of curly brackets
 : returns : yield generator"""

implementation:
for i , v in enumerate ( itr , n ) :
    with prefix ( format_msg , i ) :
        yield v


signature:
def err ( format_msg , * args , ** kwargs ) :
    '''print format _ msg to stderr'''

implementation:
exc_info = kwargs . pop ( "exc_info" , False )
stderr . warning ( str ( format_msg ) . format ( * args , ** kwargs ) , exc_info = exc_info )


signature:
def out ( format_msg = "" , * args , ** kwargs ) :
    '''print format _ msg to stdout , taking into account - - quiet setting'''

implementation:
logmethod = kwargs . get ( "logmethod" , stdout . info )
if format_msg != "" :
    if Prefix . has ( ) :
        if isinstance ( format_msg , basestring ) :
            format_msg = Prefix . get ( ) + format_msg
        else :
            format_msg = Prefix . get ( ) + str ( format_msg )
    if isinstance ( format_msg , basestring ) :
        if args or kwargs :
            s = format_msg . format ( * args , ** kwargs )
        else :
            s = format_msg
        logmethod ( s )
    # width = globals ( ) [ " width " ]
    # s = textwrap . fill ( s , width = width )
    # stdout . info ( s )
    else :
        logmethod ( str ( format_msg ) )
else :
    logmethod ( "" )


signature:
def verbose ( format_msg = "" , * args , ** kwargs ) :
    '''print format _ msg to stdout , taking into account - - verbose flag'''

implementation:
kwargs [ "logmethod" ] = stdout . debug
out ( format_msg , * args , ** kwargs )


signature:
def banner ( * lines , ** kwargs ) :
    """prints a banner
 sep - - string - - the character that will be on the line on the top and bottom
 and before any of the lines , defaults to *
 count - - integer - - the line width , defaults to 80"""

implementation:
sep = kwargs . get ( "sep" , "*" )
count = kwargs . get ( "width" , globals ( ) [ "WIDTH" ] )
out ( sep * count )
if lines :
    out ( sep )
    for line in lines :
        out ( "{} {}" . format ( sep , line ) )
    out ( sep )
    out ( sep * count )


signature:
def table ( * columns , ** kwargs ) :
    """format columned data so we can easily print it out on a console , this just takes
 columns of data and it will format it into properly aligned columns , it ' s not
 fancy , but it works for most type of strings that I need it for , like server name
 lists .
 other formatting options :
 http : / / stackoverflow . com / a / 8234511/5006
 other packages that probably do this way better :
 https : / / stackoverflow . com / a / 26937531/5006
 : Example :
 > > > echo . table ( [ ( 1 , 2 ) , ( 3 , 4 ) , ( 5 , 6 ) , ( 7 , 8 ) , ( 9 , 0 ) ] )
 1 2
 3 4
 5 6
 7 8
 9 0
 > > > echo . table ( [ 1 , 3 , 5 , 7 , 9 ] , [ 2 , 4 , 6 , 8 , 0 ] )
 1 2
 3 4
 5 6
 7 8
 9 0
 : param * columns : can either be a list of rows or multiple lists representing each
 column in the table
 : param * * kwargs : dict
 prefix - - string - - what you want before each row ( eg , a tab )
 buf _ count - - integer - - how many spaces between longest col value and its neighbor
 headers - - list - - the headers you want , must match column count
 widths - - list - - the widths of each column you want to use , this doesn ' t have
 to match column count , so you can do something like [ 0 , 5 ] to set the
 width of the second column
 width - - int - - similar to widths except it will set this value for all columns"""

implementation:
ret = [ ]
prefix = kwargs . get ( 'prefix' , '' )
buf_count = kwargs . get ( 'buf_count' , 2 )
if len ( columns ) == 1 :
    columns = list ( columns [ 0 ] )
else : # without the list the zip iterator gets spent , I ' m sure I can make this
# better
    columns = list ( zip ( * columns ) )
headers = kwargs . get ( "headers" , [ ] )
if headers :
    columns . insert ( 0 , headers )
# we have to go through all the rows and calculate the length of each
# column of each row
widths = kwargs . get ( "widths" , [ ] )
row_counts = Counter ( )
for i in range ( len ( widths ) ) :
    row_counts [ i ] = int ( widths [ i ] )
width = int ( kwargs . get ( "width" , 0 ) )
for row in columns :
    for i , c in enumerate ( row ) :
        if isinstance ( c , basestring ) :
            cl = len ( c )
        else :
            cl = len ( str ( c ) )
        if cl > row_counts [ i ] :
            row_counts [ i ] = cl
width = int ( kwargs . get ( "width" , 0 ) )
if width :
    for i in row_counts :
        if row_counts [ i ] < width :
            row_counts [ i ] = width
# actually go through and format each row
def colstr ( c ) :
    if isinstance ( c , basestring ) :
        return c
    return str ( c )
def rowstr ( row , prefix , row_counts ) :
    row_format = prefix
    cols = list ( map ( colstr , row ) )
    for i in range ( len ( row_counts ) ) :
        c = cols [ i ]
        # build the format string for each row , we use the row _ counts found
        # above to decide how much padding each column should get
        # https : / / stackoverflow . com / a / 9536084/5006
        if re . match ( r"^\d+(?:\.\d+)?$" , c ) :
            if i == 0 :
                row_format += "{:>" + str ( row_counts [ i ] ) + "}"
            else :
                row_format += "{:>" + str ( row_counts [ i ] + buf_count ) + "}"
        else :
            row_format += "{:<" + str ( row_counts [ i ] + buf_count ) + "}"
    return row_format . format ( * cols )
for row in columns :
    ret . append ( rowstr ( row , prefix , row_counts ) )
out ( os . linesep . join ( ret ) )


signature:
def prompt ( question , choices = None ) :
    """echo a prompt to the user and wait for an answer
 question - - string - - the prompt for the user
 choices - - list - - if given , only exit when prompt matches one of the choices
 return - - string - - the answer that was given by the user"""

implementation:
if not re . match ( "\s$" , question ) :
    question = "{}: " . format ( question )
while True :
    if sys . version_info [ 0 ] > 2 :
        answer = input ( question )
    else :
        answer = raw_input ( question )
    if not choices or answer in choices :
        break
return answer


signature:
def get_records ( self , url ) :
    """Returns the records listed in the webpage given as
 parameter as a xml String .
 @ param url : the url of the Journal , Book , Protocol or Reference work"""

implementation:
page = urllib2 . urlopen ( url )
pages = [ BeautifulSoup ( page ) ]
# content spread over several pages ?
numpag = pages [ 0 ] . body . findAll ( 'span' , attrs = { 'class' : 'number-of-pages' } )
if len ( numpag ) > 0 :
    if re . search ( '^\d+$' , numpag [ 0 ] . string ) :
        for i in range ( int ( numpag [ 0 ] . string ) - 1 ) :
            page = urllib2 . urlopen ( '%s/page/%i' % ( url , i + 2 ) )
            pages . append ( BeautifulSoup ( page ) )
    else :
        print ( "number of pages %s not an integer" % ( numpag [ 0 ] . string ) )
impl = getDOMImplementation ( )
doc = impl . createDocument ( None , "collection" , None )
links = [ ]
for page in pages :
    links += page . body . findAll ( 'p' , attrs = { 'class' : 'title' } )
    links += page . body . findAll ( 'h3' , attrs = { 'class' : 'title' } )
for link in links :
    record = self . _get_record ( link )
    doc . firstChild . appendChild ( record )
return doc . toprettyxml ( )


signature:
def inject_quiet ( levels ) :
    """see - - quiet flag help for what this does"""

implementation:
loggers = list ( Logger . manager . loggerDict . items ( ) )
loggers . append ( ( "root" , getLogger ( ) ) )
level_filter = LevelFilter ( levels )
for logger_name , logger in loggers :
    for handler in getattr ( logger , "handlers" , [ ] ) :
        handler . addFilter ( level_filter )


signature:
def connect ( self ) :
    """Logs into the specified ftp server and returns connector ."""

implementation:
for tried_connection_count in range ( CFG_FTP_CONNECTION_ATTEMPTS ) :
    try :
        self . ftp = FtpHandler ( self . config . OXFORD . URL , self . config . OXFORD . LOGIN , self . config . OXFORD . PASSWORD )
        self . logger . debug ( ( "Successful connection to the " "Oxford University Press server" ) )
        return
    except socket_timeout_exception as err :
        self . logger . error ( ( 'Failed to connect %d of %d times. ' 'Will sleep for %d seconds and try again.' ) % ( tried_connection_count + 1 , CFG_FTP_CONNECTION_ATTEMPTS , CFG_FTP_TIMEOUT_SLEEP_DURATION ) )
        time . sleep ( CFG_FTP_TIMEOUT_SLEEP_DURATION )
    except Exception as err :
        self . logger . error ( ( 'Failed to connect to the Oxford ' 'University Press server. %s' ) % ( err , ) )
        break
raise LoginException ( err )


signature:
def _extract_packages ( self ) :
    """Extract a package in a new directory ."""

implementation:
if not hasattr ( self , "retrieved_packages_unpacked" ) :
    self . retrieved_packages_unpacked = [ self . package_name ]
for path in self . retrieved_packages_unpacked :
    package_name = basename ( path )
    self . path_unpacked = join ( CFG_UNPACKED_FILES , package_name . split ( '.' ) [ 0 ] )
    self . logger . debug ( "Extracting package: %s" % ( path . split ( "/" ) [ - 1 ] , ) )
    try :
        if "_archival_pdf" in self . path_unpacked :
            self . path_unpacked = ( self . path_unpacked . rstrip ( "_archival_pdf" ) )
            ZipFile ( path ) . extractall ( join ( self . path_unpacked , "archival_pdfs" ) )
        else :
            ZipFile ( path ) . extractall ( self . path_unpacked )
        # TarFile . open ( path ) . extractall ( self . path _ unpacked )
    except Exception :
        register_exception ( alert_admin = True , prefix = "OUP error extracting package." )
        self . logger . error ( "Error extraction package file: %s" % ( path , ) )
if hasattr ( self , "path_unpacked" ) :
    return self . path_unpacked


signature:
def _crawl_oxford_and_find_main_xml ( self ) :
    """A package contains several subdirectory corresponding to each article .
 An article is actually identified by the existence of a main . pdf and
 a main . xml in a given directory ."""

implementation:
self . found_articles = [ ]
def visit ( arg , dirname , names ) :
    files = [ filename for filename in names if ".xml" in filename ]
    if files :
        try :
            for f in files :
                self . found_articles . append ( join ( dirname , f ) )
        except Exception as err :
            register_exception ( )
            print ( "ERROR: can't normalize %s: %s" % ( dirname , err ) , file = sys . stderr )
if hasattr ( self , 'path_unpacked' ) :
    walk ( self . path_unpacked , visit , None )
elif self . path :
    walk ( self . path , visit , None )
else :
    self . logger . info ( "Nothing to do." )


signature:
def get_data ( self ) :
    """Fetch / refresh the current instance ' s data from the NuHeat API"""

implementation:
params = { "serialnumber" : self . serial_number }
data = self . _session . request ( config . THERMOSTAT_URL , params = params )
self . _data = data
self . heating = data . get ( "Heating" )
self . online = data . get ( "Online" )
self . room = data . get ( "Room" )
self . serial_number = data . get ( "SerialNumber" )
self . temperature = data . get ( "Temperature" )
self . min_temperature = data . get ( "MinTemp" )
self . max_temperature = data . get ( "MaxTemp" )
self . target_temperature = data . get ( "SetPointTemp" )
self . _schedule_mode = data . get ( "ScheduleMode" )


signature:
def schedule_mode ( self , mode ) :
    """Set the thermostat mode
 : param mode : The desired mode integer value .
 Auto = 1
 Temporary hold = 2
 Permanent hold = 3"""

implementation:
modes = [ config . SCHEDULE_RUN , config . SCHEDULE_TEMPORARY_HOLD , config . SCHEDULE_HOLD ]
if mode not in modes :
    raise Exception ( "Invalid mode. Please use one of: {}" . format ( modes ) )
self . set_data ( { "ScheduleMode" : mode } )


signature:
def set_target_fahrenheit ( self , fahrenheit , mode = config . SCHEDULE_HOLD ) :
    """Set the target temperature to the desired fahrenheit , with more granular control of the
 hold mode
 : param fahrenheit : The desired temperature in F
 : param mode : The desired mode to operate in"""

implementation:
temperature = fahrenheit_to_nuheat ( fahrenheit )
self . set_target_temperature ( temperature , mode )


signature:
def set_target_celsius ( self , celsius , mode = config . SCHEDULE_HOLD ) :
    """Set the target temperature to the desired celsius , with more granular control of the hold
 mode
 : param celsius : The desired temperature in C
 : param mode : The desired mode to operate in"""

implementation:
temperature = celsius_to_nuheat ( celsius )
self . set_target_temperature ( temperature , mode )


signature:
def set_target_temperature ( self , temperature , mode = config . SCHEDULE_HOLD ) :
    """Updates the target temperature on the NuHeat API
 : param temperature : The desired temperature in NuHeat format
 : param permanent : Permanently hold the temperature . If set to False , the schedule will
 resume at the next programmed event"""

implementation:
if temperature < self . min_temperature :
    temperature = self . min_temperature
if temperature > self . max_temperature :
    temperature = self . max_temperature
modes = [ config . SCHEDULE_TEMPORARY_HOLD , config . SCHEDULE_HOLD ]
if mode not in modes :
    raise Exception ( "Invalid mode. Please use one of: {}" . format ( modes ) )
self . set_data ( { "SetPointTemp" : temperature , "ScheduleMode" : mode } )


signature:
def set_data ( self , post_data ) :
    """Update ( patch ) the current instance ' s data on the NuHeat API"""

implementation:
params = { "serialnumber" : self . serial_number }
self . _session . request ( config . THERMOSTAT_URL , method = "POST" , data = post_data , params = params )


signature:
def load_config ( filename = None , section_option_dict = { } ) :
    """This function returns a Bunch object from the stated config file .
 NOTE :
 The values are not evaluated by default .
 filename :
 The desired config file to read .
 The config file must be written in a syntax readable to the
 ConfigParser module - > INI syntax
 [ sectionA ]
 optionA1 = . . .
 optionA2 = . . .
 section _ option _ dict :
 A dictionary that contains keys , which are associated to the sections
 in the config file , and values , which are a list of the desired
 options .
 If empty , everything will be loaded .
 If the lists are empty , everything from the sections will be loaded .
 Example :
 dict = { ' sectionA ' : [ ' optionA1 ' , ' optionA2 ' , . . . ] ,
 ' sectionB ' : [ ' optionB1 ' , ' optionB2 ' , . . . ] }
 config = get _ config ( ' config . cfg ' , dict )
 config . sectionA . optionA1
 Other :
 Bunch can be found in configparser . py"""

implementation:
config = ConfigParser ( )
config . read ( filename )
working_dict = _prepare_working_dict ( config , section_option_dict )
tmp_dict = { }
for section , options in working_dict . iteritems ( ) :
    tmp_dict [ section ] = { }
    for option in options :
        tmp_dict [ section ] [ option ] = config . get ( section , option )
return Bunch ( tmp_dict )


signature:
def authenticate ( self ) :
    """Authenticate against the NuHeat API"""

implementation:
if self . _session_id :
    _LOGGER . debug ( "Using existing NuHeat session" )
    return
_LOGGER . debug ( "Creating NuHeat session" )
post_data = { "Email" : self . username , "Password" : self . password , "application" : "0" }
data = self . request ( config . AUTH_URL , method = "POST" , data = post_data )
session_id = data . get ( "SessionId" )
if not session_id :
    raise Exception ( "Authentication error" )
self . _session_id = session_id


signature:
def request ( self , url , method = "GET" , data = None , params = None , retry = True ) :
    """Make a request to the NuHeat API
 : param url : The URL to request
 : param method : The type of request to make ( GET , POST )
 : param data : Data to be sent along with POST requests
 : param params : Querystring parameters
 : param retry : Attempt to re - authenticate and retry request if necessary"""

implementation:
headers = config . REQUEST_HEADERS
if params and self . _session_id :
    params [ 'sessionid' ] = self . _session_id
if method == "GET" :
    response = requests . get ( url , headers = headers , params = params )
elif method == "POST" :
    response = requests . post ( url , headers = headers , params = params , data = data )
# Handle expired sessions
if response . status_code == 401 and retry :
    _LOGGER . warn ( "NuHeat APIrequest unauthorized [401]. Try to re-authenticate." )
    self . _session_id = None
    self . authenticate ( )
    return self . request ( url , method = method , data = data , params = params , retry = False )
response . raise_for_status ( )
try :
    return response . json ( )
except ValueError : # No JSON object
    return response


signature:
def handle_starttag ( self , tag , attrs ) :
    """Return representation of html start tag and attributes ."""

implementation:
if tag in self . mathml_elements :
    final_attr = ""
    for key , value in attrs :
        final_attr += ' {0}="{1}"' . format ( key , value )
    self . fed . append ( "<{0}{1}>" . format ( tag , final_attr ) )


signature:
def handle_endtag ( self , tag ) :
    """Return representation of html end tag ."""

implementation:
if tag in self . mathml_elements :
    self . fed . append ( "</{0}>" . format ( tag ) )


signature:
def html_to_text ( cls , html ) :
    """Return stripped HTML , keeping only MathML ."""

implementation:
s = cls ( )
s . feed ( html )
unescaped_data = s . unescape ( s . get_data ( ) )
return escape_for_xml ( unescaped_data , tags_to_keep = s . mathml_elements )


signature:
def is_instance ( self ) :
    """return True if callback is an instance of a class"""

implementation:
ret = False
val = self . callback
if self . is_class ( ) :
    return False
ret = not inspect . isfunction ( val ) and not inspect . ismethod ( val )
# if is _ py2:
# ret = isinstance ( val , types . InstanceType ) or hasattr ( val , ' _ _ dict _ _ ' ) \
# and not ( hasattr ( val , ' func _ name ' ) or hasattr ( val , ' im _ func ' ) )
# else :
# ret = not inspect . isfunction ( val ) and not inspect . ismethod ( val )
return ret


signature:
def is_function ( self ) :
    """return True if callback is a vanilla plain jane function"""

implementation:
if self . is_instance ( ) or self . is_class ( ) :
    return False
return isinstance ( self . callback , ( Callable , classmethod ) )


signature:
def merge_kwargs ( self , kwargs ) :
    """these kwargs come from the @ arg decorator , they are then merged into any
 keyword arguments that were automatically generated from the main function
 introspection"""

implementation:
if kwargs :
    self . parser_kwargs . update ( kwargs )
# self . parser _ kwargs [ ' dest ' ] = self . name
self . parser_kwargs . setdefault ( 'dest' , self . name )
# special handling of any passed in values
if 'default' in kwargs : # NOTE - - this doesn ' t use . set _ default ( ) because that is meant to
# parse from the function definition so it actually has different syntax
# than what the . set _ default ( ) method does . eg , @ arg ( " - - foo " , default = [ 1 , 2 ] ) means
# that the default value should be an array with 1 and 2 in it , where main ( foo = [ 1 , 2 ] )
# means foo should be constrained to choices = [ 1 , 2]
    self . parser_kwargs [ "default" ] = kwargs [ "default" ]
    self . parser_kwargs [ "required" ] = False
elif 'action' in kwargs :
    if kwargs [ 'action' ] in set ( [ 'store_false' , 'store_true' ] ) :
        self . parser_kwargs [ 'required' ] = False
    elif kwargs [ 'action' ] in set ( [ 'version' ] ) :
        self . parser_kwargs . pop ( 'required' , False )
else :
    self . parser_kwargs . setdefault ( "required" , True )


signature:
def set_default ( self , na ) :
    """this is used for introspection from the main ( ) method when there is an
 argument with a default value , this figures out how to set up the ArgParse
 arguments"""

implementation:
kwargs = { }
if isinstance ( na , ( type , types . FunctionType ) ) : # if foo = some _ func then some _ func ( foo ) will be ran if foo is passed in
    kwargs [ 'type' ] = na
    kwargs [ 'required' ] = True
    kwargs [ "default" ] = argparse . SUPPRESS
elif isinstance ( na , bool ) : # if false then passing - - foo will set to true , if True then - - foo will
# set foo to False
    kwargs [ 'action' ] = 'store_false' if na else 'store_true'
    kwargs [ 'required' ] = False
elif isinstance ( na , ( int , float , str ) ) : # for things like foo = int , this says that any value of foo is an integer
    kwargs [ 'type' ] = type ( na )
    kwargs [ 'default' ] = na
    kwargs [ 'required' ] = False
elif isinstance ( na , ( list , set ) ) : # list is strange , [ int ] would mean we want a list of all integers , if
# there is a value in the list : [ " foo " , " bar " ] then it would mean only
# those choices are valid
    na = list ( na )
    kwargs [ 'action' ] = 'append'
    kwargs [ 'required' ] = True
    if len ( na ) > 0 :
        if isinstance ( na [ 0 ] , type ) :
            kwargs [ 'type' ] = na [ 0 ]
        else : # we are now reverting this to a choices check
            kwargs [ 'action' ] = 'store'
            l = set ( )
            ltype = None
            for elt in na :
                vtype = type ( elt )
                l . add ( elt )
                if ltype is None :
                    ltype = vtype
                else :
                    if ltype is not vtype :
                        ltype = str
            kwargs [ 'choices' ] = l
            kwargs [ 'type' ] = ltype
# self . merge _ kwargs ( kwargs )
self . parser_kwargs . update ( kwargs )


signature:
def _fill_text ( self , text , width , indent ) :
    """Overridden to not get rid of newlines
 https : / / github . com / python / cpython / blob / 2.7 / Lib / argparse . py # L620"""

implementation:
lines = [ ]
for line in text . splitlines ( False ) :
    if line : # https : / / docs . python . org / 2 / library / textwrap . html
        lines . extend ( textwrap . wrap ( line . strip ( ) , width , initial_indent = indent , subsequent_indent = indent ) )
    else :
        lines . append ( line )
text = "\n" . join ( lines )
return text


signature:
def parse_callback_args ( self , raw_args ) :
    """This is the method that is called from Script . run ( ) , this is the insertion
 point for parsing all the arguments though on init this will find all args it
 can , so this method pulls already found args from class variables"""

implementation:
args = [ ]
arg_info = self . arg_info
kwargs = dict ( arg_info [ 'optional' ] )
parsed_args = [ ]
unknown_args = getattr ( self , "unknown_args" , False )
if unknown_args :
    parsed_args , parsed_unknown_args = self . parse_known_args ( raw_args )
    # TODO - - can this be moved to UnknownParser ?
    # * * kwargs have to be in - - key = val form
    # http : / / stackoverflow . com / a / 12807809/5006
    d = defaultdict ( list )
    for k , v in ( ( k . lstrip ( '-' ) , v ) for k , v in ( a . split ( '=' ) for a in parsed_unknown_args ) ) :
        d [ k ] . append ( v )
    for k in ( k for k in d if len ( d [ k ] ) == 1 ) :
        d [ k ] = d [ k ] [ 0 ]
    kwargs . update ( d )
else :
    parsed_args = self . parse_args ( raw_args )
# http : / / parezcoydigo . wordpress . com / 2012/08/04 / from - argparse - to - dictionary - in - python - 2-7/
kwargs . update ( vars ( parsed_args ) )
# because of how args works , we need to make sure the kwargs are put in correct
# order to be passed to the function , otherwise our real * args won ' t make it
# to the * args variable
for k in arg_info [ 'order' ] :
    args . append ( kwargs . pop ( k ) )
# now that we have the correct order , tack the real * args on the end so they
# get correctly placed into the function ' s * args variable
if arg_info [ 'args' ] :
    args . extend ( kwargs . pop ( arg_info [ 'args' ] ) )
return args , kwargs


signature:
def find_args ( self ) :
    """when a new parser is created this is the method that is called from its
 _ _ init _ _ method to find all the arguments"""

implementation:
arg_info = self . arg_info
main = self . callback
cbi = CallbackInspect ( main )
all_arg_names = set ( )
decorator_args = cbi . args
args , args_name , kwargs_name , args_defaults = cbi . argspec
arg_info [ 'order' ] = args
default_offset = len ( args ) - len ( args_defaults )
# pout . v ( args , args _ name , kwargs _ name , args _ defaults , default _ offset )
# pout . v ( args , decorator _ args )
# build a list of potential * args , basically , if an arg _ name matches exactly
# then it is an * arg and we shouldn ' t mess with it in the function
comp_args = set ( )
for da in decorator_args :
    comp_args . update ( da [ 0 ] )
for i , arg_name in enumerate ( args ) :
    if arg_name in comp_args :
        continue
    a = ScriptKwarg ( arg_name )
    # set the default if it is available
    default_i = i - default_offset
    if default_i >= 0 :
        na = args_defaults [ default_i ]
        a . set_default ( na )
    a . merge_from_list ( decorator_args )
    if a . required :
        arg_info [ 'required' ] . append ( a . name )
    else :
        arg_info [ 'optional' ] [ a . name ] = a . default
    # pout . v ( a . parser _ args , a . parser _ kwargs )
    all_arg_names |= a . parser_args
    # if the callback arg is just a value , respect the parent parser ' s config
    if "default" not in a . parser_kwargs and "action" not in a . parser_kwargs and "choices" not in a . parser_kwargs :
        keys = self . _option_string_actions . keys ( )
        found_arg = False
        for pa in a . parser_args :
            if pa in keys :
                found_arg = True
                break
        if not found_arg :
            self . add_argument ( * a . parser_args , ** a . parser_kwargs )
    else : # we want to override parent parser
        self . add_argument ( * a . parser_args , ** a . parser_kwargs )
self . unknown_args = False
if self . add_help :
    if args_name :
        a = ScriptArg ( args_name , nargs = '*' )
        a . merge_from_list ( decorator_args )
        all_arg_names |= a . parser_args
        self . add_argument ( * a . parser_args , ** a . parser_kwargs )
        arg_info [ 'args' ] = args_name
    if kwargs_name :
        self . unknown_args = True
        arg_info [ 'kwargs' ] = kwargs_name
# pick up any stragglers
for da , dkw in decorator_args :
    if da [ 0 ] not in all_arg_names :
        arg_name = da [ 0 ]
        if arg_name . startswith ( "-" ) :
            a = ScriptKwarg ( * da )
        else :
            a = ScriptArg ( * da )
        a . merge_kwargs ( dkw )
        self . add_argument ( * a . parser_args , ** a . parser_kwargs )
self . arg_info = arg_info


signature:
def normalize_quiet_arg ( self , arg_strings ) :
    """This is a hack to allow ` - - quiet ` and ` - - quiet DI ` to work correctly ,
 basically it goes through all arg _ strings and if it finds - - quiet it checks
 the next argument to see if it is some combination of DIWEC , if it is then
 it combines it to ` - - quiet = ARG ` and returns the modified arg _ strings list
 : param arg _ strings : list , the raw arguments
 : returns : list , the arg _ strings changed if needed"""

implementation:
if not self . has_injected_quiet ( ) :
    return arg_strings
action = self . _option_string_actions . get ( self . quiet_flags [ 0 ] )
if action :
    count = len ( arg_strings )
    new_args = [ ]
    i = 0
    while i < count :
        arg_string = arg_strings [ i ]
        if arg_string in action . option_strings :
            if ( i + 1 ) < count :
                narg_string = arg_strings [ i + 1 ]
                if narg_string in self . _option_string_actions : # make sure a flag like - D isn ' t mistaken for a
                # - - quiet value
                    new_args . append ( "{}={}" . format ( arg_string , action . const ) )
                elif re . match ( r"^\-?[{}]+$" . format ( action . const ) , narg_string ) :
                    new_args . append ( "{}={}" . format ( arg_string , narg_string ) )
                    i += 1
                else :
                    new_args . append ( "{}={}" . format ( arg_string , action . const ) )
            else :
                new_args . append ( "{}={}" . format ( arg_string , action . const ) )
        else :
            new_args . append ( arg_string )
        i += 1
    arg_strings = new_args
return arg_strings


signature:
def make_user_agent ( component = None ) :
    """create string suitable for HTTP User - Agent header"""

implementation:
packageinfo = pkg_resources . require ( "harvestingkit" ) [ 0 ]
useragent = "{0}/{1}" . format ( packageinfo . project_name , packageinfo . version )
if component is not None :
    useragent += " {0}" . format ( component )
return useragent


signature:
def record_add_field ( rec , tag , ind1 = '' , ind2 = '' , subfields = [ ] , controlfield_value = '' ) :
    """Add a MARCXML datafield as a new child to a XML document ."""

implementation:
if controlfield_value :
    doc = etree . Element ( "controlfield" , attrib = { "tag" : tag , } )
    doc . text = unicode ( controlfield_value )
else :
    doc = etree . Element ( "datafield" , attrib = { "tag" : tag , "ind1" : ind1 , "ind2" : ind2 , } )
    for code , value in subfields :
        field = etree . SubElement ( doc , "subfield" , attrib = { "code" : code } )
        field . text = value
rec . append ( doc )
return rec


signature:
def record_xml_output ( rec , pretty = True ) :
    """Given a document , return XML prettified ."""

implementation:
from . html_utils import MathMLParser
ret = etree . tostring ( rec , xml_declaration = False )
# Special MathML handling
ret = re . sub ( "(<)(([\/]?{0}))" . format ( "|[\/]?" . join ( MathMLParser . mathml_elements ) ) , '<\g<2>' , ret )
ret = re . sub ( ">" , '>' , ret )
if pretty : # We are doing our own prettyfication as etree pretty _ print is too insane .
    ret = ret . replace ( '</datafield>' , '  </datafield>\n' )
    ret = re . sub ( r'<datafield(.*?)>' , r'  <datafield\1>\n' , ret )
    ret = ret . replace ( '</subfield>' , '</subfield>\n' )
    ret = ret . replace ( '<subfield' , '    <subfield' )
    ret = ret . replace ( 'record>' , 'record>\n' )
return ret


signature:
def escape_for_xml ( data , tags_to_keep = None ) :
    """Transform & and < to XML valid & amp ; and & lt .
 Pass a list of tags as string to enable replacement of
 ' < ' globally but keep any XML tags in the list ."""

implementation:
data = re . sub ( "&" , "&" , data )
if tags_to_keep :
    data = re . sub ( r"(<)(?![\/]?({0})\b)" . format ( "|" . join ( tags_to_keep ) ) , '<' , data )
else :
    data = re . sub ( "<" , "<" , data )
return data


signature:
def format_arxiv_id ( arxiv_id ) :
    """Properly format arXiv IDs ."""

implementation:
if arxiv_id and "/" not in arxiv_id and "arXiv" not in arxiv_id :
    return "arXiv:%s" % ( arxiv_id , )
elif arxiv_id and '.' not in arxiv_id and arxiv_id . lower ( ) . startswith ( 'arxiv:' ) :
    return arxiv_id [ 6 : ]
    # strip away arxiv : for old identifiers
else :
    return arxiv_id


signature:
def collapse_initials ( name ) :
    """Remove the space between initials , eg T . A . - - > T . A ."""


signature:
def fix_journal_name ( journal , knowledge_base ) :
    """Convert journal name to Inspire ' s short form ."""

implementation:
if not journal :
    return '' , ''
if not knowledge_base :
    return journal , ''
if len ( journal ) < 2 :
    return journal , ''
volume = ''
if ( journal [ - 1 ] <= 'Z' and journal [ - 1 ] >= 'A' ) and ( journal [ - 2 ] == '.' or journal [ - 2 ] == ' ' ) :
    volume += journal [ - 1 ]
    journal = journal [ : - 1 ]
journal = journal . strip ( )
if journal . upper ( ) in knowledge_base :
    journal = knowledge_base [ journal . upper ( ) ] . strip ( )
elif journal in knowledge_base :
    journal = knowledge_base [ journal ] . strip ( )
elif '.' in journal :
    journalnodots = journal . replace ( '. ' , ' ' )
    journalnodots = journalnodots . replace ( '.' , ' ' ) . strip ( ) . upper ( )
    if journalnodots in knowledge_base :
        journal = knowledge_base [ journalnodots ] . strip ( )
journal = journal . replace ( '. ' , '.' )
return journal , volume


signature:
def add_nations_field ( authors_subfields ) :
    """Add correct nations field according to mapping in NATIONS _ DEFAULT _ MAP ."""

implementation:
from . config import NATIONS_DEFAULT_MAP
result = [ ]
for field in authors_subfields :
    if field [ 0 ] == 'v' :
        values = [ x . replace ( '.' , '' ) for x in field [ 1 ] . split ( ', ' ) ]
        possible_affs = filter ( lambda x : x is not None , map ( NATIONS_DEFAULT_MAP . get , values ) )
        if 'CERN' in possible_affs and 'Switzerland' in possible_affs : # Don ' t use remove in case of multiple Switzerlands
            possible_affs = [ x for x in possible_affs if x != 'Switzerland' ]
        result . extend ( possible_affs )
result = sorted ( list ( set ( result ) ) )
if result :
    authors_subfields . extend ( [ ( 'w' , res ) for res in result ] )
else :
    authors_subfields . append ( ( 'w' , 'HUMAN CHECK' ) )


signature:
def fix_dashes ( string ) :
    """Fix bad Unicode special dashes in string ."""

implementation:
string = string . replace ( u'\u05BE' , '-' )
string = string . replace ( u'\u1806' , '-' )
string = string . replace ( u'\u2E3A' , '-' )
string = string . replace ( u'\u2E3B' , '-' )
string = unidecode ( string )
return re . sub ( r'--+' , '-' , string )


signature:
def fix_title_capitalization ( title ) :
    """Try to capitalize properly a title string ."""

implementation:
if re . search ( "[A-Z]" , title ) and re . search ( "[a-z]" , title ) :
    return title
word_list = re . split ( ' +' , title )
final = [ word_list [ 0 ] . capitalize ( ) ]
for word in word_list [ 1 : ] :
    if word . upper ( ) in COMMON_ACRONYMS :
        final . append ( word . upper ( ) )
    elif len ( word ) > 3 :
        final . append ( word . capitalize ( ) )
    else :
        final . append ( word . lower ( ) )
return " " . join ( final )


signature:
def convert_html_subscripts_to_latex ( text ) :
    """Convert some HTML tags to latex equivalents ."""


signature:
def download_file ( from_url , to_filename = None , chunk_size = 1024 * 8 , retry_count = 3 ) :
    """Download URL to a file ."""

implementation:
if not to_filename :
    to_filename = get_temporary_file ( )
session = requests . Session ( )
adapter = requests . adapters . HTTPAdapter ( max_retries = retry_count )
session . mount ( from_url , adapter )
response = session . get ( from_url , stream = True )
with open ( to_filename , 'wb' ) as fd :
    for chunk in response . iter_content ( chunk_size ) :
        fd . write ( chunk )
return to_filename


signature:
def run_shell_command ( commands , ** kwargs ) :
    """Run a shell command ."""

implementation:
p = subprocess . Popen ( commands , stdout = subprocess . PIPE , stderr = subprocess . PIPE , ** kwargs )
output , error = p . communicate ( )
return p . returncode , output , error


signature:
def create_logger ( name , filename = None , logging_level = logging . DEBUG ) :
    """Create a logger object ."""

implementation:
logger = logging . getLogger ( name )
formatter = logging . Formatter ( ( '%(asctime)s - %(name)s - ' '%(levelname)-8s - %(message)s' ) )
if filename :
    fh = logging . FileHandler ( filename = filename )
    fh . setFormatter ( formatter )
    logger . addHandler ( fh )
ch = logging . StreamHandler ( )
ch . setFormatter ( formatter )
logger . addHandler ( ch )
logger . setLevel ( logging_level )
return logger