signature:
def unzip ( zipped_file , output_directory = None , prefix = "harvestingkit_unzip_" , suffix = "" ) :
    """Uncompress a zipped file from given filepath to an ( optional ) location .
 If no location is given , a temporary folder will be generated inside
 CFG _ TMPDIR , prefixed with " apsharvest _ unzip _ " ."""

implementation:
if not output_directory : # We create a temporary directory to extract our stuff in
    try :
        output_directory = mkdtemp ( suffix = suffix , prefix = prefix )
    except Exception , e :
        try :
            os . removedirs ( output_directory )
        except TypeError :
            pass
        raise e
return _do_unzip ( zipped_file , output_directory )


signature:
def _do_unzip ( zipped_file , output_directory ) :
    """Perform the actual uncompression ."""

implementation:
z = zipfile . ZipFile ( zipped_file )
for path in z . namelist ( ) :
    relative_path = os . path . join ( output_directory , path )
    dirname , dummy = os . path . split ( relative_path )
    try :
        if relative_path . endswith ( os . sep ) and not os . path . exists ( dirname ) :
            os . makedirs ( relative_path )
        elif not os . path . exists ( relative_path ) :
            dirname = os . path . join ( output_directory , os . path . dirname ( path ) )
            if os . path . dirname ( path ) and not os . path . exists ( dirname ) :
                os . makedirs ( dirname )
            fd = open ( relative_path , "w" )
            fd . write ( z . read ( path ) )
            fd . close ( )
    except IOError , e :
        raise e
return output_directory


signature:
def locate ( pattern , root = os . curdir ) :
    """Locate all files matching supplied filename pattern recursively ."""

implementation:
for path , dummy , files in os . walk ( os . path . abspath ( root ) ) :
    for filename in fnmatch . filter ( files , pattern ) :
        yield os . path . join ( path , filename )


signature:
def punctuate_authorname ( an ) :
    """Punctuate author names properly .
 Expects input in the form ' Bloggs , J K ' and will return ' Bloggs , J . K . ' ."""

implementation:
name = an . strip ( )
parts = [ x for x in name . split ( ',' ) if x != '' ]
ret_str = ''
for idx , part in enumerate ( parts ) :
    subparts = part . strip ( ) . split ( ' ' )
    for sidx , substr in enumerate ( subparts ) :
        ret_str += substr
        if len ( substr ) == 1 :
            ret_str += '.'
        if sidx < ( len ( subparts ) - 1 ) :
            ret_str += ' '
    if idx < ( len ( parts ) - 1 ) :
        ret_str += ', '
return ret_str . strip ( )


signature:
def convert_date_to_iso ( value ) :
    """Convert a date - value to the ISO date standard ."""

implementation:
date_formats = [ "%d %b %Y" , "%Y/%m/%d" ]
for dformat in date_formats :
    try :
        date = datetime . strptime ( value , dformat )
        return date . strftime ( "%Y-%m-%d" )
    except ValueError :
        pass
return value


signature:
def convert_date_from_iso_to_human ( value ) :
    """Convert a date - value to the ISO date standard for humans ."""

implementation:
try :
    year , month , day = value . split ( "-" )
except ValueError : # Not separated by " - " . Space ?
    try :
        year , month , day = value . split ( " " )
    except ValueError : # What gives ? OK , lets just return as is
        return value
try :
    date_object = datetime ( int ( year ) , int ( month ) , int ( day ) )
except TypeError :
    return value
return date_object . strftime ( "%d %b %Y" )


signature:
def convert_images ( image_list ) :
    """Convert list of images to PNG format .
 @ param : image _ list ( [ string , string , . . . ] ) : the list of image files
 extracted from the tarball in step 1
 @ return : image _ list ( [ str , str , . . . ] ) : The list of image files when all
 have been converted to PNG format ."""

implementation:
png_output_contains = 'PNG image'
ret_list = [ ]
for image_file in image_list :
    if os . path . isdir ( image_file ) :
        continue
    dummy1 , cmd_out , dummy2 = run_shell_command ( 'file %s' , ( image_file , ) )
    if cmd_out . find ( png_output_contains ) > - 1 :
        ret_list . append ( image_file )
    else : # we ' re just going to assume that ImageMagick can convert all
    # the image types that we may be faced with
    # for sure it can do EPS - > PNG and JPG - > PNG and PS - > PNG
    # and PSTEX - > PNG
        converted_image_file = get_converted_image_name ( image_file )
        cmd_list = [ 'convert' , image_file , converted_image_file ]
        dummy1 , cmd_out , cmd_err = run_shell_command ( cmd_list )
        if cmd_err == '' :
            ret_list . append ( converted_image_file )
        else :
            raise Exception ( cmd_err )
return ret_list


signature:
def get_temporary_file ( prefix = "tmp_" , suffix = "" , directory = None ) :
    """Generate a safe and closed filepath ."""

implementation:
try :
    file_fd , filepath = mkstemp ( prefix = prefix , suffix = suffix , dir = directory )
    os . close ( file_fd )
except IOError , e :
    try :
        os . remove ( filepath )
    except Exception :
        pass
    raise e
return filepath


signature:
def return_letters_from_string ( text ) :
    """Get letters from string only ."""

implementation:
out = ""
for letter in text :
    if letter . isalpha ( ) :
        out += letter
return out


signature:
def license_is_oa ( license ) :
    """Return True if license is compatible with Open Access"""

implementation:
for oal in OA_LICENSES :
    if re . search ( oal , license ) :
        return True
return False


signature:
def _extract_package ( self ) :
    """Extract a package in a new temporary directory ."""

implementation:
self . path = mkdtemp ( prefix = "scoap3_package_" , dir = CFG_TMPSHAREDDIR )
self . logger . debug ( "Extracting package: %s" % ( self . package_name , ) )
scoap3utils_extract_package ( self . package_name , self . path , self . logger )


signature:
def _crawl_elsevier_and_find_main_xml ( self ) :
    """A package contains several subdirectory corresponding to each article .
 An article is actually identified by the existence of a main . pdf and
 a main . xml in a given directory ."""

implementation:
self . found_articles = [ ]
if not self . path and not self . package_name :
    for doc in self . conn . found_articles :
        dirname = doc [ 'xml' ] . rstrip ( '/main.xml' )
        try :
            self . _normalize_article_dir_with_dtd ( dirname )
            self . found_articles . append ( dirname )
        except Exception as err :
            register_exception ( )
            print ( "ERROR: can't normalize %s: %s" % ( dirname , err ) )
else :
    def visit ( dummy , dirname , names ) :
        if "main.xml" in names and "main.pdf" in names :
            try :
                self . _normalize_article_dir_with_dtd ( dirname )
                self . found_articles . append ( dirname )
            except Exception as err :
                register_exception ( )
                print ( "ERROR: can't normalize %s: %s" % ( dirname , err ) )
    walk ( self . path , visit , None )


signature:
def _crawl_elsevier_and_find_issue_xml ( self ) :
    """Information about the current volume , issue , etc . is available
 in a file called issue . xml that is available in a higher directory ."""

implementation:
self . _found_issues = [ ]
if not self . path and not self . package_name :
    for issue in self . conn . _get_issues ( ) :
        dirname = issue . rstrip ( '/issue.xml' )
        try :
            self . _normalize_issue_dir_with_dtd ( dirname )
            self . _found_issues . append ( dirname )
        except Exception as err :
            register_exception ( )
            print ( "ERROR: can't normalize %s: %s" % ( dirname , err ) )
else :
    def visit ( dummy , dirname , names ) :
        if "issue.xml" in names :
            try :
                self . _normalize_issue_dir_with_dtd ( dirname )
                self . _found_issues . append ( dirname )
            except Exception as err :
                register_exception ( )
                print ( "ERROR: can't normalize %s: %s" % ( dirname , err ) )
    walk ( self . path , visit , None )


signature:
def _normalize_issue_dir_with_dtd ( self , path ) :
    """issue . xml from Elsevier assume the existence of a local DTD .
 This procedure install the DTDs next to the issue . xml file
 and normalize it using xmllint in order to resolve all namespaces
 and references ."""

implementation:
if exists ( join ( path , 'resolved_issue.xml' ) ) :
    return
issue_xml_content = open ( join ( path , 'issue.xml' ) ) . read ( )
sis = [ 'si510.dtd' , 'si520.dtd' , 'si540.dtd' ]
tmp_extracted = 0
for si in sis :
    if si in issue_xml_content :
        self . _extract_correct_dtd_package ( si . split ( '.' ) [ 0 ] , path )
        tmp_extracted = 1
if not tmp_extracted :
    message = "It looks like the path " + path
    message += " does not contain an si510, si520 or si540 in issue.xml file"
    self . logger . error ( message )
    raise ValueError ( message )
command = [ "xmllint" , "--format" , "--loaddtd" , join ( path , 'issue.xml' ) , "--output" , join ( path , 'resolved_issue.xml' ) ]
dummy , dummy , cmd_err = run_shell_command ( command )
if cmd_err :
    message = "Error in cleaning %s: %s" % ( join ( path , 'issue.xml' ) , cmd_err )
    self . logger . error ( message )
    raise ValueError ( message )


signature:
def _normalize_article_dir_with_dtd ( self , path ) :
    """main . xml from Elsevier assume the existence of a local DTD .
 This procedure install the DTDs next to the main . xml file
 and normalize it using xmllint in order to resolve all namespaces
 and references ."""

implementation:
if exists ( join ( path , 'resolved_main.xml' ) ) :
    return
main_xml_content = open ( join ( path , 'main.xml' ) ) . read ( )
arts = [ 'art501.dtd' , 'art510.dtd' , 'art520.dtd' , 'art540.dtd' ]
tmp_extracted = 0
for art in arts :
    if art in main_xml_content :
        self . _extract_correct_dtd_package ( art . split ( '.' ) [ 0 ] , path )
        tmp_extracted = 1
if not tmp_extracted :
    message = "It looks like the path " + path
    message += "does not contain an art501, art510, art520 or art540 in main.xml file"
    self . logger . error ( message )
    raise ValueError ( message )
command = [ "xmllint" , "--format" , "--loaddtd" , join ( path , 'main.xml' ) , "--output" , join ( path , 'resolved_main.xml' ) ]
dummy , dummy , cmd_err = run_shell_command ( command )
if cmd_err :
    message = "Error in cleaning %s: %s" % ( join ( path , 'main.xml' ) , cmd_err )
    self . logger . error ( message )
    raise ValueError ( message )


signature:
def get_publication_date ( self , xml_doc ) :
    """Return the best effort start _ date ."""

implementation:
start_date = get_value_in_tag ( xml_doc , "prism:coverDate" )
if not start_date :
    start_date = get_value_in_tag ( xml_doc , "prism:coverDisplayDate" )
    if not start_date :
        start_date = get_value_in_tag ( xml_doc , 'oa:openAccessEffective' )
        if start_date :
            start_date = datetime . datetime . strptime ( start_date , "%Y-%m-%dT%H:%M:%SZ" )
            return start_date . strftime ( "%Y-%m-%d" )
    import dateutil . parser
    # dateutil . parser . parse cant process dates like April - June 2016
    start_date = re . sub ( '([A-Z][a-z]+)[\s\-][A-Z][a-z]+ (\d{4})' , r'\1 \2' , start_date )
    try :
        date = dateutil . parser . parse ( start_date )
    except ValueError :
        return ''
    # Special case where we ignore the deduced day form dateutil
    # in case it was not given in the first place .
    if len ( start_date . split ( " " ) ) == 3 :
        return date . strftime ( "%Y-%m-%d" )
    else :
        return date . strftime ( "%Y-%m" )
else :
    if len ( start_date ) is 8 :
        start_date = time . strftime ( '%Y-%m-%d' , time . strptime ( start_date , '%Y%m%d' ) )
    elif len ( start_date ) is 6 :
        start_date = time . strftime ( '%Y-%m' , time . strptime ( start_date , '%Y%m' ) )
    return start_date


signature:
def get_record ( self , path = None , no_pdf = False , test = False , refextract_callback = None ) :
    """Convert a record to MARCXML format .
 : param path : path to a record .
 : type path : string
 : param test : flag to determine if it is a test call .
 : type test : bool
 : param refextract _ callback : callback to be used to extract
 unstructured references . It should
 return a marcxml formated string
 of the reference .
 : type refextract _ callback : callable
 : returns : marcxml formated string ."""

implementation:
xml_doc = self . get_article ( path )
rec = create_record ( )
title = self . get_title ( xml_doc )
if title :
    record_add_field ( rec , '245' , subfields = [ ( 'a' , title ) ] )
( journal , dummy , volume , issue , first_page , last_page , year , start_date , doi ) = self . get_publication_information ( xml_doc , path )
if not journal :
    journal = self . get_article_journal ( xml_doc )
if start_date :
    record_add_field ( rec , '260' , subfields = [ ( 'c' , start_date ) , ( 't' , 'published' ) ] )
else :
    record_add_field ( rec , '260' , subfields = [ ( 'c' , time . strftime ( '%Y-%m-%d' ) ) ] )
if doi :
    record_add_field ( rec , '024' , ind1 = '7' , subfields = [ ( 'a' , doi ) , ( '2' , 'DOI' ) ] )
license , license_url = self . get_license ( xml_doc )
if license and license_url :
    record_add_field ( rec , '540' , subfields = [ ( 'a' , license ) , ( 'u' , license_url ) ] )
elif license_url :
    record_add_field ( rec , '540' , subfields = [ ( 'u' , license_url ) ] )
self . logger . info ( "Creating record: %s %s" % ( path , doi ) )
authors = self . get_authors ( xml_doc )
first_author = True
for author in authors :
    author_name = ( author [ 'surname' ] , author . get ( 'given_name' ) or author . get ( 'initials' ) )
    subfields = [ ( 'a' , '%s, %s' % author_name ) ]
    if 'orcid' in author :
        subfields . append ( ( 'j' , author [ 'orcid' ] ) )
    if 'affiliation' in author :
        for aff in author [ "affiliation" ] :
            subfields . append ( ( 'v' , aff ) )
        if self . extract_nations :
            add_nations_field ( subfields )
    if author . get ( 'email' ) :
        subfields . append ( ( 'm' , author [ 'email' ] ) )
    if first_author :
        record_add_field ( rec , '100' , subfields = subfields )
        first_author = False
    else :
        record_add_field ( rec , '700' , subfields = subfields )
abstract = self . get_abstract ( xml_doc )
if abstract :
    record_add_field ( rec , '520' , subfields = [ ( 'a' , abstract ) , ( '9' , 'Elsevier' ) ] )
record_copyright = self . get_copyright ( xml_doc )
if record_copyright :
    record_add_field ( rec , '542' , subfields = [ ( 'f' , record_copyright ) ] )
keywords = self . get_keywords ( xml_doc )
if self . CONSYN :
    for tag in xml_doc . getElementsByTagName ( 'ce:collaboration' ) :
        collaboration = get_value_in_tag ( tag , 'ce:text' )
        if collaboration :
            record_add_field ( rec , '710' , subfields = [ ( 'g' , collaboration ) ] )
    # We add subjects also as author keywords
    subjects = xml_doc . getElementsByTagName ( 'dct:subject' )
    for subject in subjects :
        for listitem in subject . getElementsByTagName ( 'rdf:li' ) :
            keyword = xml_to_text ( listitem )
            if keyword not in keywords :
                keywords . append ( keyword )
    for keyword in keywords :
        record_add_field ( rec , '653' , ind1 = '1' , subfields = [ ( 'a' , keyword ) , ( '9' , 'author' ) ] )
    journal , dummy = fix_journal_name ( journal . strip ( ) , self . journal_mappings )
    subfields = [ ]
    doctype = self . get_doctype ( xml_doc )
    try :
        page_count = int ( last_page ) - int ( first_page ) + 1
        record_add_field ( rec , '300' , subfields = [ ( 'a' , str ( page_count ) ) ] )
    except ValueError : # do nothing
        pass
    if doctype == 'err' :
        subfields . append ( ( 'm' , 'Erratum' ) )
    elif doctype == 'add' :
        subfields . append ( ( 'm' , 'Addendum' ) )
    elif doctype == 'pub' :
        subfields . append ( ( 'm' , 'Publisher Note' ) )
    elif doctype == 'rev' :
        record_add_field ( rec , '980' , subfields = [ ( 'a' , 'Review' ) ] )
    if journal :
        subfields . append ( ( 'p' , journal ) )
    if first_page and last_page :
        subfields . append ( ( 'c' , '%s-%s' % ( first_page , last_page ) ) )
    elif first_page :
        subfields . append ( ( 'c' , first_page ) )
    if volume :
        subfields . append ( ( 'v' , volume ) )
    if year :
        subfields . append ( ( 'y' , year ) )
    record_add_field ( rec , '773' , subfields = subfields )
    if not test :
        if license :
            url = 'http://www.sciencedirect.com/science/article/pii/' + path . split ( '/' ) [ - 1 ] [ : - 4 ]
            record_add_field ( rec , '856' , ind1 = '4' , subfields = [ ( 'u' , url ) , ( 'y' , 'Elsevier server' ) ] )
            record_add_field ( rec , 'FFT' , subfields = [ ( 'a' , path ) , ( 't' , 'INSPIRE-PUBLIC' ) , ( 'd' , 'Fulltext' ) ] )
        else :
            record_add_field ( rec , 'FFT' , subfields = [ ( 'a' , path ) , ( 't' , 'Elsevier' ) , ( 'o' , 'HIDDEN' ) ] )
    record_add_field ( rec , '980' , subfields = [ ( 'a' , 'HEP' ) ] )
    record_add_field ( rec , '980' , subfields = [ ( 'a' , 'Citeable' ) ] )
    record_add_field ( rec , '980' , subfields = [ ( 'a' , 'Published' ) ] )
    self . _add_references ( xml_doc , rec , refextract_callback )
else :
    licence = 'http://creativecommons.org/licenses/by/3.0/'
    record_add_field ( rec , '540' , subfields = [ ( 'a' , 'CC-BY-3.0' ) , ( 'u' , licence ) ] )
    if keywords :
        for keyword in keywords :
            record_add_field ( rec , '653' , ind1 = '1' , subfields = [ ( 'a' , keyword ) , ( '9' , 'author' ) ] )
    pages = ''
    if first_page and last_page :
        pages = '{0}-{1}' . format ( first_page , last_page )
    elif first_page :
        pages = first_page
    subfields = filter ( lambda x : x [ 1 ] and x [ 1 ] != '-' , [ ( 'p' , journal ) , ( 'v' , volume ) , ( 'n' , issue ) , ( 'c' , pages ) , ( 'y' , year ) ] )
    record_add_field ( rec , '773' , subfields = subfields )
    if not no_pdf :
        from invenio . search_engine import perform_request_search
        query = '0247_a:"%s" AND NOT 980:DELETED"' % ( doi , )
        prev_version = perform_request_search ( p = query )
        old_pdf = False
        if prev_version :
            from invenio . bibdocfile import BibRecDocs
            prev_rec = BibRecDocs ( prev_version [ 0 ] )
            try :
                pdf_path = prev_rec . get_bibdoc ( 'main' )
                pdf_path = pdf_path . get_file ( ".pdf;pdfa" , exact_docformat = True )
                pdf_path = pdf_path . fullpath
                old_pdf = True
                record_add_field ( rec , 'FFT' , subfields = [ ( 'a' , pdf_path ) , ( 'n' , 'main' ) , ( 'f' , '.pdf;pdfa' ) ] )
                message = ( 'Leaving previously delivered PDF/A for: ' + doi )
                self . logger . info ( message )
            except :
                pass
        try :
            if exists ( join ( path , 'main_a-2b.pdf' ) ) :
                pdf_path = join ( path , 'main_a-2b.pdf' )
                record_add_field ( rec , 'FFT' , subfields = [ ( 'a' , pdf_path ) , ( 'n' , 'main' ) , ( 'f' , '.pdf;pdfa' ) ] )
                self . logger . debug ( 'Adding PDF/A to record: %s' % ( doi , ) )
            elif exists ( join ( path , 'main.pdf' ) ) :
                pdf_path = join ( path , 'main.pdf' )
                record_add_field ( rec , 'FFT' , subfields = [ ( 'a' , pdf_path ) ] )
            else :
                if not old_pdf :
                    message = "Record " + doi
                    message += " doesn't contain PDF file."
                    self . logger . warning ( message )
                    raise MissingFFTError ( message )
        except MissingFFTError :
            message = "Elsevier paper: %s is missing PDF." % ( doi , )
            register_exception ( alert_admin = True , prefix = message )
        version = self . get_elsevier_version ( find_package_name ( path ) )
        record_add_field ( rec , '583' , subfields = [ ( 'l' , version ) ] )
        xml_path = join ( path , 'main.xml' )
        record_add_field ( rec , 'FFT' , subfields = [ ( 'a' , xml_path ) ] )
        record_add_field ( rec , '980' , subfields = [ ( 'a' , 'SCOAP3' ) , ( 'b' , 'Elsevier' ) ] )
try :
    return record_xml_output ( rec )
except UnicodeDecodeError :
    message = "Found a bad char in the file for the article " + doi
    sys . stderr . write ( message )
    return ""


signature:
def extract_oembeds ( text , args = None ) :
    """Extract oembed resources from a block of text . Returns a list
 of dictionaries .
 Max width & height can be specified :
 { % for embed in block _ of _ text | extract _ oembeds : " 400x300 " % }
 Resource type can be specified :
 { % for photo _ embed in block _ of _ text | extract _ oembeds : " photo " % }
 Or both :
 { % for embed in block _ of _ text | extract _ oembeds : " 400x300xphoto " % }"""

implementation:
resource_type = width = height = None
if args :
    dimensions = args . lower ( ) . split ( 'x' )
    if len ( dimensions ) in ( 3 , 1 ) :
        resource_type = dimensions . pop ( )
    if len ( dimensions ) == 2 :
        width , height = map ( lambda x : int ( x ) , dimensions )
client = OEmbedConsumer ( )
return client . extract ( text , width , height , resource_type )


signature:
def strip_oembeds ( text , args = None ) :
    """Take a block of text and strip all the embeds from it , optionally taking
 a maxwidth , maxheight / resource _ type
 Usage :
 { { post . content | strip _ embeds } }
 { { post . content | strip _ embeds : " 600x600xphoto " } }
 { { post . content | strip _ embeds : " video " } }"""

implementation:
resource_type = width = height = None
if args :
    dimensions = args . lower ( ) . split ( 'x' )
    if len ( dimensions ) in ( 3 , 1 ) :
        resource_type = dimensions . pop ( )
    if len ( dimensions ) == 2 :
        width , height = map ( lambda x : int ( x ) , dimensions )
client = OEmbedConsumer ( )
return mark_safe ( client . strip ( text , width , height , resource_type ) )


signature:
def do_oembed ( parser , token ) :
    """A node which parses everything between its two nodes , and replaces any links
 with OEmbed - provided objects , if possible .
 Supports two optional argument , which is the maximum width and height ,
 specified like so :
 { % oembed 640x480 % } http : / / www . viddler . com / explore / SYSTM / videos / 49 / { % endoembed % }
 and or the name of a sub tempalte directory to render templates from :
 { % oembed 320x240 in " comments " % } http : / / www . viddler . com / explore / SYSTM / videos / 49 / { % endoembed % }
 or :
 { % oembed in " comments " % } http : / / www . viddler . com / explore / SYSTM / videos / 49 / { % endoembed % }
 either of those will render templates in oembed / comments / oembedtype . html
 Additionally , you can specify a context variable to drop the rendered text in :
 { % oembed 600x400 in " comments " as var _ name % } . . . { % endoembed % }
 { % oembed as var _ name % } . . . { % endoembed % }"""

implementation:
args = token . split_contents ( )
template_dir = None
var_name = None
if len ( args ) > 2 :
    if len ( args ) == 3 and args [ 1 ] == 'in' :
        template_dir = args [ 2 ]
    elif len ( args ) == 3 and args [ 1 ] == 'as' :
        var_name = args [ 2 ]
    elif len ( args ) == 4 and args [ 2 ] == 'in' :
        template_dir = args [ 3 ]
    elif len ( args ) == 4 and args [ 2 ] == 'as' :
        var_name = args [ 3 ]
    elif len ( args ) == 6 and args [ 4 ] == 'as' :
        template_dir = args [ 3 ]
        var_name = args [ 5 ]
    else :
        raise template . TemplateSyntaxError ( "OEmbed either takes a single " "(optional) argument: WIDTHxHEIGHT, where WIDTH and HEIGHT " "are positive integers, and or an optional 'in " " \"template_dir\"' argument set." )
    if template_dir :
        if not ( template_dir [ 0 ] == template_dir [ - 1 ] and template_dir [ 0 ] in ( '"' , "'" ) ) :
            raise template . TemplateSyntaxError ( "template_dir must be quoted" )
        template_dir = template_dir [ 1 : - 1 ]
if len ( args ) >= 2 and 'x' in args [ 1 ] :
    width , height = args [ 1 ] . lower ( ) . split ( 'x' )
    if not width and height :
        raise template . TemplateSyntaxError ( "OEmbed's optional WIDTHxHEIGH" "T argument requires WIDTH and HEIGHT to be positive integers." )
else :
    width , height = None , None
nodelist = parser . parse ( ( 'endoembed' , ) )
parser . delete_first_token ( )
return OEmbedNode ( nodelist , width , height , template_dir , var_name )


signature:
def do_autodiscover ( parser , token ) :
    """Generates a & lt ; link & gt ; tag with oembed autodiscovery bits for an object .
 { % oembed _ autodiscover video % }"""

implementation:
args = token . split_contents ( )
if len ( args ) != 2 :
    raise template . TemplateSyntaxError ( '%s takes an object as its parameter.' % args [ 0 ] )
else :
    obj = args [ 1 ]
return OEmbedAutodiscoverNode ( obj )


signature:
def do_url_scheme ( parser , token ) :
    """Generates a & lt ; link & gt ; tag with oembed autodiscovery bits .
 { % oembed _ url _ scheme % }"""

implementation:
args = token . split_contents ( )
if len ( args ) != 1 :
    raise template . TemplateSyntaxError ( '%s takes no parameters.' % args [ 0 ] )
return OEmbedURLSchemeNode ( )


signature:
def exit ( mod_name = "" ) :
    """A stand - in for the normal sys . exit ( )
 all the magic happens here , when this is called at the end of a script it will
 figure out all the available commands and arguments that can be passed in ,
 then handle exiting the script and returning the status code .
 : Example :
 from captain import exit
 exit ( _ _ name _ _ )
 This also acts as a guard against the script being traditionally imported , so
 even if you have this at the end of the script , it won ' t actually exit if the
 script is traditionally imported"""

implementation:
if mod_name and mod_name == "__main__" :
    calling_mod = sys . modules . get ( "__main__" , None )
else :
    calling_mod = discover_if_calling_mod ( )
if calling_mod :
    s = Script ( inspect . getfile ( calling_mod ) , module = calling_mod )
    raw_args = sys . argv [ 1 : ]
    try :
        ret_code = s . run ( raw_args )
    except Stop as e :
        ret_code = e . code
        msg = str ( e )
        if msg :
            if ret_code != 0 :
                echo . err ( msg )
            else :
                echo . out ( msg )
    sys . exit ( ret_code )


signature:
def parser ( self ) :
    """return the parser for the current name"""

implementation:
module = self . module
subcommands = self . subcommands
if subcommands :
    module_desc = inspect . getdoc ( module )
    parser = Parser ( description = module_desc , module = module )
    subparsers = parser . add_subparsers ( )
    for sc_name , callback in subcommands . items ( ) :
        sc_name = sc_name . replace ( "_" , "-" )
        cb_desc = inspect . getdoc ( callback )
        sc_parser = subparsers . add_parser ( sc_name , callback = callback , help = cb_desc )
else :
    parser = Parser ( callback = self . callbacks [ self . function_name ] , module = module )
return parser


signature:
def module ( self ) :
    """load the module so we can actually run the script ' s function"""

implementation:
# we have to guard this value because :
# https : / / thingspython . wordpress . com / 2010/09/27 / another - super - wrinkle - raising - typeerror /
if not hasattr ( self , '_module' ) :
    if "__main__" in sys . modules :
        mod = sys . modules [ "__main__" ]
        path = self . normalize_path ( mod . __file__ )
        if os . path . splitext ( path ) == os . path . splitext ( self . path ) :
            self . _module = mod
        else : # http : / / stackoverflow . com / questions / 67631 / how - to - import - a - module - given - the - full - path
            self . _module = imp . load_source ( 'captain_script' , self . path )
            # self . _ module = imp . load _ source ( self . module _ name , self . path )
return self . _module


signature:
def body ( self ) :
    """get the contents of the script"""

implementation:
if not hasattr ( self , '_body' ) :
    self . _body = inspect . getsource ( self . module )
return self . _body


signature:
def run ( self , raw_args ) :
    """parse and import the script , and then run the script ' s main function"""

implementation:
parser = self . parser
args , kwargs = parser . parse_callback_args ( raw_args )
callback = kwargs . pop ( "main_callback" )
if parser . has_injected_quiet ( ) :
    levels = kwargs . pop ( "quiet_inject" , "" )
    logging . inject_quiet ( levels )
try :
    ret_code = callback ( * args , ** kwargs )
    ret_code = int ( ret_code ) if ret_code else 0
except ArgError as e : # https : / / hg . python . org / cpython / file / 2.7 / Lib / argparse . py # l2374
    echo . err ( "{}: error: {}" , parser . prog , str ( e ) )
    ret_code = 2
return ret_code


signature:
def call_path ( self , basepath ) :
    """return that path to be able to call this script from the passed in
 basename
 example - -
 basepath = / foo / bar
 self . path = / foo / bar / che / baz . py
 self . call _ path ( basepath ) # che / baz . py
 basepath - - string - - the directory you would be calling this script in
 return - - string - - the minimum path that you could use to execute this script
 in basepath"""

implementation:
rel_filepath = self . path
if basepath :
    rel_filepath = os . path . relpath ( self . path , basepath )
basename = self . name
if basename in set ( [ '__init__.py' , '__main__.py' ] ) :
    rel_filepath = os . path . dirname ( rel_filepath )
return rel_filepath


signature:
def parse ( self ) :
    """load the script and set the parser and argument info
 I feel that this is way too brittle to be used long term , I think it just
 might be best to import the stupid module , the thing I don ' t like about that
 is then we import basically everything , which seems bad ?"""

implementation:
if self . parsed :
    return
self . callbacks = { }
# search for main and any main _ * callable objects
regex = re . compile ( "^{}_?" . format ( self . function_name ) , flags = re . I )
mains = set ( )
body = self . body
ast_tree = ast . parse ( self . body , self . path )
for n in ast_tree . body :
    if hasattr ( n , 'name' ) :
        if regex . match ( n . name ) :
            mains . add ( n . name )
    if hasattr ( n , 'value' ) :
        ns = n . value
        if hasattr ( ns , 'id' ) :
            if regex . match ( ns . id ) :
                mains . add ( ns . id )
    if hasattr ( n , 'targets' ) :
        ns = n . targets [ 0 ]
        if hasattr ( ns , 'id' ) :
            if regex . match ( ns . id ) :
                mains . add ( ns . id )
    if hasattr ( n , 'names' ) :
        for ns in n . names :
            if hasattr ( ns , 'name' ) :
                if regex . match ( ns . name ) :
                    mains . add ( ns . name )
            if getattr ( ns , 'asname' , None ) :
                if regex . match ( ns . asname ) :
                    mains . add ( ns . asname )
if len ( mains ) > 0 :
    module = self . module
    for function_name in mains :
        cb = getattr ( module , function_name , None )
        if cb and callable ( cb ) :
            self . callbacks [ function_name ] = cb
else :
    raise ParseError ( "no main function found" )
self . parsed = True
return len ( self . callbacks ) > 0


signature:
def can_run_from_cli ( self ) :
    """return True if this script can be run from the command line"""

implementation:
ret = False
ast_tree = ast . parse ( self . body , self . path )
calls = self . _find_calls ( ast_tree , __name__ , "exit" )
for call in calls :
    if re . search ( "{}\(" . format ( re . escape ( call ) ) , self . body ) :
        ret = True
        break
return ret


signature:
def register_field ( cls , field ) :
    """Handles registering the fields with the FieldRegistry and creating a
 post - save signal for the model ."""

implementation:
FieldRegistry . add_field ( cls , field )
signals . post_save . connect ( handle_save_embeds , sender = cls , dispatch_uid = '%s.%s.%s' % ( cls . _meta . app_label , cls . _meta . module_name , field . name ) )


signature:
def contribute_to_class ( self , cls , name ) :
    """I need a way to ensure that this signal gets created for all child
 models , and since model inheritance doesn ' t have a ' contrubite _ to _ class '
 style hook , I am creating a fake virtual field which will be added to
 all subclasses and handles creating the signal"""

implementation:
super ( EmbeddedMediaField , self ) . contribute_to_class ( cls , name )
register_field ( cls , self )
# add a virtual field that will create signals on any / all subclasses
cls . _meta . add_virtual_field ( EmbeddedSignalCreator ( self ) )


signature:
def render_oembed ( self , oembed_resource , original_url , template_dir = None , context = None ) :
    """Render the oembed resource and return as a string .
 Template directory will always fall back to ' oembed / [ type ] . html ' , but
 a custom template dir can be passed in using the kwargs .
 Templates are given two context variables :
 - response : an OEmbedResource
 - original _ url : the url that was passed to the consumer"""

implementation:
provided_context = context or Context ( )
context = RequestContext ( context . get ( "request" ) or mock_request ( ) )
context . update ( provided_context )
# templates are named for the resources they display , i . e . video . html
template_name = '%s.html' % oembed_resource . type
# set up template finder to fall back to the link template
templates = [ os . path . join ( 'oembed' , template_name ) , 'oembed/link.html' ]
# if there ' s a custom template dir , look there first
if template_dir :
    templates . insert ( 0 , os . path . join ( 'oembed' , template_dir , template_name ) )
template = select_template ( templates )
context . push ( )
context [ 'response' ] = oembed_resource
context [ 'original_url' ] = original_url
rendered = template . render ( context )
context . pop ( )
return rendered . strip ( )


signature:
def parse ( self , text , maxwidth = None , maxheight = None , template_dir = None , context = None , urlize_all_links = CONSUMER_URLIZE_ALL ) :
    """Scans a block of text , replacing anything matching a provider pattern
 with an OEmbed html snippet , if possible .
 Templates should be stored at oembed / { format } . html , so for example :
 oembed / video . html
 An optional template _ dir can be provided , allowing for
 oembed / [ template _ dir ] / video . html
 These templates are passed a context variable , ` ` response ` ` , which is
 an OEmbedResource , as well as the ` ` original _ url ` `"""

implementation:
context = context or Context ( )
context [ 'maxwidth' ] = maxwidth
context [ 'maxheight' ] = maxheight
try :
    text = unicode ( text )
except UnicodeDecodeError :
    text = unicode ( text . decode ( 'utf-8' ) )
return self . parse_data ( text , maxwidth , maxheight , template_dir , context , urlize_all_links )


signature:
def size_to_nearest ( width = None , height = None , allowed_sizes = OEMBED_ALLOWED_SIZES , force_fit = False ) :
    """Generate some dimensions for resizing an object . This function DOES NOT handle
 scaling , it simply calculates maximums . These values should then be passed to
 the resize ( ) method which will scale it and return the scaled width & height ."""

implementation:
minwidth , minheight = min ( allowed_sizes )
maxwidth , maxheight = max ( allowed_sizes )
if not width and not height :
    return maxwidth , maxheight
if width :
    width = int ( width ) > minwidth and int ( width ) or minwidth
elif force_fit :
    width = maxwidth
if height :
    height = int ( height ) > minheight and int ( height ) or minheight
elif force_fit :
    height = maxheight
for size in sorted ( allowed_sizes ) :
    if width and not height :
        if width >= size [ 0 ] :
            maxwidth = size [ 0 ]
            if force_fit :
                maxheight = size [ 1 ]
        else :
            break
    elif height and not width :
        if height >= size [ 1 ] :
            maxheight = size [ 1 ]
            if force_fit :
                maxwidth = size [ 0 ]
        else :
            break
    else :
        if force_fit :
            if ( width >= size [ 0 ] ) and ( height >= size [ 1 ] ) :
                maxwidth , maxheight = size
            else :
                break
        else :
            if width >= size [ 0 ] :
                maxwidth = size [ 0 ]
            if height >= size [ 1 ] :
                maxheight = size [ 1 ]
return maxwidth , maxheight


signature:
def fetch_url ( url , method = 'GET' , user_agent = 'django-oembed' , timeout = SOCKET_TIMEOUT ) :
    """Fetch response headers and data from a URL , raising a generic exception
 for any kind of failure ."""

implementation:
sock = httplib2 . Http ( timeout = timeout )
request_headers = { 'User-Agent' : user_agent , 'Accept-Encoding' : 'gzip' }
try :
    headers , raw = sock . request ( url , headers = request_headers , method = method )
except :
    raise OEmbedHTTPException ( 'Error fetching %s' % url )
return headers , raw


signature:
def relative_to_full ( url , example_url ) :
    """Given a url which may or may not be a relative url , convert it to a full
 url path given another full url as an example"""

implementation:
if re . match ( 'https?:\/\/' , url ) :
    return url
domain = get_domain ( example_url )
if domain :
    return '%s%s' % ( domain , url )
return url


signature:
def mock_request ( ) :
    """Generate a fake request object to allow oEmbeds to use context processors ."""

implementation:
current_site = Site . objects . get_current ( )
request = HttpRequest ( )
request . META [ 'SERVER_NAME' ] = current_site . domain
return request


signature:
def load_class ( path ) :
    """dynamically load a class given a string of the format
 package . Class"""

implementation:
package , klass = path . rsplit ( '.' , 1 )
module = import_module ( package )
return getattr ( module , klass )


signature:
def cleaned_sites ( ) :
    """Create a list of tuples mapping domains from the sites table to their
 site name . The domains will be cleaned into regexes that may be
 more permissive than the site domain is in the db .
 [ ( domain _ regex , domain _ name , domain _ string ) , . . . ]"""

implementation:
mappings = { }
for site in Site . objects . all ( ) : # match the site domain , breaking it into several pieces
    match = re . match ( r'(https?://)?(www[^\.]*\.)?([^/]+)' , site . domain )
    if match is not None :
        http , www , domain = match . groups ( )
        # if the protocol is specified , use it , otherwise accept 80/443
        http_re = http or r'https?:\/\/'
        # whether or not there ' s a www ( or www2 : x ) allow it in the match
        www_re = r'(?:www[^\.]*\.)?'
        # build a regex of the permissive http re , the www re , and the domain
        domain_re = http_re + www_re + domain
        # now build a pretty string representation of the domain
        http = http or r'http://'
        www = www or ''
        normalized = http + www + domain
        mappings [ site . pk ] = ( domain_re , site . name , normalized )
return mappings


signature:
def get_record ( self , fileName , ref_extract_callback = None ) :
    """Gets the Marc xml of the files in xaml _ jp directory
 : param fileName : the name of the file to parse .
 : type fileName : string
 : param refextract _ callback : callback to be used to extract
 unstructured references . It should
 return a marcxml formated string
 of the reference .
 : type refextract _ callback : callable
 : returns : a string with the marc xml version of the file ."""

implementation:
self . document = parse ( fileName )
article_type = self . _get_article_type ( )
if article_type not in [ 'research-article' , 'introduction' , 'letter' ] :
    return ''
rec = create_record ( )
title , subtitle , notes = self . _get_title ( )
subfields = [ ]
if subtitle :
    subfields . append ( ( 'b' , subtitle ) )
if title :
    subfields . append ( ( 'a' , title ) )
    record_add_field ( rec , '245' , subfields = subfields )
subjects = self . document . getElementsByTagName ( 'kwd' )
subjects = map ( xml_to_text , subjects )
for note_id in notes :
    note = self . _get_note ( note_id )
    if note :
        record_add_field ( rec , '500' , subfields = [ ( 'a' , note ) ] )
for subject in subjects :
    record_add_field ( rec , '650' , ind1 = '1' , ind2 = '7' , subfields = [ ( '2' , 'EDPSciences' ) , ( 'a' , subject ) ] )
keywords = self . _get_keywords ( )
for keyword in keywords :
    record_add_field ( rec , '653' , ind1 = '1' , subfields = [ ( 'a' , keyword ) , ( '9' , 'author' ) ] )
journal , volume , issue , year , date , doi , page , fpage , lpage = self . _get_publication_information ( )
astronomy_journals = [ 'EAS Publ.Ser.' , 'Astron.Astrophys.' ]
if journal in astronomy_journals :
    record_add_field ( rec , '650' , ind1 = '1' , ind2 = '7' , subfields = [ ( '2' , 'INSPIRE' ) , ( 'a' , 'Astrophysics' ) ] )
if date :
    record_add_field ( rec , '260' , subfields = [ ( 'c' , date ) , ( 't' , 'published' ) ] )
if doi :
    record_add_field ( rec , '024' , ind1 = '7' , subfields = [ ( 'a' , doi ) , ( '2' , 'DOI' ) ] )
abstract = self . _get_abstract ( )
abstract = self . _format_abstract ( abstract )
if abstract :
    record_add_field ( rec , '520' , subfields = [ ( 'a' , abstract ) , ( '9' , 'EDPSciences' ) ] )
license , license_type , license_url = self . _get_license ( )
subfields = [ ]
if license :
    subfields . append ( ( 'a' , license ) )
if license_url :
    subfields . append ( ( 'u' , license_url ) )
if subfields :
    record_add_field ( rec , '540' , subfields = subfields )
if license_type == 'open-access' :
    self . _attach_fulltext ( rec , doi )
number_of_pages = self . _get_page_count ( )
if number_of_pages :
    record_add_field ( rec , '300' , subfields = [ ( 'a' , number_of_pages ) ] )
c_holder , c_year , c_statement = self . _get_copyright ( )
if c_holder and c_year :
    record_add_field ( rec , '542' , subfields = [ ( 'd' , c_holder ) , ( 'g' , c_year ) , ( 'e' , 'Article' ) ] )
elif c_statement :
    record_add_field ( rec , '542' , subfields = [ ( 'f' , c_statement ) , ( 'e' , 'Article' ) ] )
subfields = [ ]
if journal :
    subfields . append ( ( 'p' , journal ) )
if issue :
    subfields . append ( ( 'n' , issue ) )
if volume :
    subfields . append ( ( 'v' , volume ) )
if fpage and lpage :
    subfields . append ( ( 'c' , '%s-%s' % ( fpage , lpage ) ) )
elif page :
    subfields . append ( ( 'c' , page ) )
if year :
    subfields . append ( ( 'y' , year ) )
record_add_field ( rec , '773' , subfields = subfields )
record_add_field ( rec , '980' , subfields = [ ( 'a' , 'HEP' ) ] )
conference = ''
for tag in self . document . getElementsByTagName ( 'conference' ) :
    conference = xml_to_text ( tag )
if conference :
    record_add_field ( rec , '980' , subfields = [ ( 'a' , 'ConferencePaper' ) ] )
    record_add_field ( rec , '500' , subfields = [ ( 'a' , conference ) ] )
self . _add_references ( rec , ref_extract_callback )
self . _add_authors ( rec )
try :
    return record_xml_output ( rec )
except UnicodeDecodeError :
    message = "Found a bad char in the file for the article " + doi
    sys . stderr . write ( message )
    return ""


signature:
def get_record_rich ( self , filename , ref_extract_callback = None ) :
    """Gets the Marc xml of the files in xaml _ rich directory
 : param fileName : the name of the file to parse .
 : type fileName : string
 : returns : a string with the marc xml version of the file ."""

implementation:
self . document = parse ( filename )
rec = create_record ( )
articles = self . document . getElementsByTagName ( 'ArticleID' )
for article in articles :
    article_type = article . getAttribute ( 'Type' )
    if not article_type == 'Article' :
        return ''
    doi = get_value_in_tag ( self . document , 'DOI' )
    date = ''
    for tag in self . document . getElementsByTagName ( 'Accepted' ) :
        year = get_value_in_tag ( tag , 'Year' )
        month = get_value_in_tag ( tag , 'Month' ) . zfill ( 2 )
        day = get_value_in_tag ( tag , 'Day' ) . zfill ( 2 )
        date = "%s-%s-%s" % ( year , month , day )
    if not date :
        for tag in self . document . getElementsByTagName ( 'OnlineDate' ) :
            year = get_value_in_tag ( tag , 'Year' )
            month = get_value_in_tag ( tag , 'Month' ) . zfill ( 2 )
            day = get_value_in_tag ( tag , 'Day' ) . zfill ( 2 )
            date = "%s-%s-%s" % ( year , month , day )
    first_page = get_value_in_tag ( article , 'FirstPage' )
    last_page = get_value_in_tag ( article , 'LastPage' )
    subjects = article . getElementsByTagName ( 'Keyword' )
    subjects = map ( xml_to_text , subjects )
    subject = ', ' . join ( subjects )
    copyright_statement = get_value_in_tag ( article , 'Copyright' )
journal = get_value_in_tag ( self . document , 'JournalTitle' )
journal , volume = fix_journal_name ( journal , self . journal_mappings )
issues = self . document . getElementsByTagName ( 'IssueID' )
for issue in issues :
    volume += get_value_in_tag ( issue , 'Volume' )
    year = get_value_in_tag ( issue , 'Year' )
title = get_value_in_tag ( self . document , 'Title' )
authors = self . document . getElementsByTagName ( 'Author' )
affiliations = self . document . getElementsByTagName ( 'Affiliation' )
def affiliation_pair ( a ) :
    return a . getAttribute ( 'ID' ) , get_value_in_tag ( a , 'UnstructuredAffiliation' )
affiliations = map ( affiliation_pair , affiliations )
affiliations = dict ( affiliations )
def author_pair ( a ) :
    surname = get_value_in_tag ( a , 'LastName' )
    first_name = get_value_in_tag ( a , 'FirstName' )
    middle_name = get_value_in_tag ( a , 'MiddleName' )
    if middle_name :
        name = '%s, %s %s' % ( surname , first_name , middle_name )
    else :
        name = '%s, %s' % ( surname , first_name )
    try :
        affid = a . getElementsByTagName ( 'AffiliationID' ) [ 0 ] . getAttribute ( 'Label' )
        affiliation = affiliations [ affid ]
    except IndexError :
        affiliation = ''
    except KeyError :
        affiliation = ''
    return name , affiliation
authors = map ( author_pair , authors )
abstract = get_value_in_tag ( self . document , 'Abstract' )
references = self . document . getElementsByTagName ( 'Bibliomixed' )
for reference in references :
    subfields = [ ]
    label = reference . getAttribute ( 'N' )
    if label :
        subfields . append ( ( 'o' , label ) )
    bibliosets = reference . getElementsByTagName ( 'Biblioset' )
    for tag in bibliosets :
        ref_year = get_value_in_tag ( tag , 'Date' )
        ref_journal = get_value_in_tag ( tag , 'JournalShortTitle' )
        ref_journal , ref_volume = fix_journal_name ( ref_journal , self . journal_mappings )
        ref_volume += get_value_in_tag ( tag , 'Volume' )
        ref_page = get_value_in_tag ( tag , 'ArtPageNums' )
        if ref_year :
            subfields . append ( ( 'y' , ref_year ) )
        if ref_journal and ref_volume and ref_page :
            subfields . append ( ( 's' , '%s,%s,%s' % ( ref_journal , ref_volume , ref_page ) ) )
        reference . removeChild ( tag )
    text_ref = xml_to_text ( reference )
    if ref_extract_callback :
        ref_xml = ref_extract_callback ( text_ref )
        dom = parseString ( ref_xml )
        fields = dom . getElementsByTagName ( "datafield" ) [ 0 ]
        fields = fields . getElementsByTagName ( "subfield" )
        if fields :
            subfields . append ( ( '9' , 'refextract' ) )
        for field in fields :
            data = field . firstChild . data
            code = field . getAttribute ( "code" )
            if code == 'm' and bibliosets :
                continue
            else :
                subfields . append ( ( code , data ) )
    else :
        subfields . append ( ( 'm' , text_ref ) )
    if subfields :
        record_add_field ( rec , '999' , ind1 = 'C' , ind2 = '5' , subfields = subfields )
if title :
    record_add_field ( rec , '245' , subfields = [ ( 'a' , title ) ] )
if date :
    record_add_field ( rec , '260' , subfields = [ ( 'c' , date ) , ( 't' , 'published' ) ] )
if doi :
    record_add_field ( rec , '024' , ind1 = '7' , subfields = [ ( 'a' , doi ) , ( '2' , 'DOI' ) ] )
if abstract :
    record_add_field ( rec , '520' , subfields = [ ( 'a' , abstract ) , ( '9' , 'EDPSciences' ) ] )
first_author = True
for author in authors :
    if first_author :
        subfields = [ ( 'a' , author [ 0 ] ) ]
        if author [ 1 ] :
            subfields . append ( ( 'v' , author [ 1 ] ) )
        record_add_field ( rec , '100' , subfields = subfields )
        first_author = False
    else :
        subfields = [ ( 'a' , author [ 0 ] ) ]
        if author [ 1 ] :
            subfields . append ( ( 'v' , author [ 1 ] ) )
        record_add_field ( rec , '700' , subfields = subfields )
subfields = [ ]
if journal and volume and first_page :
    subfields . append ( ( 's' , "%s,%s,%s" % ( journal , volume , first_page ) ) )
if first_page and last_page :
    try :
        nuber_of_pages = int ( last_page ) - int ( first_page )
        record_add_field ( rec , '300' , subfields = [ ( 'a' , str ( nuber_of_pages ) ) ] )
    except ValueError :
        pass
    subfields . append ( ( 'c' , '%s-%s' % ( first_page , last_page ) ) )
if year :
    subfields . append ( ( 'y' , year ) )
record_add_field ( rec , '773' , subfields = subfields )
record_add_field ( rec , '980' , subfields = [ ( 'a' , 'HEP' ) ] )
if copyright_statement :
    record_add_field ( rec , '542' , subfields = [ ( 'f' , copyright_statement ) ] )
if subject :
    record_add_field ( rec , '650' , ind1 = '1' , ind2 = '7' , subfields = [ ( '2' , 'EDPSciences' ) , ( 'a' , subject ) ] )
try :
    return record_xml_output ( rec )
except UnicodeDecodeError :
    message = "Found a bad char in the file for the article " + doi
    sys . stderr . write ( message )
    return ""


signature:
def get_record ( self ) :
    """Override the base get _ record ."""

implementation:
self . update_system_numbers ( )
self . add_systemnumber ( "CDS" )
self . fields_list = [ "024" , "041" , "035" , "037" , "088" , "100" , "110" , "111" , "242" , "245" , "246" , "260" , "269" , "300" , "502" , "650" , "653" , "693" , "700" , "710" , "773" , "856" , "520" , "500" , "980" ]
self . keep_only_fields ( )
self . determine_collections ( )
self . add_cms_link ( )
self . update_languages ( )
self . update_reportnumbers ( )
self . update_date ( )
self . update_pagenumber ( )
self . update_authors ( )
self . update_subject_categories ( "SzGeCERN" , "INSPIRE" , "categories_inspire" )
self . update_keywords ( )
self . update_experiments ( )
self . update_collaboration ( )
self . update_journals ( )
self . update_links_and_ffts ( )
if 'THESIS' in self . collections :
    self . update_thesis_supervisors ( )
    self . update_thesis_information ( )
if 'NOTE' in self . collections :
    self . add_notes ( )
for collection in self . collections :
    record_add_field ( self . record , tag = '980' , subfields = [ ( 'a' , collection ) ] )
self . remove_controlfields ( )
return self . record


signature:
def determine_collections ( self ) :
    """Try to determine which collections this record should belong to ."""

implementation:
for value in record_get_field_values ( self . record , '980' , code = 'a' ) :
    if 'NOTE' in value . upper ( ) :
        self . collections . add ( 'NOTE' )
    if 'THESIS' in value . upper ( ) :
        self . collections . add ( 'THESIS' )
    if 'CONFERENCEPAPER' in value . upper ( ) :
        self . collections . add ( 'ConferencePaper' )
    if "HIDDEN" in value . upper ( ) :
        self . hidden = True
if self . is_published ( ) :
    self . collections . add ( "PUBLISHED" )
    self . collections . add ( "CITEABLE" )
if 'NOTE' not in self . collections :
    from itertools import product
    # TODO : Move this to a KB
    kb = [ 'ATLAS-CONF-' , 'CMS-PAS-' , 'ATL-' , 'CMS-DP-' , 'ALICE-INT-' , 'LHCb-PUB-' ]
    values = record_get_field_values ( self . record , "088" , code = 'a' )
    for val , rep in product ( values , kb ) :
        if val . startswith ( rep ) :
            self . collections . add ( 'NOTE' )
            break
# 980 Arxiv tag
if record_get_field_values ( self . record , '035' , filter_subfield_code = "a" , filter_subfield_value = "arXiv" ) :
    self . collections . add ( "arXiv" )
# 980 HEP & & CORE
self . collections . add ( 'HEP' )
self . collections . add ( 'CORE' )
# 980 Conference Note
if 'ConferencePaper' not in self . collections :
    for value in record_get_field_values ( self . record , tag = '962' , code = 'n' ) :
        if value [ - 2 : ] . isdigit ( ) :
            self . collections . add ( 'ConferencePaper' )
            break
# Clear out any existing ones .
record_delete_fields ( self . record , "980" )


signature:
def is_published ( self ) :
    """Check fields 980 and 773 to see if the record has already been published .
 : return : True is published , else False"""

implementation:
field980 = record_get_field_instances ( self . record , '980' )
field773 = record_get_field_instances ( self . record , '773' )
for f980 in field980 :
    if 'a' in field_get_subfields ( f980 ) :
        for f773 in field773 :
            if 'p' in field_get_subfields ( f773 ) :
                return True
return False


signature:
def add_cms_link ( self ) :
    """Special handling if record is a CMS NOTE ."""

implementation:
intnote = record_get_field_values ( self . record , '690' , filter_subfield_code = "a" , filter_subfield_value = 'INTNOTE' )
if intnote :
    val_088 = record_get_field_values ( self . record , tag = '088' , filter_subfield_code = "a" )
    for val in val_088 :
        if 'CMS' in val :
            url = ( 'http://weblib.cern.ch/abstract?CERN-CMS' + val . split ( 'CMS' , 1 ) [ - 1 ] )
            record_add_field ( self . record , tag = '856' , ind1 = '4' , subfields = [ ( 'u' , url ) ] )


signature:
def update_system_numbers ( self ) :
    """035 Externals ."""

implementation:
scn_035_fields = record_get_field_instances ( self . record , '035' )
forbidden_values = [ "cercer" , "inspire" , "xx" , "cern annual report" , "cmscms" , "wai01" ]
for field in scn_035_fields :
    subs = field_get_subfields ( field )
    if '9' in subs :
        if 'a' not in subs :
            continue
        for sub in subs [ '9' ] :
            if sub . lower ( ) in forbidden_values :
                break
        else : # No forbidden values ( We did not " break " )
            suffixes = [ s . lower ( ) for s in subs [ '9' ] ]
            if 'spires' in suffixes :
                new_subs = [ ( 'a' , 'SPIRES-%s' % subs [ 'a' ] [ 0 ] ) ]
                record_add_field ( self . record , '970' , subfields = new_subs )
                continue
    if 'a' in subs :
        for sub in subs [ 'a' ] :
            if sub . lower ( ) in forbidden_values :
                record_delete_field ( self . record , tag = "035" , field_position_global = field [ 4 ] )


signature:
def update_reportnumbers ( self ) :
    """Handle reportnumbers ."""

implementation:
rep_088_fields = record_get_field_instances ( self . record , '088' )
for field in rep_088_fields :
    subs = field_get_subfields ( field )
    if '9' in subs :
        for val in subs [ '9' ] :
            if val . startswith ( 'P0' ) or val . startswith ( 'CM-P0' ) :
                sf = [ ( '9' , 'CERN' ) , ( 'b' , val ) ]
                record_add_field ( self . record , '595' , subfields = sf )
    for key , val in field [ 0 ] :
        if key in [ 'a' , '9' ] and not val . startswith ( 'SIS-' ) :
            record_add_field ( self . record , '037' , subfields = [ ( 'a' , val ) ] )
record_delete_fields ( self . record , "088" )
# 037 Externals also . . .
rep_037_fields = record_get_field_instances ( self . record , '037' )
for field in rep_037_fields :
    subs = field_get_subfields ( field )
    if 'a' in subs :
        for value in subs [ 'a' ] :
            if 'arXiv' in value :
                new_subs = [ ( 'a' , value ) , ( '9' , 'arXiv' ) ]
                for fld in record_get_field_instances ( self . record , '695' ) :
                    for key , val in field_get_subfield_instances ( fld ) :
                        if key == 'a' :
                            new_subs . append ( ( 'c' , val ) )
                            break
                nf = create_field ( subfields = new_subs )
                record_replace_field ( self . record , '037' , nf , field [ 4 ] )
    for key , val in field [ 0 ] :
        if key in [ 'a' , '9' ] and val . startswith ( 'SIS-' ) :
            record_delete_field ( self . record , '037' , field_position_global = field [ 4 ] )


signature:
def update_date ( self ) :
    """269 Date normalization ."""

implementation:
for field in record_get_field_instances ( self . record , '269' ) :
    for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
        if key == "c" :
            field [ 0 ] [ idx ] = ( "c" , convert_date_to_iso ( value ) )
            record_delete_fields ( self . record , "260" )
if 'THESIS' not in self . collections :
    for field in record_get_field_instances ( self . record , '260' ) :
        record_add_field ( self . record , '269' , subfields = field [ 0 ] )
    record_delete_fields ( self . record , '260' )


signature:
def update_pagenumber ( self ) :
    """300 page number ."""

implementation:
for field in record_get_field_instances ( self . record , '300' ) :
    for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
        if key == 'a' :
            if "mult." not in value and value != " p" :
                field [ 0 ] [ idx ] = ( 'a' , re . sub ( r'[^\d-]+' , '' , value ) )
            else :
                record_delete_field ( self . record , '300' , field_position_global = field [ 4 ] )
                break


signature:
def update_authors ( self ) :
    """100 & 700 punctuate author names ."""

implementation:
author_names = record_get_field_instances ( self . record , '100' )
author_names . extend ( record_get_field_instances ( self . record , '700' ) )
for field in author_names :
    subs = field_get_subfields ( field )
    if 'i' not in subs or 'XX' in subs [ 'i' ] :
        if 'j' not in subs or 'YY' in subs [ 'j' ] :
            for idx , ( key , value ) in enumerate ( field [ 0 ] ) :
                if key == 'a' :
                    field [ 0 ] [ idx ] = ( 'a' , punctuate_authorname ( value ) )


signature:
def update_thesis_supervisors ( self ) :
    """700 - > 701 Thesis supervisors ."""

implementation:
for field in record_get_field_instances ( self . record , '700' ) :
    record_add_field ( self . record , '701' , subfields = field [ 0 ] )
record_delete_fields ( self . record , '700' )


signature:
def update_thesis_information ( self ) :
    """501 degree info - move subfields ."""

implementation:
fields_501 = record_get_field_instances ( self . record , '502' )
for idx , field in enumerate ( fields_501 ) :
    new_subs = [ ]
    for key , value in field [ 0 ] :
        if key == 'a' :
            new_subs . append ( ( 'b' , value ) )
        elif key == 'b' :
            new_subs . append ( ( 'c' , value ) )
        elif key == 'c' :
            new_subs . append ( ( 'd' , value ) )
        else :
            new_subs . append ( ( key , value ) )
    fields_501 [ idx ] = field_swap_subfields ( field , new_subs )


signature:
def update_keywords ( self ) :
    """653 Free Keywords ."""

implementation:
for field in record_get_field_instances ( self . record , '653' , ind1 = '1' ) :
    subs = field_get_subfields ( field )
    new_subs = [ ]
    if 'a' in subs :
        for val in subs [ 'a' ] :
            new_subs . extend ( [ ( '9' , 'author' ) , ( 'a' , val ) ] )
    new_field = create_field ( subfields = new_subs , ind1 = '1' )
    record_replace_field ( self . record , '653' , new_field , field_position_global = field [ 4 ] )


signature:
def update_experiments ( self ) :
    """Experiment mapping ."""

implementation:
# 693 Remove if ' not applicable '
for field in record_get_field_instances ( self . record , '693' ) :
    subs = field_get_subfields ( field )
    all_subs = subs . get ( 'a' , [ ] ) + subs . get ( 'e' , [ ] )
    if 'not applicable' in [ x . lower ( ) for x in all_subs ] :
        record_delete_field ( self . record , '693' , field_position_global = field [ 4 ] )
    new_subs = [ ]
    experiment_a = ""
    experiment_e = ""
    for ( key , value ) in subs . iteritems ( ) :
        if key == 'a' :
            experiment_a = value [ 0 ]
            new_subs . append ( ( key , value [ 0 ] ) )
        elif key == 'e' :
            experiment_e = value [ 0 ]
    experiment = "%s---%s" % ( experiment_a . replace ( " " , "-" ) , experiment_e )
    translated_experiments = self . get_config_item ( experiment , "experiments" )
    new_subs . append ( ( "e" , translated_experiments ) )
    record_delete_field ( self . record , tag = "693" , field_position_global = field [ 4 ] )
    record_add_field ( self . record , "693" , subfields = new_subs )


signature:
def update_collaboration ( self ) :
    """710 Collaboration ."""

implementation:
for field in record_get_field_instances ( self . record , '710' ) :
    subs = field_get_subfield_instances ( field )
    for idx , ( key , value ) in enumerate ( subs [ : ] ) :
        if key == '5' :
            subs . pop ( idx )
        elif value . startswith ( 'CERN. Geneva' ) :
            subs . pop ( idx )
    if len ( subs ) == 0 :
        record_delete_field ( self . record , tag = '710' , field_position_global = field [ 4 ] )


signature:
def update_journals ( self ) :
    """773 journal translations ."""

implementation:
for field in record_get_field_instances ( self . record , '773' ) :
    subs = field_get_subfield_instances ( field )
    new_subs = [ ]
    for idx , ( key , value ) in enumerate ( subs ) :
        if key == 'p' :
            journal_name = self . get_config_item ( value , "journals" , allow_substring = False )
            journal_name = journal_name . replace ( '. ' , '.' ) . strip ( )
            new_subs . append ( ( key , journal_name ) )
        else :
            new_subs . append ( ( key , value ) )
    record_delete_field ( self . record , tag = "773" , field_position_global = field [ 4 ] )
    record_add_field ( self . record , "773" , subfields = new_subs )


signature:
def update_links_and_ffts ( self ) :
    """FFT ( 856 ) Dealing with graphs ."""

implementation:
figure_counter = 0
for field in record_get_field_instances ( self . record , tag = '856' , ind1 = '4' ) :
    subs = field_get_subfields ( field )
    newsubs = [ ]
    remove = False
    if 'z' in subs :
        is_figure = [ s for s in subs [ 'z' ] if "figure" in s . lower ( ) ]
        if is_figure and 'u' in subs :
            is_subformat = [ s for s in subs [ 'u' ] if "subformat" in s . lower ( ) ]
            if not is_subformat :
                url = subs [ 'u' ] [ 0 ]
                if url . endswith ( ".pdf" ) : # We try to convert
                    fd , local_url = mkstemp ( suffix = os . path . basename ( url ) )
                    os . close ( fd )
                    self . logger . info ( "Downloading %s into %s" % ( url , local_url ) )
                    plotfile = ""
                    try :
                        plotfile = download_file ( url = url , download_to_file = local_url )
                    except Exception as e :
                        self . logger . exception ( e )
                        remove = True
                    if plotfile :
                        converted = convert_images ( [ plotfile ] )
                        if converted :
                            url = converted . pop ( )
                            msg = "Successfully converted %s to %s" % ( local_url , url )
                            self . logger . info ( msg )
                        else :
                            msg = "Conversion failed on %s" % ( local_url , )
                            self . logger . error ( msg )
                            url = None
                            remove = True
                if url :
                    newsubs . append ( ( 'a' , url ) )
                    newsubs . append ( ( 't' , 'Plot' ) )
                    figure_counter += 1
                    if 'y' in subs :
                        newsubs . append ( ( 'd' , "%05d %s" % ( figure_counter , subs [ 'y' ] [ 0 ] ) ) )
                        newsubs . append ( ( 'n' , subs [ 'y' ] [ 0 ] ) )
                    else : # Get basename without extension .
                        name = os . path . basename ( os . path . splitext ( subs [ 'u' ] [ 0 ] ) [ 0 ] )
                        newsubs . append ( ( 'd' , "%05d %s" % ( figure_counter , name ) ) )
                        newsubs . append ( ( 'n' , name ) )
    if not newsubs and 'u' in subs :
        is_fulltext = [ s for s in subs [ 'u' ] if ".pdf" in s ]
        if is_fulltext :
            newsubs = [ ( 't' , 'INSPIRE-PUBLIC' ) , ( 'a' , subs [ 'u' ] [ 0 ] ) ]
    if not newsubs and 'u' in subs :
        remove = True
        is_zipfile = [ s for s in subs [ 'u' ] if ".zip" in s ]
        if is_zipfile :
            url = is_zipfile [ 0 ]
            local_url = os . path . join ( self . get_local_folder ( ) , os . path . basename ( url ) )
            self . logger . info ( "Downloading %s into %s" % ( url , local_url ) )
            zipped_archive = ""
            try :
                zipped_archive = download_file ( url = is_zipfile [ 0 ] , download_to_file = local_url )
            except Exception as e :
                self . logger . exception ( e )
                remove = True
            if zipped_archive :
                unzipped_archive = unzip ( zipped_archive )
                list_of_pngs = locate ( "*.png" , unzipped_archive )
                for png in list_of_pngs :
                    if "_vti_" in png or "__MACOSX" in png :
                        continue
                    figure_counter += 1
                    plotsubs = [ ]
                    plotsubs . append ( ( 'a' , png ) )
                    caption = '%05d %s' % ( figure_counter , os . path . basename ( png ) )
                    plotsubs . append ( ( 'd' , caption ) )
                    plotsubs . append ( ( 't' , 'Plot' ) )
                    record_add_field ( self . record , 'FFT' , subfields = plotsubs )
    if not remove and not newsubs and 'u' in subs :
        urls = ( 'http://cdsweb.cern.ch' , 'http://cms.cern.ch' , 'http://cmsdoc.cern.ch' , 'http://documents.cern.ch' , 'http://preprints.cern.ch' , 'http://cds.cern.ch' )
        for val in subs [ 'u' ] :
            if any ( url in val for url in urls ) :
                remove = True
                break
            if val . endswith ( 'ps.gz' ) :
                remove = True
    if newsubs :
        record_add_field ( self . record , 'FFT' , subfields = newsubs )
        remove = True
    if remove :
        record_delete_field ( self . record , '856' , ind1 = '4' , field_position_global = field [ 4 ] )


signature:
def _extract_packages ( self ) :
    """Extract a package in a new directory ."""

implementation:
self . path_unpacked = [ ]
if not hasattr ( self , "retrieved_packages_unpacked" ) :
    self . retrieved_packages_unpacked = [ self . package_name ]
for path in self . retrieved_packages_unpacked :
    self . logger . debug ( "Extracting package: %s" % ( path , ) )
    p_name = 'EPJC' if 'EPJC' in path else 'JHEP'
    p_message = 'scoap3_package_%s_%s_' % ( p_name , datetime . now ( ) )
    self . path_unpacked . append ( mkdtemp ( prefix = p_message , dir = CFG_TMPSHAREDDIR ) )
    try :
        ZipFile ( path ) . extractall ( self . path_unpacked [ - 1 ] )
    except Exception :
        register_exception ( alert_admin = True , prefix = "Springer error extracting package." )
        self . logger . error ( "Error extraction package file: %s" % ( path , ) )
return self . path_unpacked


signature:
def _crawl_springer_and_find_main_xml ( self ) :
    """A package contains several subdirectory corresponding to each article .
 An article is actually identified by the existence of a main . pdf and
 a main . xml in a given directory ."""

implementation:
self . found_articles = [ ]
def visit ( arg , dirname , names ) :
    files = [ filename for filename in names if "nlm.xml" in filename ]
    if not files :
        files = [ filename for filename in names if ".xml.scoap" in filename ]
    if files :
        try : # self . _ normalize _ article _ dir _ with _ dtd ( dirname )
            self . found_articles . append ( dirname )
        except Exception as err :
            register_exception ( )
            print "ERROR: can't normalize %s: %s" % ( dirname , err )
if hasattr ( self , 'path_unpacked' ) :
    for path in self . path_unpacked :
        walk ( path , visit , None )
elif self . path :
    walk ( self . path , visit , None )
else :
    self . logger . info ( "Nothing to do." )


signature:
def _normalize_article_dir_with_dtd ( self , path ) :
    """TODO : main . xml from Springer assume the existence of a local DTD .
 This procedure install the DTDs next to the main . xml file
 and normalize it using xmllint in order to resolve all namespaces
 and references ."""

implementation:
files = [ filename for filename in listdir ( path ) if "nlm.xml" in filename ]
if not files :
    files = [ filename for filename in listdir ( path ) if ".xml.scoap" in filename ]
if exists ( join ( path , 'resolved_main.xml' ) ) :
    return
if 'JATS-archivearticle1.dtd' in open ( join ( path , files [ 0 ] ) ) . read ( ) :
    path_normalized = mkdtemp ( prefix = "scoap3_normalized_jats_" , dir = CFG_TMPSHAREDDIR )
    ZipFile ( CFG_SPRINGER_JATS_PATH ) . extractall ( path_normalized )
elif 'A++V2.4.dtd' in open ( join ( path , files [ 0 ] ) ) . read ( ) :
    path_normalized = mkdtemp ( prefix = "scoap3_normalized_app_" , dir = CFG_TMPSHAREDDIR )
    ZipFile ( CFG_SPRINGER_AV24_PATH ) . extractall ( path_normalized )
else :
    error_msg = ( "It looks like the path %s does not contain an " "JATS-archivearticle1.dtd nor A++V2.4.dtd XML file." )
    self . logger . error ( error_msg % path )
    raise ValueError ( error_msg % path )
print "Normalizing %s" % ( files [ 0 ] , )
( cmd_exit_code , cmd_out , cmd_err ) = run_shell_command ( ( "xmllint --format " "--loaddtd %s --output %s" ) , ( join ( path , files [ 0 ] ) , join ( path_normalized , 'resolved_main.xml' ) ) )
if cmd_err :
    error_msg = "Error in cleaning %s: %s"
    self . logger . error ( error_msg % ( join ( path , 'issue.xml' ) , cmd_err ) )
    raise ValueError ( error_msg % ( join ( path , 'main.xml' ) , cmd_err ) )
self . articles_normalized . append ( path_normalized )


signature:
def add_all_filters ( pelican ) :
    """Add ( register ) all filters to Pelican ."""

implementation:
pelican . env . filters . update ( { 'datetime' : filters . datetime } )
pelican . env . filters . update ( { 'article_date' : filters . article_date } )
pelican . env . filters . update ( { 'breaking_spaces' : filters . breaking_spaces } )
pelican . env . filters . update ( { 'titlecase' : filters . titlecase } )


signature:
def create_field ( subfields = None , ind1 = ' ' , ind2 = ' ' , controlfield_value = '' , global_position = - 1 ) :
    """Return a field created with the provided elements .
 Global position is set arbitrary to - 1."""

implementation:
if subfields is None :
    subfields = [ ]
ind1 , ind2 = _wash_indicators ( ind1 , ind2 )
field = ( subfields , ind1 , ind2 , controlfield_value , global_position )
_check_field_validity ( field )
return field


signature:
def create_records ( marcxml , verbose = CFG_BIBRECORD_DEFAULT_VERBOSE_LEVEL , correct = CFG_BIBRECORD_DEFAULT_CORRECT , parser = '' , keep_singletons = CFG_BIBRECORD_KEEP_SINGLETONS ) :
    """Create a list of records from the marcxml description .
 : returns : a list of objects initiated by the function create _ record ( ) .
 Please see that function ' s docstring ."""

implementation:
# Use the DOTALL flag to include newlines .
regex = re . compile ( '<record.*?>.*?</record>' , re . DOTALL )
record_xmls = regex . findall ( marcxml )
return [ create_record ( record_xml , verbose = verbose , correct = correct , parser = parser , keep_singletons = keep_singletons ) for record_xml in record_xmls ]


signature:
def create_record ( marcxml = None , verbose = CFG_BIBRECORD_DEFAULT_VERBOSE_LEVEL , correct = CFG_BIBRECORD_DEFAULT_CORRECT , parser = '' , sort_fields_by_indicators = False , keep_singletons = CFG_BIBRECORD_KEEP_SINGLETONS ) :
    """Create a record object from the marcxml description .
 Uses the lxml parser .
 The returned object is a tuple ( record , status _ code , list _ of _ errors ) ,
 where status _ code is 0 when there are errors , 1 when no errors .
 The return record structure is as follows : :
 Record : = { tag : [ Field ] }
 Field : = ( Subfields , ind1 , ind2 , value )
 Subfields : = [ ( code , value ) ]
 . . code - block : : none
 | record |
 | record [ ' 001 ' ] | record [ ' 909 ' ] | record [ ' 520 ' ] |
 [ list of fields ] [ list of fields ] [ list of fields ] . . .
 | [ 0 ] | [ 0 ] | [ 1 ] . . . | [ 0]
 | Field 001 | | Field 909 | | Field 909 | | Field 520 |
 | [ 0 ] | [ 1 ] | [ 2 ] |
 [ list of subfields ] ' C ' ' 4 ' . . .
 ( ' a ' , ' value ' ) | ( ' a ' , ' value for another a ' )
 ( ' b ' , ' value for subfield b ' )
 : param marcxml : an XML string representation of the record to create
 : param verbose : the level of verbosity : 0 ( silent ) , 1-2 ( warnings ) ,
 3 ( strict : stop when errors )
 : param correct : 1 to enable correction of marcxml syntax . Else 0.
 : return : a tuple ( record , status _ code , list _ of _ errors ) , where status
 code is 0 where there are errors , 1 when no errors"""

implementation:
if marcxml is None :
    return { }
try :
    rec = _create_record_lxml ( marcxml , verbose , correct , keep_singletons = keep_singletons )
except InvenioBibRecordParserError as ex1 :
    return ( None , 0 , str ( ex1 ) )
if sort_fields_by_indicators :
    _record_sort_by_indicators ( rec )
errs = [ ]
if correct : # Correct the structure of the record .
    errs = _correct_record ( rec )
return ( rec , int ( not errs ) , errs )


signature:
def filter_field_instances ( field_instances , filter_subcode , filter_value , filter_mode = 'e' ) :
    """Filter the given field .
 Filters given field and returns only that field instances that contain
 filter _ subcode with given filter _ value . As an input for search function
 accepts output from record _ get _ field _ instances function . Function can be
 run in three modes :
 - ' e ' - looking for exact match in subfield value
 - ' s ' - looking for substring in subfield value
 - ' r ' - looking for regular expression in subfield value
 Example :
 record _ filter _ field ( record _ get _ field _ instances ( rec , ' 999 ' , ' % ' , ' % ' ) ,
 ' y ' , ' 2001 ' )
 In this case filter _ subcode is ' y ' and filter _ value is ' 2001 ' .
 : param field _ instances : output from record _ get _ field _ instances
 : param filter _ subcode : name of the subfield
 : type filter _ subcode : string
 : param filter _ value : value of the subfield
 : type filter _ value : string
 : param filter _ mode : ' e ' , ' s ' or ' r '"""

implementation:
matched = [ ]
if filter_mode == 'e' :
    to_match = ( filter_subcode , filter_value )
    for instance in field_instances :
        if to_match in instance [ 0 ] :
            matched . append ( instance )
elif filter_mode == 's' :
    for instance in field_instances :
        for subfield in instance [ 0 ] :
            if subfield [ 0 ] == filter_subcode and subfield [ 1 ] . find ( filter_value ) > - 1 :
                matched . append ( instance )
                break
elif filter_mode == 'r' :
    reg_exp = re . compile ( filter_value )
    for instance in field_instances :
        for subfield in instance [ 0 ] :
            if subfield [ 0 ] == filter_subcode and reg_exp . match ( subfield [ 1 ] ) is not None :
                matched . append ( instance )
                break
return matched


signature:
def record_drop_duplicate_fields ( record ) :
    """Return a record where all the duplicate fields have been removed .
 Fields are considered identical considering also the order of their
 subfields ."""

implementation:
out = { }
position = 0
tags = sorted ( record . keys ( ) )
for tag in tags :
    fields = record [ tag ]
    out [ tag ] = [ ]
    current_fields = set ( )
    for full_field in fields :
        field = ( tuple ( full_field [ 0 ] ) , ) + full_field [ 1 : 4 ]
        if field not in current_fields :
            current_fields . add ( field )
            position += 1
            out [ tag ] . append ( full_field [ : 4 ] + ( position , ) )
return out


signature:
def records_identical ( rec1 , rec2 , skip_005 = True , ignore_field_order = False , ignore_subfield_order = False , ignore_duplicate_subfields = False , ignore_duplicate_controlfields = False ) :
    """Return True if rec1 is identical to rec2.
 It does so regardless of a difference in the 005 tag ( i . e . the timestamp ) ."""

implementation:
rec1_keys = set ( rec1 . keys ( ) )
rec2_keys = set ( rec2 . keys ( ) )
if skip_005 :
    rec1_keys . discard ( "005" )
    rec2_keys . discard ( "005" )
if rec1_keys != rec2_keys :
    return False
for key in rec1_keys :
    if ignore_duplicate_controlfields and key . startswith ( '00' ) :
        if set ( field [ 3 ] for field in rec1 [ key ] ) != set ( field [ 3 ] for field in rec2 [ key ] ) :
            return False
        continue
    rec1_fields = rec1 [ key ]
    rec2_fields = rec2 [ key ]
    if len ( rec1_fields ) != len ( rec2_fields ) : # They already differs in length . . .
        return False
    if ignore_field_order : # We sort the fields , first by indicators and then by anything else
        rec1_fields = sorted ( rec1_fields , key = lambda elem : ( elem [ 1 ] , elem [ 2 ] , elem [ 3 ] , elem [ 0 ] ) )
        rec2_fields = sorted ( rec2_fields , key = lambda elem : ( elem [ 1 ] , elem [ 2 ] , elem [ 3 ] , elem [ 0 ] ) )
    else : # We sort the fields , first by indicators , then by global position
    # and then by anything else
        rec1_fields = sorted ( rec1_fields , key = lambda elem : ( elem [ 1 ] , elem [ 2 ] , elem [ 4 ] , elem [ 3 ] , elem [ 0 ] ) )
        rec2_fields = sorted ( rec2_fields , key = lambda elem : ( elem [ 1 ] , elem [ 2 ] , elem [ 4 ] , elem [ 3 ] , elem [ 0 ] ) )
    for field1 , field2 in zip ( rec1_fields , rec2_fields ) :
        if ignore_duplicate_subfields :
            if field1 [ 1 : 4 ] != field2 [ 1 : 4 ] or set ( field1 [ 0 ] ) != set ( field2 [ 0 ] ) :
                return False
        elif ignore_subfield_order :
            if field1 [ 1 : 4 ] != field2 [ 1 : 4 ] or sorted ( field1 [ 0 ] ) != sorted ( field2 [ 0 ] ) :
                return False
        elif field1 [ : 4 ] != field2 [ : 4 ] :
            return False
return True


signature:
def record_get_field_instances ( rec , tag = "" , ind1 = " " , ind2 = " " ) :
    """Return the list of field instances for the specified tag and indications .
 Return empty list if not found .
 If tag is empty string , returns all fields
 Parameters ( tag , ind1 , ind2 ) can contain wildcard % .
 : param rec : a record structure as returned by create _ record ( )
 : param tag : a 3 characters long string
 : param ind1 : a 1 character long string
 : param ind2 : a 1 character long string
 : param code : a 1 character long string
 : return : a list of field tuples ( Subfields , ind1 , ind2 , value ,
 field _ position _ global ) where subfields is list of ( code , value )"""

implementation:
if not rec :
    return [ ]
if not tag :
    return rec . items ( )
else :
    out = [ ]
    ind1 , ind2 = _wash_indicators ( ind1 , ind2 )
    if '%' in tag : # Wildcard in tag . Check all possible
        for field_tag in rec :
            if _tag_matches_pattern ( field_tag , tag ) :
                for possible_field_instance in rec [ field_tag ] :
                    if ( ind1 in ( '%' , possible_field_instance [ 1 ] ) and ind2 in ( '%' , possible_field_instance [ 2 ] ) ) :
                        out . append ( possible_field_instance )
    else : # Completely defined tag . Use dict
        for possible_field_instance in rec . get ( tag , [ ] ) :
            if ( ind1 in ( '%' , possible_field_instance [ 1 ] ) and ind2 in ( '%' , possible_field_instance [ 2 ] ) ) :
                out . append ( possible_field_instance )
    return out


signature:
def record_add_field ( rec , tag , ind1 = ' ' , ind2 = ' ' , controlfield_value = '' , subfields = None , field_position_global = None , field_position_local = None ) :
    """Add a new field into the record .
 If field _ position _ global or field _ position _ local is specified then
 this method will insert the new field at the desired position .
 Otherwise a global field position will be computed in order to
 insert the field at the best position ( first we try to keep the
 order of the tags and then we insert the field at the end of the
 fields with the same tag ) .
 If both field _ position _ global and field _ position _ local are present ,
 then field _ position _ local takes precedence .
 : param rec : the record data structure
 : param tag : the tag of the field to be added
 : param ind1 : the first indicator
 : param ind2 : the second indicator
 : param controlfield _ value : the value of the controlfield
 : param subfields : the subfields ( a list of tuples ( code , value ) )
 : param field _ position _ global : the global field position ( record wise )
 : param field _ position _ local : the local field position ( tag wise )
 : return : the global field position of the newly inserted field or - 1 if the
 operation failed"""

implementation:
error = _validate_record_field_positions_global ( rec )
if error : # FIXME one should write a message here
    pass
# Clean the parameters .
if subfields is None :
    subfields = [ ]
ind1 , ind2 = _wash_indicators ( ind1 , ind2 )
if controlfield_value and ( ind1 != ' ' or ind2 != ' ' or subfields ) :
    return - 1
# Detect field number to be used for insertion :
# Dictionaries for uniqueness .
tag_field_positions_global = { } . fromkeys ( [ field [ 4 ] for field in rec . get ( tag , [ ] ) ] )
all_field_positions_global = { } . fromkeys ( [ field [ 4 ] for fields in rec . values ( ) for field in fields ] )
if field_position_global is None and field_position_local is None : # Let ' s determine the global field position of the new field .
    if tag in rec :
        try :
            field_position_global = max ( [ field [ 4 ] for field in rec [ tag ] ] ) + 1
        except IndexError :
            if tag_field_positions_global :
                field_position_global = max ( tag_field_positions_global ) + 1
            elif all_field_positions_global :
                field_position_global = max ( all_field_positions_global ) + 1
            else :
                field_position_global = 1
    else :
        if tag in ( 'FMT' , 'FFT' , 'BDR' , 'BDM' ) : # Add the new tag to the end of the record .
            if tag_field_positions_global :
                field_position_global = max ( tag_field_positions_global ) + 1
            elif all_field_positions_global :
                field_position_global = max ( all_field_positions_global ) + 1
            else :
                field_position_global = 1
        else : # Insert the tag in an ordered way by selecting the
        # right global field position .
            immediate_lower_tag = '000'
            for rec_tag in rec :
                if ( tag not in ( 'FMT' , 'FFT' , 'BDR' , 'BDM' ) and rec [ rec_tag ] and immediate_lower_tag < rec_tag < tag ) :
                    immediate_lower_tag = rec_tag
            if immediate_lower_tag == '000' :
                field_position_global = 1
            else :
                field_position_global = rec [ immediate_lower_tag ] [ - 1 ] [ 4 ] + 1
    field_position_local = len ( rec . get ( tag , [ ] ) )
    _shift_field_positions_global ( rec , field_position_global , 1 )
elif field_position_local is not None :
    if tag in rec :
        if field_position_local >= len ( rec [ tag ] ) :
            field_position_global = rec [ tag ] [ - 1 ] [ 4 ] + 1
        else :
            field_position_global = rec [ tag ] [ field_position_local ] [ 4 ]
        _shift_field_positions_global ( rec , field_position_global , 1 )
    else :
        if all_field_positions_global :
            field_position_global = max ( all_field_positions_global ) + 1
        else : # Empty record .
            field_position_global = 1
elif field_position_global is not None : # If the user chose an existing global field position , shift all the
# global field positions greater than the input global field position .
    if tag not in rec :
        if all_field_positions_global :
            field_position_global = max ( all_field_positions_global ) + 1
        else :
            field_position_global = 1
        field_position_local = 0
    elif field_position_global < min ( tag_field_positions_global ) :
        field_position_global = min ( tag_field_positions_global )
        _shift_field_positions_global ( rec , min ( tag_field_positions_global ) , 1 )
        field_position_local = 0
    elif field_position_global > max ( tag_field_positions_global ) :
        field_position_global = max ( tag_field_positions_global ) + 1
        _shift_field_positions_global ( rec , max ( tag_field_positions_global ) + 1 , 1 )
        field_position_local = len ( rec . get ( tag , [ ] ) )
    else :
        if field_position_global in tag_field_positions_global :
            _shift_field_positions_global ( rec , field_position_global , 1 )
        field_position_local = 0
        for position , field in enumerate ( rec [ tag ] ) :
            if field [ 4 ] == field_position_global + 1 :
                field_position_local = position
# Create the new field .
newfield = ( subfields , ind1 , ind2 , str ( controlfield_value ) , field_position_global )
rec . setdefault ( tag , [ ] ) . insert ( field_position_local , newfield )
# Return new field number :
return field_position_global


signature:
def record_delete_field ( rec , tag , ind1 = ' ' , ind2 = ' ' , field_position_global = None , field_position_local = None ) :
    """Delete the field with the given position .
 If global field position is specified , deletes the field with the
 corresponding global field position .
 If field _ position _ local is specified , deletes the field with the
 corresponding local field position and tag .
 Else deletes all the fields matching tag and optionally ind1 and
 ind2.
 If both field _ position _ global and field _ position _ local are present ,
 then field _ position _ local takes precedence .
 : param rec : the record data structure
 : param tag : the tag of the field to be deleted
 : param ind1 : the first indicator of the field to be deleted
 : param ind2 : the second indicator of the field to be deleted
 : param field _ position _ global : the global field position ( record wise )
 : param field _ position _ local : the local field position ( tag wise )
 : return : the list of deleted fields"""

implementation:
error = _validate_record_field_positions_global ( rec )
if error : # FIXME one should write a message here .
    pass
if tag not in rec :
    return False
ind1 , ind2 = _wash_indicators ( ind1 , ind2 )
deleted = [ ]
newfields = [ ]
if field_position_global is None and field_position_local is None : # Remove all fields with tag ' tag ' .
    for field in rec [ tag ] :
        if field [ 1 ] != ind1 or field [ 2 ] != ind2 :
            newfields . append ( field )
        else :
            deleted . append ( field )
    rec [ tag ] = newfields
elif field_position_global is not None : # Remove the field with ' field _ position _ global ' .
    for field in rec [ tag ] :
        if ( field [ 1 ] != ind1 and field [ 2 ] != ind2 or field [ 4 ] != field_position_global ) :
            newfields . append ( field )
        else :
            deleted . append ( field )
    rec [ tag ] = newfields
elif field_position_local is not None : # Remove the field with ' field _ position _ local ' .
    try :
        del rec [ tag ] [ field_position_local ]
    except IndexError :
        return [ ]
if not rec [ tag ] : # Tag is now empty , remove it .
    del rec [ tag ]
return deleted


signature:
def record_delete_fields ( rec , tag , field_positions_local = None ) :
    """Delete all / some fields defined with MARC tag ' tag ' from record ' rec ' .
 : param rec : a record structure .
 : type rec : tuple
 : param tag : three letter field .
 : type tag : string
 : param field _ position _ local : if set , it is the list of local positions
 within all the fields with the specified tag , that should be deleted .
 If not set all the fields with the specified tag will be deleted .
 : type field _ position _ local : sequence
 : return : the list of deleted fields .
 : rtype : list
 : note : the record is modified in place ."""

implementation:
if tag not in rec :
    return [ ]
new_fields , deleted_fields = [ ] , [ ]
for position , field in enumerate ( rec . get ( tag , [ ] ) ) :
    if field_positions_local is None or position in field_positions_local :
        deleted_fields . append ( field )
    else :
        new_fields . append ( field )
if new_fields :
    rec [ tag ] = new_fields
else :
    del rec [ tag ]
return deleted_fields


signature:
def record_add_fields ( rec , tag , fields , field_position_local = None , field_position_global = None ) :
    """Add the fields into the record at the required position .
 The position is specified by the tag and the field _ position _ local in the
 list of fields .
 : param rec : a record structure
 : param tag : the tag of the fields to be moved
 : param field _ position _ local : the field _ position _ local to which the field
 will be inserted . If not specified , appends
 the fields to the tag .
 : param a : list of fields to be added
 : return : - 1 if the operation failed , or the field _ position _ local if it was
 successful"""

implementation:
if field_position_local is None and field_position_global is None :
    for field in fields :
        record_add_field ( rec , tag , ind1 = field [ 1 ] , ind2 = field [ 2 ] , subfields = field [ 0 ] , controlfield_value = field [ 3 ] )
else :
    fields . reverse ( )
    for field in fields :
        record_add_field ( rec , tag , ind1 = field [ 1 ] , ind2 = field [ 2 ] , subfields = field [ 0 ] , controlfield_value = field [ 3 ] , field_position_local = field_position_local , field_position_global = field_position_global )
return field_position_local


signature:
def record_move_fields ( rec , tag , field_positions_local , field_position_local = None ) :
    """Move some fields to the position specified by ' field _ position _ local ' .
 : param rec : a record structure as returned by create _ record ( )
 : param tag : the tag of the fields to be moved
 : param field _ positions _ local : the positions of the fields to move
 : param field _ position _ local : insert the field before that
 field _ position _ local . If unspecified , appends
 the fields : return : the field _ position _ local
 is the operation was successful"""

implementation:
fields = record_delete_fields ( rec , tag , field_positions_local = field_positions_local )
return record_add_fields ( rec , tag , fields , field_position_local = field_position_local )


signature:
def record_delete_subfield ( rec , tag , subfield_code , ind1 = ' ' , ind2 = ' ' ) :
    """Delete all subfields with subfield _ code in the record ."""

implementation:
ind1 , ind2 = _wash_indicators ( ind1 , ind2 )
for field in rec . get ( tag , [ ] ) :
    if field [ 1 ] == ind1 and field [ 2 ] == ind2 :
        field [ 0 ] [ : ] = [ subfield for subfield in field [ 0 ] if subfield_code != subfield [ 0 ] ]


signature:
def record_get_field ( rec , tag , field_position_global = None , field_position_local = None ) :
    """Return the the matching field .
 One has to enter either a global field position or a local field position .
 : return : a list of subfield tuples ( subfield code , value ) .
 : rtype : list"""

implementation:
if field_position_global is None and field_position_local is None :
    raise InvenioBibRecordFieldError ( "A field position is required to " "complete this operation." )
elif field_position_global is not None and field_position_local is not None :
    raise InvenioBibRecordFieldError ( "Only one field position is required " "to complete this operation." )
elif field_position_global :
    if tag not in rec :
        raise InvenioBibRecordFieldError ( "No tag '%s' in record." % tag )
    for field in rec [ tag ] :
        if field [ 4 ] == field_position_global :
            return field
    raise InvenioBibRecordFieldError ( "No field has the tag '%s' and the " "global field position '%d'." % ( tag , field_position_global ) )
else :
    try :
        return rec [ tag ] [ field_position_local ]
    except KeyError :
        raise InvenioBibRecordFieldError ( "No tag '%s' in record." % tag )
    except IndexError :
        raise InvenioBibRecordFieldError ( "No field has the tag '%s' and " "the local field position '%d'." % ( tag , field_position_local ) )


signature:
def record_replace_field ( rec , tag , new_field , field_position_global = None , field_position_local = None ) :
    """Replace a field with a new field ."""

implementation:
if field_position_global is None and field_position_local is None :
    raise InvenioBibRecordFieldError ( "A field position is required to " "complete this operation." )
elif field_position_global is not None and field_position_local is not None :
    raise InvenioBibRecordFieldError ( "Only one field position is required " "to complete this operation." )
elif field_position_global :
    if tag not in rec :
        raise InvenioBibRecordFieldError ( "No tag '%s' in record." % tag )
    replaced = False
    for position , field in enumerate ( rec [ tag ] ) :
        if field [ 4 ] == field_position_global :
            rec [ tag ] [ position ] = new_field
            replaced = True
    if not replaced :
        raise InvenioBibRecordFieldError ( "No field has the tag '%s' and " "the global field position '%d'." % ( tag , field_position_global ) )
else :
    try :
        rec [ tag ] [ field_position_local ] = new_field
    except KeyError :
        raise InvenioBibRecordFieldError ( "No tag '%s' in record." % tag )
    except IndexError :
        raise InvenioBibRecordFieldError ( "No field has the tag '%s' and " "the local field position '%d'." % ( tag , field_position_local ) )


signature:
def record_get_subfields ( rec , tag , field_position_global = None , field_position_local = None ) :
    """Return the subfield of the matching field .
 One has to enter either a global field position or a local field position .
 : return : a list of subfield tuples ( subfield code , value ) .
 : rtype : list"""

implementation:
field = record_get_field ( rec , tag , field_position_global = field_position_global , field_position_local = field_position_local )
return field [ 0 ]


signature:
def record_delete_subfield_from ( rec , tag , subfield_position , field_position_global = None , field_position_local = None ) :
    """Delete subfield from position specified .
 Specify the subfield by tag , field number and subfield position ."""

implementation:
subfields = record_get_subfields ( rec , tag , field_position_global = field_position_global , field_position_local = field_position_local )
try :
    del subfields [ subfield_position ]
except IndexError :
    raise InvenioBibRecordFieldError ( "The record does not contain the subfield " "'%(subfieldIndex)s' inside the field (local: " "'%(fieldIndexLocal)s, global: '%(fieldIndexGlobal)s' ) of tag " "'%(tag)s'." % { "subfieldIndex" : subfield_position , "fieldIndexLocal" : str ( field_position_local ) , "fieldIndexGlobal" : str ( field_position_global ) , "tag" : tag } )
if not subfields :
    if field_position_global is not None :
        for position , field in enumerate ( rec [ tag ] ) :
            if field [ 4 ] == field_position_global :
                del rec [ tag ] [ position ]
    else :
        del rec [ tag ] [ field_position_local ]
    if not rec [ tag ] :
        del rec [ tag ]


signature:
def record_add_subfield_into ( rec , tag , subfield_code , value , subfield_position = None , field_position_global = None , field_position_local = None ) :
    """Add subfield into specified position .
 Specify the subfield by tag , field number and optionally by subfield
 position ."""

implementation:
subfields = record_get_subfields ( rec , tag , field_position_global = field_position_global , field_position_local = field_position_local )
if subfield_position is None :
    subfields . append ( ( subfield_code , value ) )
else :
    subfields . insert ( subfield_position , ( subfield_code , value ) )


signature:
def record_modify_controlfield ( rec , tag , controlfield_value , field_position_global = None , field_position_local = None ) :
    """Modify controlfield at position specified by tag and field number ."""

implementation:
field = record_get_field ( rec , tag , field_position_global = field_position_global , field_position_local = field_position_local )
new_field = ( field [ 0 ] , field [ 1 ] , field [ 2 ] , controlfield_value , field [ 4 ] )
record_replace_field ( rec , tag , new_field , field_position_global = field_position_global , field_position_local = field_position_local )


signature:
def record_modify_subfield ( rec , tag , subfield_code , value , subfield_position , field_position_global = None , field_position_local = None ) :
    """Modify subfield at specified position .
 Specify the subfield by tag , field number and subfield position ."""

implementation:
subfields = record_get_subfields ( rec , tag , field_position_global = field_position_global , field_position_local = field_position_local )
try :
    subfields [ subfield_position ] = ( subfield_code , value )
except IndexError :
    raise InvenioBibRecordFieldError ( "There is no subfield with position '%d'." % subfield_position )


signature:
def record_move_subfield ( rec , tag , subfield_position , new_subfield_position , field_position_global = None , field_position_local = None ) :
    """Move subfield at specified position .
 Sspecify the subfield by tag , field number and subfield position to new
 subfield position ."""

implementation:
subfields = record_get_subfields ( rec , tag , field_position_global = field_position_global , field_position_local = field_position_local )
try :
    subfield = subfields . pop ( subfield_position )
    subfields . insert ( new_subfield_position , subfield )
except IndexError :
    raise InvenioBibRecordFieldError ( "There is no subfield with position '%d'." % subfield_position )