signature:
def remove_file ( self , relativePath , removeFromSystem = False , raiseError = True , ntrials = 3 ) :
    """Remove file from repository .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the
 file to remove .
 # . removeFromSystem ( boolean ) : Whether to remove file from disk as
 well .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration ."""

implementation:
assert isinstance ( raiseError , bool ) , "removeFromSystem must be boolean"
assert isinstance ( removeFromSystem , bool ) , "removeFromSystem must be boolean"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
# check name and path
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
realPath = os . path . join ( self . __path , relativePath )
fPath , fName = os . path . split ( realPath )
# lock repository
LF = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( fPath , self . __fileLock % fName ) )
acquired , code = LF . acquire_lock ( )
if not acquired :
    error = "Code %s. Unable to aquire the lock when adding '%s'" % ( code , relativePath )
    assert not raiseError , error
    return False , error
# remove file
for _trial in range ( ntrials ) :
    removed = False
    message = [ ]
    try : # check whether it ' s a repository file
        isRepoFile , fileOnDisk , infoOnDisk , classOnDisk = self . is_repository_file ( relativePath )
        if not isRepoFile :
            message ( "File '%s' is not a repository file" % ( relativePath , ) )
            if fileOnDisk :
                message . append ( "File itself is found on disk" )
            if infoOnDisk :
                message . append ( "%s is found on disk" % self . __fileInfo % fName )
            if classOnDisk :
                message . append ( "%s is found on disk" % self . __fileClass % fName )
        else :
            dirList = self . __get_repository_directory ( fPath )
            findex = dirList . index ( fName )
            dirList . pop ( findex )
            if os . path . isfile ( realPath ) :
                os . remove ( realPath )
            if os . path . isfile ( os . path . join ( fPath , self . __fileInfo % fName ) ) :
                os . remove ( os . path . join ( fPath , self . __fileInfo % fName ) )
            if os . path . isfile ( os . path . join ( fPath , self . __fileClass % fName ) ) :
                os . remove ( os . path . join ( fPath , self . __fileClass % fName ) )
    except Exception as err :
        removed = False
        message . append ( str ( err ) )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , '\n' . join ( message ) ) )
    else :
        removed = True
        break
# release lock
LF . release_lock ( )
# always clean
try :
    if os . path . isfile ( os . path . join ( fPath , self . __fileLock % fName ) ) :
        os . remove ( os . path . join ( fPath , self . __fileLock % fName ) )
except :
    pass
# check and return
assert removed or not raiseError , "Unable to remove file '%s' after %i trials (%s)" % ( relativePath , ntrials , '\n' . join ( message ) , )
return removed , '\n' . join ( message )


signature:
def rename ( self , key : Any , new_key : Any ) :
    """Renames an item in this collection as a transaction .
 Will override if new key name already exists .
 : param key : the current name of the item
 : param new _ key : the new name that the item should have"""

implementation:
if new_key == key :
    return
required_locks = [ self . _key_locks [ key ] , self . _key_locks [ new_key ] ]
ordered_required_locks = sorted ( required_locks , key = lambda x : id ( x ) )
for lock in ordered_required_locks :
    lock . acquire ( )
try :
    if key not in self . _data :
        raise KeyError ( "Attribute to rename \"%s\" does not exist" % key )
    self . _data [ new_key ] = self [ key ]
    del self . _data [ key ]
finally :
    for lock in required_locks :
        lock . release ( )


signature:
def get_text_fingerprint ( text , hash_meth , encoding = "utf-8" ) : # pragma : no cover
    """Use default hash method to return hash value of a piece of string
 default setting use ' utf - 8 ' encoding ."""

implementation:
m = hash_meth ( )
m . update ( text . encode ( encoding ) )
return m . hexdigest ( )


signature:
def md5file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) :
    """Return md5 hash value of a piece of a file
 Estimate processing time on :
 : param abspath : the absolute path to the file
 : param nbytes : only has first N bytes of the file . if 0 or None ,
 hash all file
 CPU = i7-4600U 2.10GHz - 2.70GHz , RAM = 8.00 GB
 1 second can process 0.25GB data
 - 0.59G - 2.43 sec
 - 1.3G - 5.68 sec
 - 1.9G - 7.72 sec
 - 2.5G - 10.32 sec
 - 3.9G - 16.0 sec"""

implementation: return get_file_fingerprint ( abspath , hashlib . md5 , nbytes = nbytes , chunk_size = chunk_size )


signature:
def sha256file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) :
    """Return sha256 hash value of a piece of a file
 Estimate processing time on :
 : param abspath : the absolute path to the file
 : param nbytes : only has first N bytes of the file . if 0 or None ,
 hash all file"""

implementation: return get_file_fingerprint ( abspath , hashlib . sha256 , nbytes = nbytes , chunk_size = chunk_size )


signature:
def sha512file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) :
    """Return sha512 hash value of a piece of a file
 Estimate processing time on :
 : param abspath : the absolute path to the file
 : param nbytes : only has first N bytes of the file . if 0 or None ,
 hash all file"""

implementation: return get_file_fingerprint ( abspath , hashlib . sha512 , nbytes = nbytes , chunk_size = chunk_size )


signature:
def register ( registerable : Any ) :
    """Registers an object , notifying any listeners that may be interested in it .
 : param registerable : the object to register"""

implementation:
listenable = registration_event_listenable_map [ type ( registerable ) ]
event = RegistrationEvent ( registerable , RegistrationEvent . Type . REGISTERED )
listenable . notify_listeners ( event )


signature:
def unregister ( registerable : Any ) :
    """Unregisters an object , notifying any listeners that may be interested in it .
 : param registerable : the object to unregister"""

implementation:
listenable = registration_event_listenable_map [ type ( registerable ) ]
event = RegistrationEvent ( registerable , RegistrationEvent . Type . UNREGISTERED )
listenable . notify_listeners ( event )


signature:
def _load_module ( path : str ) :
    """Dynamically loads the python module at the given path .
 : param path : the path to load the module from"""

implementation:
spec = spec_from_file_location ( os . path . basename ( path ) , path )
module = module_from_spec ( spec )
spec . loader . exec_module ( module )


signature:
def is_empty ( self , strict = True ) :
    """- If it ' s a file , check if it is a empty file . ( 0 bytes content )
 - If it ' s a directory , check if there ' s no file and dir in it .
 But if ` ` strict = False ` ` , then only check if there ' s no file in it .
 : param strict : only useful when it is a directory . if True , only
 return True if this dir has no dir and file . if False , return True
 if it doesn ' t have any file ."""

implementation:
if self . exists ( ) :
    if self . is_file ( ) :
        return self . size == 0
    elif self . is_dir ( ) :
        if strict :
            return len ( list ( self . select ( recursive = True ) ) ) == 0
        else : # pragma : no cover
            return len ( list ( self . select_file ( recursive = True ) ) ) == 0
    else : # pragma : no cover
        msg = "'%s' is not either file or directory! (maybe simlink)" % self
        raise EnvironmentError ( msg )
else :
    raise EnvironmentError ( "'%s' not exists!" % self )


signature:
def auto_complete_choices ( self , case_sensitive = False ) :
    """A command line auto complete similar behavior . Find all item with same
 prefix of this one .
 : param case _ sensitive : toggle if it is case sensitive .
 : return : list of : class : ` pathlib _ mate . pathlib2 . Path ` ."""

implementation:
self_basename = self . basename
self_basename_lower = self . basename . lower ( )
if case_sensitive : # pragma : no cover
    def match ( basename ) :
        return basename . startswith ( self_basename )
else :
    def match ( basename ) :
        return basename . lower ( ) . startswith ( self_basename_lower )
choices = list ( )
if self . is_dir ( ) :
    choices . append ( self )
    for p in self . sort_by_abspath ( self . select ( recursive = False ) ) :
        choices . append ( p )
else :
    p_parent = self . parent
    if p_parent . is_dir ( ) :
        for p in self . sort_by_abspath ( p_parent . select ( recursive = False ) ) :
            if match ( p . basename ) :
                choices . append ( p )
    else : # pragma : no cover
        raise ValueError ( "'%s' directory does not exist!" % p_parent )
return choices


signature:
def print_big_dir ( self , top_n = 5 ) :
    """Print ` ` top _ n ` ` big dir in this dir ."""

implementation:
self . assert_is_dir_and_exists ( )
size_table = sorted ( [ ( p , p . dirsize ) for p in self . select_dir ( recursive = False ) ] , key = lambda x : x [ 1 ] , reverse = True , )
for p , size in size_table [ : top_n ] :
    print ( "{:<9}    {:<9}" . format ( repr_data_size ( size ) , p . abspath ) )


signature:
def print_big_file ( self , top_n = 5 ) :
    """Print ` ` top _ n ` ` big file in this dir ."""

implementation:
self . assert_is_dir_and_exists ( )
size_table = sorted ( [ ( p , p . size ) for p in self . select_file ( recursive = True ) ] , key = lambda x : x [ 1 ] , reverse = True , )
for p , size in size_table [ : top_n ] :
    print ( "{:<9}    {:<9}" . format ( repr_data_size ( size ) , p . abspath ) )


signature:
def print_big_dir_and_big_file ( self , top_n = 5 ) :
    """Print ` ` top _ n ` ` big dir and ` ` top _ n ` ` big file in each dir ."""

implementation:
self . assert_is_dir_and_exists ( )
size_table1 = sorted ( [ ( p , p . dirsize ) for p in self . select_dir ( recursive = False ) ] , key = lambda x : x [ 1 ] , reverse = True , )
for p1 , size1 in size_table1 [ : top_n ] :
    print ( "{:<9}    {:<9}" . format ( repr_data_size ( size1 ) , p1 . abspath ) )
    size_table2 = sorted ( [ ( p , p . size ) for p in p1 . select_file ( recursive = True ) ] , key = lambda x : x [ 1 ] , reverse = True , )
    for p2 , size2 in size_table2 [ : top_n ] :
        print ( "    {:<9}    {:<9}" . format ( repr_data_size ( size2 ) , p2 . abspath ) )


signature:
def file_stat_for_all ( self , filters = all_true ) : # pragma : no cover
    """Find out how many files , directories and total size ( Include file in
 it ' s sub - folder ) it has for each folder and sub - folder .
 : returns : stat , a dict like ` ` { " directory path " : {
 " file " : number of files , " dir " : number of directories ,
 " size " : total size in bytes } } ` `
 * * 中文文档 * *
 返回一个目录中的每个子目录的 , 文件 , 文件夹 , 大小的统计数据 。"""

implementation:
self . assert_is_dir_and_exists ( )
from collections import OrderedDict
stat = OrderedDict ( )
stat [ self . abspath ] = { "file" : 0 , "dir" : 0 , "size" : 0 }
for p in self . select ( filters = filters , recursive = True ) :
    if p . is_file ( ) :
        size = p . size
        while 1 :
            parent = p . parent
            stat [ parent . abspath ] [ "file" ] += 1
            stat [ parent . abspath ] [ "size" ] += size
            if parent . abspath == self . abspath :
                break
            p = parent
    elif p . is_dir ( ) :
        stat [ p . abspath ] = { "file" : 0 , "dir" : 0 , "size" : 0 }
        while 1 :
            parent = p . parent
            stat [ parent . abspath ] [ "dir" ] += 1
            if parent . abspath == self . abspath :
                break
            p = parent
return stat


signature:
def file_stat ( self , filters = all_true ) :
    """Find out how many files , directorys and total size ( Include file in
 it ' s sub - folder ) .
 : returns : stat , a dict like ` ` { " file " : number of files ,
 " dir " : number of directorys , " size " : total size in bytes } ` `
 * * 中文文档 * *
 返回一个目录中的文件 , 文件夹 , 大小的统计数据 。"""

implementation:
self . assert_is_dir_and_exists ( )
stat = { "file" : 0 , "dir" : 0 , "size" : 0 }
for p in self . select ( filters = filters , recursive = True ) :
    if p . is_file ( ) :
        stat [ "file" ] += 1
        stat [ "size" ] += p . size
    elif p . is_dir ( ) :
        stat [ "dir" ] += 1
return stat


signature:
def mirror_to ( self , dst ) : # pragma : no cover
    """Create a new folder having exactly same structure with this directory .
 However , all files are just empty file with same file name .
 : param dst : destination directory . The directory can ' t exists before
 you execute this .
 * * 中文文档 * *
 创建一个目录的镜像拷贝 , 与拷贝操作不同的是 , 文件的副本只是在文件名上
 与原件一致 , 但是是空文件 , 完全没有内容 , 文件大小为0。"""

implementation:
self . assert_is_dir_and_exists ( )
src = self . abspath
dst = os . path . abspath ( dst )
if os . path . exists ( dst ) : # pragma : no cover
    raise Exception ( "distination already exist!" )
folder_to_create = list ( )
file_to_create = list ( )
for current_folder , _ , file_list in os . walk ( self . abspath ) :
    current_folder = current_folder . replace ( src , dst )
    try :
        os . mkdir ( current_folder )
    except : # pragma : no cover
        pass
    for basename in file_list :
        abspath = os . path . join ( current_folder , basename )
        with open ( abspath , "wb" ) as _ :
            pass


signature:
def execute_pyfile ( self , py_exe = None ) : # pragma : no cover
    """Execute every ` ` . py ` ` file as main script .
 : param py _ exe : str , python command or python executable path .
 * * 中文文档 * *
 将目录下的所有Python文件作为主脚本用当前解释器运行 。"""

implementation:
import subprocess
self . assert_is_dir_and_exists ( )
if py_exe is None :
    if six . PY2 :
        py_exe = "python2"
    elif six . PY3 :
        py_exe = "python3"
for p in self . select_by_ext ( ".py" ) :
    subprocess . Popen ( '%s "%s"' % ( py_exe , p . abspath ) )


signature:
def trail_space ( self , filters = lambda p : p . ext == ".py" ) : # pragma : no cover
    """Trail white space at end of each line for every ` ` . py ` ` file .
 * * 中文文档 * *
 将目录下的所有被选择的文件中行末的空格删除 。"""

implementation:
self . assert_is_dir_and_exists ( )
for p in self . select_file ( filters ) :
    try :
        with open ( p . abspath , "rb" ) as f :
            lines = list ( )
            for line in f :
                lines . append ( line . decode ( "utf-8" ) . rstrip ( ) )
        with open ( p . abspath , "wb" ) as f :
            f . write ( "\n" . join ( lines ) . encode ( "utf-8" ) )
    except Exception as e : # pragma : no cover
        raise e


signature:
def autopep8 ( self , ** kwargs ) : # pragma : no cover
    """Auto convert your python code in a directory to pep8 styled code .
 : param kwargs : arguments for ` ` autopep8 . fix _ code ` ` method .
 * * 中文文档 * *
 将目录下的所有Python文件用pep8风格格式化 。 增加其可读性和规范性 。"""

implementation:
self . assert_is_dir_and_exists ( )
for p in self . select_by_ext ( ".py" ) :
    with open ( p . abspath , "rb" ) as f :
        code = f . read ( ) . decode ( "utf-8" )
    formatted_code = autopep8 . fix_code ( code , ** kwargs )
    with open ( p . abspath , "wb" ) as f :
        f . write ( formatted_code . encode ( "utf-8" ) )


signature:
def notify_listeners ( self , data : Optional [ _ListenableDataType ] = _NO_DATA_MARKER ) :
    """Notify event listeners , passing them the given data ( if any ) .
 : param data : the data to pass to the event listeners"""

implementation:
for listener in self . _listeners :
    if data is not Listenable . _NO_DATA_MARKER :
        listener ( data )
    else :
        listener ( )


signature:
def size ( self ) :
    """File size in bytes ."""

implementation:
try :
    return self . _stat . st_size
except : # pragma : no cover
    self . _stat = self . stat ( )
    return self . size


signature:
def mtime ( self ) :
    """Get most recent modify time in timestamp ."""

implementation:
try :
    return self . _stat . st_mtime
except : # pragma : no cover
    self . _stat = self . stat ( )
    return self . mtime


signature:
def atime ( self ) :
    """Get most recent access time in timestamp ."""

implementation:
try :
    return self . _stat . st_atime
except : # pragma : no cover
    self . _stat = self . stat ( )
    return self . atime


signature:
def ctime ( self ) :
    """Get most recent create time in timestamp ."""

implementation:
try :
    return self . _stat . st_ctime
except : # pragma : no cover
    self . _stat = self . stat ( )
    return self . ctime


signature:
def get_metainfo ( scriptfile , keywords = [ 'author' , 'contact' , 'copyright' , 'download' , 'git' , 'subversion' , 'version' , 'website' ] , special = { } , first_line_pattern = r'^(?P<progname>.+)(\s+v(?P<version>\S+))?' , keyword_pattern_template = r'^\s*%(pretty)s:\s*(?P<%(keyword)s>\S.+?)\s*$' , prettify = lambda kw : kw . capitalize ( ) . replace ( '_' , ' ' ) ) :
    """Dumb helper for pulling metainfo from a script _ _ doc _ _ string .
 Returns a metainfo dict with command , description , progname and the given
 keywords ( if present ) .
 This function will only make minimal efforts to succeed . If you need
 anything else : roll your own .
 The docstring needs to be multiline and the closing quotes need to be first
 on a line , optionally preceeded by whitespace .
 The first non - whitespace line is re . search ' ed using first _ line _ pattern ,
 default e . g ( version optional , contains no whitespace ) : PROGNAME [ vVERSION ]
 The next non - whitespace , non - keyword line is expected to be the program
 description .
 The following lines are re . search ' ed against a keyword : pattern dict which
 is constructed using
 keyword _ pattern % dict ( pretty = prettify ( keyword ) , keyword = keyword )
 Default prettify is keyword . capitalize ( ) . replace ( ' _ ' , ' ' ) . Example ,
 for the keyword " licence " will match the following line :
 License : The MIT license .
 and set the license metainfo to " The MIT license . " .
 Any keyword : pattern pairs that need special treatment can be supplied with
 special ."""

implementation:
patterns = dict ( ( kw , re . compile ( keyword_pattern_template % dict ( pretty = prettify ( kw ) , keyword = kw ) ) ) for kw in keywords )
patterns . update ( special )
metainfo = dict ( )
if scriptfile [ - 4 : ] in [ '.pyc' , '.pyo' ] :
    scriptfile = scriptfile [ : - 1 ]
script = open ( scriptfile )
closer = ''
for line in script :
    line = line . strip ( )
    if not line or line . startswith ( '#' ) :
        continue
    if line [ : 3 ] in ( '"""' , "'''" ) :
        closer = line [ : 3 ]
        break
    raise ValueError ( 'file contains no docstring' )
if not line :
    for line in script :
        line = line . strip ( )
        if line :
            break
g = re . search ( first_line_pattern , line [ 3 : ] ) . groupdict ( )
metainfo [ 'progname' ] = g [ 'progname' ]
if g [ 'version' ] :
    metainfo [ 'version' ] = g [ 'version' ]
for line in script :
    if line . strip ( ) . startswith ( closer ) :
        break
    for keyword , pattern in patterns . items ( ) :
        m = pattern . search ( line )
        if m :
            metainfo [ keyword ] = m . group ( keyword )
            break
    if line . strip ( ) and not 'description' in metainfo :
        metainfo [ 'description' ] = line . strip ( )
return metainfo


signature:
def unusedoptions ( self , sections ) :
    """Lists options that have not been used to format other values in
 their sections .
 Good for finding out if the user has misspelled any of the options ."""

implementation:
unused = set ( [ ] )
for section in _list ( sections ) :
    if not self . has_section ( section ) :
        continue
    options = self . options ( section )
    raw_values = [ self . get ( section , option , raw = True ) for option in options ]
    for option in options :
        formatter = "%(" + option + ")s"
        for raw_value in raw_values :
            if formatter in raw_value :
                break
        else :
            unused . add ( option )
    return list ( unused )


signature:
def parse ( self , argv , usedname , location ) :
    """Consume and process arguments and store the result .
 ARGS :
 argv < list str > :
 The argument list to parse .
 usedname < str > :
 The string used by the user to invoke the option .
 location < str > :
 A user friendly sring describing where the parser got this
 data from ."""

implementation:
try :
    value = self . format . parse ( argv )
except formats . BadNumberOfArguments , e :
    raise BadNumberOfArguments ( usedname , e . required , e . supplied )
except formats . BadArgument , e :
    raise BadArgument ( usedname , e . argument , e . message )
if self . recurring :
    self . value . append ( value )
else :
    self . value = value
self . location = location


signature:
def parsestr ( self , argsstr , usedname , location ) :
    """Parse a string lexically and store the result .
 ARGS :
 argsstr < str > :
 The string to parse .
 usedname < str > :
 The string used by the user to invoke the option .
 location < str > :
 A user friendly sring describing where the parser got this
 data from ."""

implementation:
try :
    value = self . format . parsestr ( argsstr )
except formats . BadNumberOfArguments , e :
    raise BadNumberOfArguments ( usedname , e . required , e . supplied )
except formats . BadArgument , e :
    raise BadArgument ( usedname , e . argument , e . message )
if self . recurring :
    self . value . append ( value )
else :
    self . value = value
self . location = location


signature:
def parse ( self , argv ) :
    """Consume and process arguments and store the result .
 argv is the list of arguments to parse ( will be modified ) .
 Recurring PositionalArgumants get a list as . value .
 Optional PositionalArguments that do not get any arguments to parse get
 None as . value , or [ ] if recurring ."""

implementation:
if not argv and self . optional :
    self . value = [ ] if self . recurring else None
    return
try :
    value = self . format . parse ( argv )
    if not self . recurring :
        self . value = value
        return
    self . value = [ value ]
    while argv :
        self . value . append ( self . format . parse ( argv ) )
except formats . BadNumberOfArguments , e :
    raise BadNumberOfArguments ( self . displayname , e . required , e . given )
except formats . BadArgument , e :
    raise BadArgument ( self . displayname , e . argument , e . details )


signature:
def keys ( self ) :
    """List names of options and positional arguments ."""

implementation: return self . options . keys ( ) + [ p . name for p in self . positional_args ]


signature:
def values ( self ) :
    """List values of options and positional arguments ."""

implementation: return self . options . values ( ) + [ p . value for p in self . positional_args ]


signature:
def items ( self ) :
    """List values of options and positional arguments ."""

implementation: return [ ( p . name , p . value ) for p in self . options . values ( ) + self . positional_args ]


signature:
def getparam ( self , key ) :
    """Get option or positional argument , by name , index or abbreviation .
 Abbreviations must be prefixed by a ' - ' character , like so : ui [ ' - a ' ]"""

implementation:
try :
    return self . options [ key ]
except :
    pass
for posarg in self . positional_args :
    if posarg . name == key :
        return posarg
try :
    return self . abbreviations [ key [ 1 : ] ]
except :
    raise KeyError ( 'no such option or positional argument' )


signature:
def _add_option ( self , option ) :
    """Add an Option object to the user interface ."""

implementation:
if option . name in self . options :
    raise ValueError ( 'name already in use' )
if option . abbreviation in self . abbreviations :
    raise ValueError ( 'abbreviation already in use' )
if option . name in [ arg . name for arg in self . positional_args ] :
    raise ValueError ( 'name already in use by a positional argument' )
self . options [ option . name ] = option
if option . abbreviation :
    self . abbreviations [ option . abbreviation ] = option
self . option_order . append ( option . name )


signature:
def _add_positional_argument ( self , posarg ) :
    """Append a positional argument to the user interface .
 Optional positional arguments must be added after the required ones .
 The user interface can have at most one recurring positional argument ,
 and if present , that argument must be the last one ."""

implementation:
if self . positional_args :
    if self . positional_args [ - 1 ] . recurring :
        raise ValueError ( "recurring positional arguments must be last" )
    if self . positional_args [ - 1 ] . optional and not posarg . optional :
        raise ValueError ( "required positional arguments must precede optional ones" )
self . positional_args . append ( posarg )


signature:
def read_docs ( self , docsfiles ) :
    """Read program documentation from a DocParser compatible file .
 docsfiles is a list of paths to potential docsfiles : parse if present .
 A string is taken as a list of one item ."""

implementation:
updates = DocParser ( )
for docsfile in _list ( docsfiles ) :
    if os . path . isfile ( docsfile ) :
        updates . parse ( docsfile )
self . docs . update ( ( k , _docs ( updates [ k ] , self . docvars ) ) for k in self . docs if updates . blocks [ k ] )
for name , text in updates [ 'parameters' ] . items ( ) :
    if name in self :
        self . getparam ( name ) . docs = text [ 0 ] % self . docvars
    elif name not in self . ignore :
        raise ValueError ( "parameter %r does not exist" % name )


signature:
def parse_files ( self , files = None , sections = None ) :
    """Parse configfiles .
 files < list str > , < str > or None :
 What files to parse . None means use self . configfiles . New
 values override old ones . A string value will be interpreted
 as a list of one item .
 sections < list str > , < str > or None :
 Which sections to parse from the files . None means use
 self . sections . A string value will be interpreted as a list
 of one item . The [ DEFAULT ]
 section is always read , and it is read as if its contents were
 copied to the beginning of all other sections ."""

implementation:
files = _list ( files , self . configfiles )
sections = _list ( sections , self . sections )
for file in files :
    parser = StrictConfigParser ( )
    parser . read ( file )
    for section in sections :
        if not parser . has_section ( section ) :
            continue
        for unused in parser . unusedoptions ( section ) :
            if unused not in self . options and unused not in self . ignore :
                templ = "The option %r in section [%s] of file %s does not exist."
                raise InvalidOption ( unused , message = templ % ( unused , section , file ) )
        for name in parser . options ( section ) :
            if name in self . options :
                if self . options [ name ] . reserved :
                    templ = "The option %s in section [%s] of file %s is reserved for command line use."
                    raise ReservedOptionError ( message = templ % ( unused , section , file ) )
                value = parser . get ( section , name )
                self . options [ name ] . parsestr ( value , name , '%s [%s]' % ( file , section ) )


signature:
def _parse_options ( self , argv , location ) :
    """Parse the options part of an argument list .
 IN :
 lsArgs < list str > :
 List of arguments . Will be altered .
 location < str > :
 A user friendly string describing where this data came from ."""

implementation:
observed = [ ]
while argv :
    if argv [ 0 ] . startswith ( '--' ) :
        name = argv . pop ( 0 ) [ 2 : ]
        # ' - - ' means end of options .
        if not name :
            break
        if name not in self . options :
            raise InvalidOption ( name )
        option = self . options [ name ]
        if not option . recurring :
            if option in observed :
                raise OptionRecurrenceError ( name )
            observed . append ( option )
        option . parse ( argv , name , location )
    elif argv [ 0 ] . startswith ( '-' ) : # A single - is not an abbreviation block , but the first positional arg .
        if argv [ 0 ] == '-' :
            break
        block = argv . pop ( 0 ) [ 1 : ]
        # Abbrevs for options that take values go last in the block .
        for abbreviation in block [ : - 1 ] :
            if self . abbreviations [ abbreviation ] . nargs != 0 :
                raise BadAbbreviationBlock ( abbreviation , block , "options that require value arguments must be last in abbreviation blocks" )
        # Parse individual options .
        for abbreviation in block :
            option = self . abbreviations [ abbreviation ]
            if not option . recurring :
                if option in observed :
                    raise OptionRecurrenceError ( option . name )
                observed . append ( option )
            option . parse ( argv , '-' + abbreviation , location )
    # only arguments that start with - - or - can be Options .
    else :
        break


signature:
def _parse_positional_arguments ( self , argv ) :
    """Parse the positional arguments part of an argument list .
 argv < list str > :
 List of arguments . Will be altered ."""

implementation:
for posarg in self . positional_args :
    posarg . parse ( argv )
if argv :
    if None in [ p . nargs for p in self . positional_args ] :
        msg = '%s too many argument%s given'
        plural_s = len ( argv ) > 1 and 's' or ''
        raise BadNumberOfArguments ( message = msg % ( len ( argv ) , plural_s ) )
    msg = 'This program accepts exactly %s positional arguments (%s given).'
    required = len ( [ p . nargs for p in self . positional_args ] )
    raise BadNumberOfArguments ( message = msg % ( required , required + len ( argv ) ) )


signature:
def parse_argv ( self , argv = None , location = 'Command line.' ) :
    """Parse command line arguments .
 args < list str > or None :
 The argument list to parse . None means use a copy of sys . argv . argv [ 0 ] is
 ignored .
 location = ' ' < str > :
 A user friendly string describing where the parser got this
 data from . ' ' means use " Command line . " if args = = None , and
 " Builtin default . " otherwise ."""

implementation:
if argv is None :
    argv = list ( sys . argv )
argv . pop ( 0 )
self . _parse_options ( argv , location )
self . _parse_positional_arguments ( argv )


signature:
def optionhelp ( self , indent = 0 , maxindent = 25 , width = 79 ) :
    """Return user friendly help on program options ."""

implementation:
def makelabels ( option ) :
    labels = '%*s--%s' % ( indent , ' ' , option . name )
    if option . abbreviation :
        labels += ', -' + option . abbreviation
    return labels + ': '
docs = [ ]
helpindent = _autoindent ( [ makelabels ( o ) for o in self . options . values ( ) ] , indent , maxindent )
for name in self . option_order :
    option = self . options [ name ]
    labels = makelabels ( option )
    helpstring = "%s(%s). %s" % ( option . formatname , option . strvalue , option . docs )
    wrapped = self . _wrap_labelled ( labels , helpstring , helpindent , width )
    docs . extend ( wrapped )
return '\n' . join ( docs )


signature:
def posarghelp ( self , indent = 0 , maxindent = 25 , width = 79 ) :
    """Return user friendly help on positional arguments in the program ."""

implementation:
docs = [ ]
makelabel = lambda posarg : ' ' * indent + posarg . displayname + ': '
helpindent = _autoindent ( [ makelabel ( p ) for p in self . positional_args ] , indent , maxindent )
for posarg in self . positional_args :
    label = makelabel ( posarg )
    text = posarg . formatname + '. ' + posarg . docs
    wrapped = self . _wrap_labelled ( label , text , helpindent , width )
    docs . extend ( wrapped )
return '\n' . join ( docs )


signature:
def format_usage ( self , usage = None ) :
    """Return a formatted usage string .
 If usage is None , use self . docs [ ' usage ' ] , and if that is also None ,
 generate one ."""

implementation:
if usage is None :
    usage = self . docs [ 'usage' ]
if usage is not None :
    return usage [ 0 ] % self . docvars
usage = self . docvars [ 'command' ]
if self . basic_option_names . get ( 'help' ) :
    usage += ' [--%s]' % self . basic_option_names . get ( 'help' )
if self . options :
    usage += ' <OPTIONS>'
optional = 0
for posarg in self . positional_args :
    usage += ' '
    if posarg . optional :
        usage += "["
        optional += 1
    usage += posarg . displayname
    if posarg . recurring :
        usage += ' [%s2 [...]]' % posarg . displayname
usage += ']' * optional
return usage


signature:
def _wrap ( self , text , indent = 0 , width = 0 ) :
    """Textwrap an indented paragraph .
 ARGS :
 width = 0 < int > :
 Maximum allowed page width . 0 means use default from
 self . iMaxHelpWidth ."""

implementation:
text = _list ( text )
if not width :
    width = self . width
paragraph = text [ 0 ] . lstrip ( )
s = ' ' * ( len ( text [ 0 ] ) - len ( paragraph ) + indent )
wrapped = textwrap . wrap ( paragraph . strip ( ) , width , initial_indent = s , subsequent_indent = s )
return '\n' . join ( wrapped )


signature:
def _wraptext ( self , text , indent = 0 , width = 0 ) :
    """Shorthand for ' \n ' . join ( self . _ wrap ( par , indent , width ) for par in text ) ."""

implementation: return '\n' . join ( self . _wrap ( par , indent , width ) for par in text )


signature:
def _wrapusage ( self , usage = None , width = 0 ) :
    """Textwrap usage instructions .
 ARGS :
 width = 0 < int > :
 Maximum allowed page width . 0 means use default from
 self . iMaxHelpWidth ."""

implementation:
if not width :
    width = self . width
return textwrap . fill ( 'USAGE: ' + self . format_usage ( usage ) , width = width , subsequent_indent = '    ...' )


signature:
def shorthelp ( self , width = 0 ) :
    """Return brief help containing Title and usage instructions .
 ARGS :
 width = 0 < int > :
 Maximum allowed page width . 0 means use default from
 self . iMaxHelpWidth ."""

implementation:
out = [ ]
out . append ( self . _wrap ( self . docs [ 'title' ] , width = width ) )
if self . docs [ 'description' ] :
    out . append ( self . _wrap ( self . docs [ 'description' ] , indent = 2 , width = width ) )
out . append ( '' )
out . append ( self . _wrapusage ( width = width ) )
out . append ( '' )
return '\n' . join ( out )


signature:
def strsettings ( self , indent = 0 , maxindent = 25 , width = 0 ) :
    """Return user friendly help on positional arguments .
 indent is the number of spaces preceeding the text on each line .
 The indent of the documentation is dependent on the length of the
 longest label that is shorter than maxindent . A label longer than
 maxindent will be printed on its own line .
 width is maximum allowed page width , use self . width if 0."""

implementation:
out = [ ]
makelabel = lambda name : ' ' * indent + name + ': '
settingsindent = _autoindent ( [ makelabel ( s ) for s in self . options ] , indent , maxindent )
for name in self . option_order :
    option = self . options [ name ]
    label = makelabel ( name )
    settingshelp = "%s(%s): %s" % ( option . formatname , option . strvalue , option . location )
    wrapped = self . _wrap_labelled ( label , settingshelp , settingsindent , width )
    out . extend ( wrapped )
return '\n' . join ( out )


signature:
def settingshelp ( self , width = 0 ) :
    """Return a summary of program options , their values and origins .
 width is maximum allowed page width , use self . width if 0."""

implementation:
out = [ ]
out . append ( self . _wrap ( self . docs [ 'title' ] , width = width ) )
if self . docs [ 'description' ] :
    out . append ( self . _wrap ( self . docs [ 'description' ] , indent = 2 , width = width ) )
out . append ( '' )
out . append ( 'SETTINGS:' )
out . append ( self . strsettings ( indent = 2 , width = width ) )
out . append ( '' )
return '\n' . join ( out )


signature:
def launch ( self , argv = None , showusageonnoargs = False , width = 0 , helphint = "Use with --help for more information.\n" , debug_parser = False ) :
    """Do the usual stuff to initiallize the program .
 Read config files and parse arguments , and if the user has used any
 of the help / version / settings options , display help and exit .
 If debug _ parser is false , don ' t catch ParseErrors and exit with user
 friendly help . Crash with traceback instead .
 configfiles is a list of config files . None means use self . configfiles .
 sections is a list of configfile section names to read . None means use
 self . sections .
 argv is a list of arguments to parse . Will be modified . None means use
 copy of sys . argv . argv [ 0 ] is ignored .
 If showusageonnoargs is true , show usage and exit if the user didn ' t
 give any args . Should be False if there are no required PositionalArgs .
 width is the maximum allowed page width . 0 means use self . width .
 helphint is a string that hints on how to get more help which is
 displayed at the end of usage help messages ."""

implementation:
if showusageonnoargs and len ( argv ) == 1 :
    print self . shorthelp ( width = width )
    if helphint :
        print self . _wrap ( helphint , indent = 2 , width = width )
    sys . exit ( 0 )
parsing_error = None
try :
    self . parse_files ( )
    self . parse_argv ( argv )
except ParseError , parsing_error :
    if debug_parser :
        raise
for optiontype in [ 'help' , 'longhelp' , 'settings' , 'version' ] :
    name = self . basic_option_names . get ( optiontype )
    if name and self [ name ] :
        methodname = optiontype . rstrip ( 'help' ) + 'help'
        print getattr ( self , methodname ) ( width )
        sys . exit ( )
if parsing_error :
    self . graceful_exit ( parsing_error , width )


signature:
def parse ( self , file ) :
    """Parse text blocks from a file ."""

implementation:
if isinstance ( file , basestring ) :
    file = open ( file )
line_number = 0
label = None
block = self . untagged
for line in file :
    line_number += 1
    line = line . rstrip ( '\n' )
    if self . tabsize > 0 :
        line = line . replace ( '\t' , ' ' * self . tabsize )
    if self . decommenter :
        line = self . decommenter . decomment ( line )
        if line is None :
            continue
    tag = line . split ( ':' , 1 ) [ 0 ] . strip ( )
    # Still in the same block ?
    if tag not in self . names :
        if block is None :
            if line and not line . isspace ( ) :
                raise ParseError ( file . name , line , "garbage before first block: %r" % line )
            continue
        block . addline ( line )
        continue
    # Open a new block .
    name = self . names [ tag ]
    label = line . split ( ':' , 1 ) [ 1 ] . strip ( )
    if name in self . labelled_classes :
        if not label :
            raise ParseError ( file . name , line , "missing label for %r block" % name )
        block = self . blocks [ name ] . setdefault ( label , self . labelled_classes [ name ] ( ) )
    else :
        if label :
            msg = "label %r present for unlabelled block %r" % ( label , name )
            raise ParseError ( file . name , line_number , msg )
        block = self . blocks [ name ]
    block . startblock ( )


signature:
def get_format ( format ) :
    """Get a format object .
 If format is a format object , return unchanged . If it is a string
 matching one of the BaseFormat subclasses in the tui . formats module
 ( case insensitive ) , return an instance of that class . Otherwise assume
 it ' a factory function for Formats ( such as a class ) so call and return ,
 and raise ValueError on error ."""

implementation:
if isinstance ( format , BaseFormat ) :
    return format
if isinstance ( format , basestring ) :
    for name , formatclass in globals ( ) . items ( ) :
        if name . lower ( ) == format . lower ( ) :
            if not issubclass ( formatclass , BaseFormat ) :
                raise ValueError ( '%s is not the name of a format class' % format )
            return formatclass ( )
try :
    return format ( )
except :
    raise ValueError ( 'no such format' )


signature:
def parsestr ( self , argstr ) :
    """Parse arguments found in settings files .
 argstr is the string that should be parsed . Use e . g . ' " " ' to pass an
 empty string .
 if self . nargs > 1 a list of parsed values will be returned .
 NOTE : formats with nargs = = 0 or None probably want to override this
 method ."""

implementation:
argv = shlex . split ( argstr , comments = True )
if len ( argv ) != self . nargs :
    raise BadNumberOfArguments ( self . nargs , len ( argv ) )
return self . parse ( argv )


signature:
def parse_argument ( self , arg ) :
    """Parse a single argument .
 Lookup arg in self . specials , or call . to _ python ( ) if absent . Raise
 BadArgument on errors ."""

implementation:
lookup = self . casesensitive and arg or arg . lower ( )
if lookup in self . special :
    return self . special [ lookup ]
try :
    return self . to_python ( arg , * self . args , ** self . kw )
except Exception , e :
    raise BadArgument ( arg , str ( e ) )


signature:
def parse ( self , argv ) :
    """Pop , parse and return the first self . nargs items from args .
 if self . nargs > 1 a list of parsed values will be returned .
 Raise BadNumberOfArguments or BadArgument on errors .
 NOTE : argv may be modified in place by this method ."""

implementation:
if len ( argv ) < self . nargs :
    raise BadNumberOfArguments ( self . nargs , len ( argv ) )
if self . nargs == 1 :
    return self . parse_argument ( argv . pop ( 0 ) )
return [ self . parse_argument ( argv . pop ( 0 ) ) for tmp in range ( self . nargs ) ]


signature:
def present ( self , value ) :
    """Return a user - friendly representation of a value .
 Lookup value in self . specials , or call . to _ literal ( ) if absent ."""

implementation:
for k , v in self . special . items ( ) :
    if v == value :
        return k
return self . to_literal ( value , * self . args , ** self . kw )


signature:
def parsestr ( self , argstr ) :
    """Parse arguments found in settings files .
 Use the values in self . true for True in settings files , or those in
 self . false for False , case insensitive ."""

implementation:
argv = shlex . split ( argstr , comments = True )
if len ( argv ) != 1 :
    raise BadNumberOfArguments ( 1 , len ( argv ) )
arg = argv [ 0 ]
lower = arg . lower ( )
if lower in self . true :
    return True
if lower in self . false :
    return False
raise BadArgument ( arg , "Allowed values are " + self . allowed + '.' )


signature:
def parse ( self , argv ) :
    """Pop , parse and return the first arg from argv .
 The arg will be . split ( ) based on self . separator and the ( optionally
 stripped ) items will be parsed by self . format and returned as a list .
 Raise BadNumberOfArguments or BadArgument on errors .
 NOTE : args will be modified ."""

implementation:
if not argv :
    raise BadNumberOfArguments ( 1 , 0 )
argument = argv . pop ( 0 )
lookup = self . casesensitive and argument or argument . lower ( )
if lookup in self . special :
    return self . special [ lookup ]
argv = [ ( self . strip and s . strip ( ) or s ) for s in argument . split ( self . separator ) ]
values = [ ]
while argv :
    values . append ( self . format . parse ( argv ) )
return values


implementation:
for k , v in self . special . items ( ) :
    if v == value :
        return k
return self . separator . join ( self . format . present ( v ) for v in value )


signature:
def get_separator ( self , i ) :
    """Return the separator that preceding format i , or ' ' for i = = 0."""

implementation: return i and self . separator [ min ( i - 1 , len ( self . separator ) - 1 ) ] or ''


signature:
def parse ( self , argv ) :
    """Pop , parse and return the first arg from argv .
 The arg will be repeatedly . split ( x , 1 ) based on self . get _ separator ( ) and the
 ( optionally stripped ) items will be parsed by self . format and returned
 as a list .
 Raise BadNumberOfArguments or BadArgument on errors .
 NOTE : args will be modified ."""

implementation:
if not argv :
    raise BadNumberOfArguments ( 1 , 0 )
remainder = argv . pop ( 0 )
lookup = self . casesensitive and remainder or remainder . lower ( )
if lookup in self . special :
    return self . special [ lookup ]
values = [ ]
for i , format in enumerate ( self . format [ : - 1 ] ) :
    print i , self . get_separator ( i )
    try :
        arg , remainder = remainder . split ( self . get_separator ( i + 1 ) , 1 )
    except :
        raise BadArgument ( remainder , 'does not contain required separator ' + repr ( self . get_separator ( i + 1 ) ) )
    if self . strip :
        arg = arg . strip ( )
    values . append ( format . parse ( [ arg ] ) )
if self . strip :
    remainder = remainder . strip ( )
values . append ( format . parse ( [ remainder ] ) )
return values


implementation:
for k , v in self . special . items ( ) :
    if v == value :
        return k
return '' . join ( self . get_separator ( i ) + self . format [ i ] . present ( v ) for i , v in enumerate ( value ) )


signature:
def authorize_url ( self ) :
    """Return a URL to redirect the user to for OAuth authentication ."""

implementation:
auth_url = OAUTH_ROOT + '/authorize'
params = { 'client_id' : self . client_id , 'redirect_uri' : self . redirect_uri , }
return "{}?{}" . format ( auth_url , urlencode ( params ) )


signature:
def exchange_token ( self , code ) :
    """Exchange the authorization code for an access token ."""

implementation:
access_token_url = OAUTH_ROOT + '/access_token'
params = { 'client_id' : self . client_id , 'client_secret' : self . client_secret , 'redirect_uri' : self . redirect_uri , 'code' : code , }
resp = requests . get ( access_token_url , params = params )
if not resp . ok :
    raise MixcloudOauthError ( "Could not get access token." )
return resp . json ( ) [ 'access_token' ]


signature:
def get_open_port ( ) -> int :
    """Gets a PORT that will ( probably ) be available on the machine .
 It is possible that in - between the time in which the open PORT of found and when it is used , another process may
 bind to it instead .
 : return : the ( probably ) available PORT"""

implementation:
free_socket = socket . socket ( socket . AF_INET , socket . SOCK_STREAM )
free_socket . bind ( ( "" , 0 ) )
free_socket . listen ( 1 )
port = free_socket . getsockname ( ) [ 1 ]
free_socket . close ( )
return port


signature:
def extract_version_number ( string : str ) -> str :
    """Extracts a version from a string in the form : ` . * [ 0-9 ] + ( _ [ 0-9 ] + ) * . * ` , e . g . Irods4_1_9CompatibleController .
 If the string contains multiple version numbers , the first ( from left ) is extracted .
 Will raise a ` ValueError ` if there is no version number in the given string .
 : param string : the string containing the version number
 : return : the extracted version"""

implementation:
matched = _EXTRACT_VERSION_PATTERN . search ( string )
if matched is None :
    raise ValueError ( "No version number in string" )
return matched . group ( ) . replace ( "_" , "." )


signature:
def acquire ( self , * args , ** kwargs ) :
    """Wraps Lock . acquire"""

implementation:
with self . _stat_lock :
    self . _waiting += 1
self . _lock . acquire ( * args , ** kwargs )
with self . _stat_lock :
    self . _locked = True
    self . _waiting -= 1


signature:
def release ( self ) :
    """Wraps Lock . release"""

implementation:
self . _lock . release ( )
with self . _stat_lock :
    self . _locked = False
    self . _last_released = datetime . now ( )


signature:
def default_decoder ( self , obj ) :
    """Handle a dict that might contain a wrapped state for a custom type ."""

implementation:
typename , marshalled_state = self . unwrap_callback ( obj )
if typename is None :
    return obj
try :
    cls , unmarshaller = self . serializer . unmarshallers [ typename ]
except KeyError :
    raise LookupError ( 'no unmarshaller found for type "{}"' . format ( typename ) ) from None
if cls is not None :
    instance = cls . __new__ ( cls )
    unmarshaller ( instance , marshalled_state )
    return instance
else :
    return unmarshaller ( marshalled_state )


signature:
def wrap_state_dict ( self , typename : str , state ) -> Dict [ str , Any ] :
    """Wrap the marshalled state in a dictionary .
 The returned dictionary has two keys , corresponding to the ` ` type _ key ` ` and ` ` state _ key ` `
 options . The former holds the type name and the latter holds the marshalled state .
 : param typename : registered name of the custom type
 : param state : the marshalled state of the object
 : return : an object serializable by the serializer"""

implementation: return { self . type_key : typename , self . state_key : state }


signature:
def unwrap_state_dict ( self , obj : Dict [ str , Any ] ) -> Union [ Tuple [ str , Any ] , Tuple [ None , None ] ] :
    """Unwraps a marshalled state previously wrapped using : meth : ` wrap _ state _ dict ` ."""

implementation:
if len ( obj ) == 2 :
    typename = obj . get ( self . type_key )
    state = obj . get ( self . state_key )
    if typename is not None :
        return typename , state
return None , None


signature:
def publish ( quiet , dataset_uri ) :
    """Enable HTTP access to a dataset .
 This only works on datasets in some systems . For example , datasets stored
 in AWS S3 object storage and Microsoft Azure Storage can be published as
 datasets accessible over HTTP . A published dataset is world readable ."""

implementation:
access_uri = http_publish ( dataset_uri )
if not quiet :
    click . secho ( "Dataset accessible at " , nl = False , fg = "green" )
click . secho ( access_uri )


signature:
def register_custom_type ( self , cls : type , marshaller : Optional [ Callable [ [ Any ] , Any ] ] = default_marshaller , unmarshaller : Union [ Callable [ [ Any , Any ] , None ] , Callable [ [ Any ] , Any ] , None ] = default_unmarshaller , * , typename : str = None , wrap_state : bool = True ) -> None :
    """Register a marshaller and / or unmarshaller for the given class .
 The state object returned by the marshaller and passed to the unmarshaller can be any
 serializable type . Usually a dictionary mapping of attribute names to values is used .
 . . warning : : Registering marshallers / unmarshallers for any custom type will override any
 serializer specific encoding / decoding hooks ( respectively ) already in place !
 : param cls : the class to register
 : param marshaller : a callable that takes the object to be marshalled as the argument and
 returns a state object
 : param unmarshaller : a callable that either :
 * takes an uninitialized instance of ` ` cls ` ` and its state object as arguments and
 restores the state of the object
 * takes a state object and returns a new instance of ` ` cls ` `
 : param typename : a unique identifier for the type ( defaults to the ` ` module : varname ` `
 reference to the class )
 : param wrap _ state : ` ` True ` ` to wrap the marshalled state before serialization so that it
 can be recognized later for unmarshalling , ` ` False ` ` to serialize it as is"""

implementation:
assert check_argument_types ( )
typename = typename or qualified_name ( cls )
if marshaller :
    self . marshallers [ cls ] = typename , marshaller , wrap_state
    self . custom_type_codec . register_object_encoder_hook ( self )
if unmarshaller and self . custom_type_codec is not None :
    target_cls = cls
    # type : Optional [ type ]
    if len ( signature ( unmarshaller ) . parameters ) == 1 :
        target_cls = None
    self . unmarshallers [ typename ] = target_cls , unmarshaller
    self . custom_type_codec . register_object_decoder_hook ( self )


signature:
def _prompt_for_values ( d ) :
    """Update the descriptive metadata interactively .
 Uses values entered by the user . Note that the function keeps recursing
 whenever a value is another ` ` CommentedMap ` ` or a ` ` list ` ` . The
 function works as passing dictionaries and lists into a function edits
 the values in place ."""

implementation:
for key , value in d . items ( ) :
    if isinstance ( value , CommentedMap ) :
        _prompt_for_values ( value )
    elif isinstance ( value , list ) :
        for item in value :
            _prompt_for_values ( item )
    else :
        typ = type ( value )
        if isinstance ( value , ScalarFloat ) : # Deal with ruamel . yaml floats .
            typ = float
        new_value = click . prompt ( key , type = typ , default = value )
        d [ key ] = new_value
return d


signature:
def create ( quiet , name , base_uri , symlink_path ) :
    """Create a proto dataset ."""

implementation:
_validate_name ( name )
admin_metadata = dtoolcore . generate_admin_metadata ( name )
parsed_base_uri = dtoolcore . utils . generous_parse_uri ( base_uri )
if parsed_base_uri . scheme == "symlink" :
    if symlink_path is None :
        raise click . UsageError ( "Need to specify symlink path using the -s/--symlink-path option" )
# NOQA
if symlink_path :
    base_uri = dtoolcore . utils . sanitise_uri ( "symlink:" + parsed_base_uri . path )
    parsed_base_uri = dtoolcore . utils . generous_parse_uri ( base_uri )
# Create the dataset .
proto_dataset = dtoolcore . generate_proto_dataset ( admin_metadata = admin_metadata , base_uri = dtoolcore . utils . urlunparse ( parsed_base_uri ) , config_path = CONFIG_PATH )
# If we are creating a symlink dataset we need to set the symlink _ path
# attribute on the storage broker .
if symlink_path :
    symlink_abspath = os . path . abspath ( symlink_path )
    proto_dataset . _storage_broker . symlink_path = symlink_abspath
try :
    proto_dataset . create ( )
except dtoolcore . storagebroker . StorageBrokerOSError as err :
    raise click . UsageError ( str ( err ) )
proto_dataset . put_readme ( "" )
if quiet :
    click . secho ( proto_dataset . uri )
else : # Give the user some feedback and hints on what to do next .
    click . secho ( "Created proto dataset " , nl = False , fg = "green" )
    click . secho ( proto_dataset . uri )
    click . secho ( "Next steps: " )
    step = 1
    if parsed_base_uri . scheme != "symlink" :
        click . secho ( "{}. Add raw data, eg:" . format ( step ) )
        click . secho ( "   dtool add item my_file.txt {}" . format ( proto_dataset . uri ) , fg = "cyan" )
        if parsed_base_uri . scheme == "file" : # Find the abspath of the data directory for user feedback .
            data_path = proto_dataset . _storage_broker . _data_abspath
            click . secho ( "   Or use your system commands, e.g: " )
            click . secho ( "   mv my_data_directory {}/" . format ( data_path ) , fg = "cyan" )
        step = step + 1
    click . secho ( "{}. Add descriptive metadata, e.g: " . format ( step ) )
    click . secho ( "   dtool readme interactive {}" . format ( proto_dataset . uri ) , fg = "cyan" )
    step = step + 1
    click . secho ( "{}. Convert the proto dataset into a dataset: " . format ( step ) )
    click . secho ( "   dtool freeze {}" . format ( proto_dataset . uri ) , fg = "cyan" )


signature:
def name ( dataset_uri , new_name ) :
    """Report / update the name of the dataset .
 It is only possible to update the name of a proto dataset ,
 i . e . a dataset that has not yet been frozen ."""

implementation:
if new_name != "" :
    _validate_name ( new_name )
    try :
        dataset = dtoolcore . ProtoDataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH )
    except dtoolcore . DtoolCoreTypeError :
        dataset = dtoolcore . DataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH )
    dataset . update_name ( new_name )
admin_metadata = dtoolcore . _admin_metadata_from_uri ( uri = dataset_uri , config_path = CONFIG_PATH )
click . secho ( admin_metadata [ "name" ] )


signature:
def interactive ( proto_dataset_uri ) :
    """Interactive prompting to populate the readme ."""

implementation:
proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri , config_path = CONFIG_PATH )
# Create an CommentedMap representation of the yaml readme template .
readme_template = _get_readme_template ( )
yaml = YAML ( )
yaml . explicit_start = True
yaml . indent ( mapping = 2 , sequence = 4 , offset = 2 )
descriptive_metadata = yaml . load ( readme_template )
descriptive_metadata = _prompt_for_values ( descriptive_metadata )
# Write out the descriptive metadata to the readme file .
stream = StringIO ( )
yaml . dump ( descriptive_metadata , stream )
proto_dataset . put_readme ( stream . getvalue ( ) )
click . secho ( "Updated readme " , fg = "green" )
click . secho ( "To edit the readme using your default editor:" )
click . secho ( "dtool readme edit {}" . format ( proto_dataset_uri ) , fg = "cyan" )


signature:
def edit ( dataset_uri ) :
    """Default editor updating of readme content ."""

implementation:
try :
    dataset = dtoolcore . ProtoDataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH )
except dtoolcore . DtoolCoreTypeError :
    dataset = dtoolcore . DataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH )
readme_content = dataset . get_readme_content ( )
try : # Python2 compatibility .
    readme_content = unicode ( readme_content , "utf-8" )
except NameError :
    pass
edited_content = click . edit ( readme_content )
if edited_content is not None :
    _validate_and_put_readme ( dataset , edited_content )
    click . secho ( "Updated readme " , nl = False , fg = "green" )
else :
    click . secho ( "Did not update readme " , nl = False , fg = "red" )
click . secho ( dataset_uri )


signature:
def show ( dataset_uri ) :
    """Show the descriptive metadata in the readme ."""

implementation:
try :
    dataset = dtoolcore . ProtoDataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH )
except dtoolcore . DtoolCoreTypeError :
    dataset = dtoolcore . DataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH )
readme_content = dataset . get_readme_content ( )
click . secho ( readme_content )


signature:
def write ( proto_dataset_uri , input ) :
    """Use YAML from a file or stdin to populate the readme .
 To stream content from stdin use " - " , e . g .
 echo " desc : my data " | dtool readme write < DS _ URI > -"""

implementation:
proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri )
_validate_and_put_readme ( proto_dataset , input . read ( ) )


signature:
def item ( proto_dataset_uri , input_file , relpath_in_dataset ) :
    """Add a file to the proto dataset ."""

implementation:
proto_dataset = dtoolcore . ProtoDataSet . from_uri ( proto_dataset_uri , config_path = CONFIG_PATH )
if relpath_in_dataset == "" :
    relpath_in_dataset = os . path . basename ( input_file )
proto_dataset . put_item ( input_file , relpath_in_dataset )


signature:
def metadata ( proto_dataset_uri , relpath_in_dataset , key , value ) :
    """Add metadata to a file in the proto dataset ."""

implementation:
proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri , config_path = CONFIG_PATH )
proto_dataset . add_item_metadata ( handle = relpath_in_dataset , key = key , value = value )


signature:
def freeze ( proto_dataset_uri ) :
    """Convert a proto dataset into a dataset .
 This step is carried out after all files have been added to the dataset .
 Freezing a dataset finalizes it with a stamp marking it as frozen ."""

implementation:
proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri , config_path = CONFIG_PATH )
num_items = len ( list ( proto_dataset . _identifiers ( ) ) )
max_files_limit = int ( dtoolcore . utils . get_config_value ( "DTOOL_MAX_FILES_LIMIT" , CONFIG_PATH , 10000 ) )
assert isinstance ( max_files_limit , int )
if num_items > max_files_limit :
    click . secho ( "Too many items ({} > {}) in proto dataset" . format ( num_items , max_files_limit ) , fg = "red" )
    click . secho ( "1. Consider splitting the dataset into smaller datasets" )
    click . secho ( "2. Consider packaging small files using tar" )
    click . secho ( "3. Increase the limit using the DTOOL_MAX_FILES_LIMIT" )
    click . secho ( "   environment variable" )
    sys . exit ( 2 )
handles = [ h for h in proto_dataset . _storage_broker . iter_item_handles ( ) ]
for h in handles :
    if not valid_handle ( h ) :
        click . secho ( "Invalid item name: {}" . format ( h ) , fg = "red" )
        click . secho ( "1. Consider renaming the item" )
        click . secho ( "2. Consider removing the item" )
        sys . exit ( 3 )
with click . progressbar ( length = len ( list ( proto_dataset . _identifiers ( ) ) ) , label = "Generating manifest" ) as progressbar :
    try :
        proto_dataset . freeze ( progressbar = progressbar )
    except dtoolcore . storagebroker . DiskStorageBrokerValidationWarning as e :
        click . secho ( "" )
        click . secho ( str ( e ) , fg = "red" , nl = False )
        sys . exit ( 4 )
click . secho ( "Dataset frozen " , nl = False , fg = "green" )
click . secho ( proto_dataset_uri )


signature:
def copy ( resume , quiet , dataset_uri , dest_base_uri ) :
    """DEPRECATED : Copy a dataset to a different location ."""

implementation:
click . secho ( "The ``dtool copy`` command is deprecated" , fg = "red" , err = True )
click . secho ( "Use ``dtool cp`` instead" , fg = "red" , err = True )
_copy ( resume , quiet , dataset_uri , dest_base_uri )


signature:
def cp ( resume , quiet , dataset_uri , dest_base_uri ) :
    """Copy a dataset to a different location ."""

implementation: _copy ( resume , quiet , dataset_uri , dest_base_uri )


signature:
def compress ( obj , level = 6 , return_type = "bytes" ) :
    """Compress anything to bytes or string .
 : params obj :
 : params level :
 : params return _ type : if bytes , then return bytes ; if str , then return
 base64 . b64encode bytes in utf - 8 string ."""

implementation:
if isinstance ( obj , binary_type ) :
    b = zlib . compress ( obj , level )
elif isinstance ( obj , string_types ) :
    b = zlib . compress ( obj . encode ( "utf-8" ) , level )
else :
    b = zlib . compress ( pickle . dumps ( obj , protocol = 2 ) , level )
if return_type == "bytes" :
    return b
elif return_type == "str" :
    return base64 . b64encode ( b ) . decode ( "utf-8" )
else :
    raise ValueError ( "'return_type' has to be one of 'bytes', 'str'!" )


signature:
def _parsems ( value ) :
    """Parse a I [ . F ] seconds value into ( seconds , microseconds ) ."""

implementation:
if "." not in value :
    return int ( value ) , 0
else :
    i , f = value . split ( "." )
    return int ( i ) , int ( f . ljust ( 6 , "0" ) [ : 6 ] )


signature:
def get_token ( self ) :
    """This function breaks the time string into lexical units ( tokens ) , which
 can be parsed by the parser . Lexical units are demarcated by changes in
 the character set , so any continuous string of letters is considered
 one unit , any continuous string of numbers is considered one unit .
 The main complication arises from the fact that dots ( ' . ' ) can be used
 both as separators ( e . g . " Sep . 20.2009 " ) or decimal points ( e . g .
 "4:30:21.447 " ) . As such , it is necessary to read the full context of
 any dot - separated strings before breaking it into tokens ; as such , this
 function maintains a " token stack " , for when the ambiguous context
 demands that multiple tokens be parsed at once ."""

implementation:
if self . tokenstack :
    return self . tokenstack . pop ( 0 )
seenletters = False
token = None
state = None
while not self . eof : # We only realize that we ' ve reached the end of a token when we
# find a character that ' s not part of the current token - since
# that character may be part of the next token , it ' s stored in the
# charstack .
    if self . charstack :
        nextchar = self . charstack . pop ( 0 )
    else :
        nextchar = self . instream . read ( 1 )
        while nextchar == '\x00' :
            nextchar = self . instream . read ( 1 )
    if not nextchar :
        self . eof = True
        break
    elif not state : # First character of the token - determines if we ' re starting
    # to parse a word , a number or something else .
        token = nextchar
        if self . isword ( nextchar ) :
            state = 'a'
        elif self . isnum ( nextchar ) :
            state = '0'
        elif self . isspace ( nextchar ) :
            token = ' '
            break
            # emit token
        else :
            break
            # emit token
    elif state == 'a' : # If we ' ve already started reading a word , we keep reading
    # letters until we find something that ' s not part of a word .
        seenletters = True
        if self . isword ( nextchar ) :
            token += nextchar
        elif nextchar == '.' :
            token += nextchar
            state = 'a.'
        else :
            self . charstack . append ( nextchar )
            break
            # emit token
    elif state == '0' : # If we ' ve already started reading a number , we keep reading
    # numbers until we find something that doesn ' t fit .
        if self . isnum ( nextchar ) :
            token += nextchar
        elif nextchar == '.' or ( nextchar == ',' and len ( token ) >= 2 ) :
            token += nextchar
            state = '0.'
        else :
            self . charstack . append ( nextchar )
            break
            # emit token
    elif state == 'a.' : # If we ' ve seen some letters and a dot separator , continue
    # parsing , and the tokens will be broken up later .
        seenletters = True
        if nextchar == '.' or self . isword ( nextchar ) :
            token += nextchar
        elif self . isnum ( nextchar ) and token [ - 1 ] == '.' :
            token += nextchar
            state = '0.'
        else :
            self . charstack . append ( nextchar )
            break
            # emit token
    elif state == '0.' : # If we ' ve seen at least one dot separator , keep going , we ' ll
    # break up the tokens later .
        if nextchar == '.' or self . isnum ( nextchar ) :
            token += nextchar
        elif self . isword ( nextchar ) and token [ - 1 ] == '.' :
            token += nextchar
            state = 'a.'
        else :
            self . charstack . append ( nextchar )
            break
# emit token
if ( state in ( 'a.' , '0.' ) and ( seenletters or token . count ( '.' ) > 1 or token [ - 1 ] in '.,' ) ) :
    l = self . _split_decimal . split ( token )
    token = l [ 0 ]
    for tok in l [ 1 : ] :
        if tok :
            self . tokenstack . append ( tok )
if state == '0.' and token . count ( '.' ) == 0 :
    token = token . replace ( ',' , '.' )
return token


signature:
def find_probable_year_index ( self , tokens ) :
    """attempt to deduce if a pre 100 year was lost
 due to padded zeros being taken off"""

implementation:
for index , token in enumerate ( self ) :
    potential_year_tokens = _ymd . find_potential_year_tokens ( token , tokens )
    if len ( potential_year_tokens ) == 1 and len ( potential_year_tokens [ 0 ] ) > 2 :
        return index


signature:
def parse ( self , timestr , default = None , ignoretz = False , tzinfos = None , ** kwargs ) :
    """Parse the date / time string into a : class : ` datetime . datetime ` object .
 : param timestr :
 Any date / time string using the supported formats .
 : param default :
 The default datetime object , if this is a datetime object and not
 ` ` None ` ` , elements specified in ` ` timestr ` ` replace elements in the
 default object .
 : param ignoretz :
 If set ` ` True ` ` , time zones in parsed strings are ignored and a
 naive : class : ` datetime . datetime ` object is returned .
 : param tzinfos :
 Additional time zone names / aliases which may be present in the
 string . This argument maps time zone names ( and optionally offsets
 from those time zones ) to time zones . This parameter can be a
 dictionary with timezone aliases mapping time zone names to time
 zones or a function taking two parameters ( ` ` tzname ` ` and
 ` ` tzoffset ` ` ) and returning a time zone .
 The timezones to which the names are mapped can be an integer
 offset from UTC in minutes or a : class : ` tzinfo ` object .
 . . doctest : :
 : options : + NORMALIZE _ WHITESPACE
 > > > from dateutil . parser import parse
 > > > from dateutil . tz import gettz
 > > > tzinfos = { " BRST " : - 10800 , " CST " : gettz ( " America / Chicago " ) }
 > > > parse ( " 2012-01-19 17:21:00 BRST " , tzinfos = tzinfos )
 datetime . datetime ( 2012 , 1 , 19 , 17 , 21 , tzinfo = tzoffset ( u ' BRST ' , - 10800 ) )
 > > > parse ( " 2012-01-19 17:21:00 CST " , tzinfos = tzinfos )
 datetime . datetime ( 2012 , 1 , 19 , 17 , 21,
 tzinfo = tzfile ( ' / usr / share / zoneinfo / America / Chicago ' ) )
 This parameter is ignored if ` ` ignoretz ` ` is set .
 : param * * kwargs :
 Keyword arguments as passed to ` ` _ parse ( ) ` ` .
 : return :
 Returns a : class : ` datetime . datetime ` object or , if the
 ` ` fuzzy _ with _ tokens ` ` option is ` ` True ` ` , returns a tuple , the
 first element being a : class : ` datetime . datetime ` object , the second
 a tuple containing the fuzzy tokens .
 : raises ValueError :
 Raised for invalid or unknown string format , if the provided
 : class : ` tzinfo ` is not in a valid format , or if an invalid date
 would be created .
 : raises TypeError :
 Raised for non - string or character stream input .
 : raises OverflowError :
 Raised if the parsed date exceeds the largest valid C integer on
 your system ."""

implementation:
if default is None :
    default = datetime . datetime . now ( ) . replace ( hour = 0 , minute = 0 , second = 0 , microsecond = 0 )
res , skipped_tokens = self . _parse ( timestr , ** kwargs )
if res is None :
    raise ValueError ( "Unknown string format" )
if len ( res ) == 0 :
    raise ValueError ( "String does not contain a date." )
repl = { }
for attr in ( "year" , "month" , "day" , "hour" , "minute" , "second" , "microsecond" ) :
    value = getattr ( res , attr )
    if value is not None :
        repl [ attr ] = value
if 'day' not in repl : # If the default day exceeds the last day of the month , fall back to
# the end of the month .
    cyear = default . year if res . year is None else res . year
    cmonth = default . month if res . month is None else res . month
    cday = default . day if res . day is None else res . day
    if cday > monthrange ( cyear , cmonth ) [ 1 ] :
        repl [ 'day' ] = monthrange ( cyear , cmonth ) [ 1 ]
ret = default . replace ( ** repl )
if res . weekday is not None and not res . day :
    ret = ret + relativedelta . relativedelta ( weekday = res . weekday )
if not ignoretz :
    if ( isinstance ( tzinfos , collections . Callable ) or tzinfos and res . tzname in tzinfos ) :
        if isinstance ( tzinfos , collections . Callable ) :
            tzdata = tzinfos ( res . tzname , res . tzoffset )
        else :
            tzdata = tzinfos . get ( res . tzname )
        if isinstance ( tzdata , datetime . tzinfo ) :
            tzinfo = tzdata
        elif isinstance ( tzdata , text_type ) :
            tzinfo = tz . tzstr ( tzdata )
        elif isinstance ( tzdata , integer_types ) :
            tzinfo = tz . tzoffset ( res . tzname , tzdata )
        else :
            raise ValueError ( "Offset must be tzinfo subclass, " "tz string, or int offset." )
        ret = ret . replace ( tzinfo = tzinfo )
    elif res . tzname and res . tzname in time . tzname :
        ret = ret . replace ( tzinfo = tz . tzlocal ( ) )
    elif res . tzoffset == 0 :
        ret = ret . replace ( tzinfo = tz . tzutc ( ) )
    elif res . tzoffset :
        ret = ret . replace ( tzinfo = tz . tzoffset ( res . tzname , res . tzoffset ) )
if kwargs . get ( 'fuzzy_with_tokens' , False ) :
    return ret , skipped_tokens
else :
    return ret


signature:
def _parse ( self , timestr , dayfirst = None , yearfirst = None , fuzzy = False , fuzzy_with_tokens = False ) :
    """Private method which performs the heavy lifting of parsing , called from
 ` ` parse ( ) ` ` , which passes on its ` ` kwargs ` ` to this function .
 : param timestr :
 The string to parse .
 : param dayfirst :
 Whether to interpret the first value in an ambiguous 3 - integer date
 ( e . g . 01/05/09 ) as the day ( ` ` True ` ` ) or month ( ` ` False ` ` ) . If
 ` ` yearfirst ` ` is set to ` ` True ` ` , this distinguishes between YDM
 and YMD . If set to ` ` None ` ` , this value is retrieved from the
 current : class : ` parserinfo ` object ( which itself defaults to
 ` ` False ` ` ) .
 : param yearfirst :
 Whether to interpret the first value in an ambiguous 3 - integer date
 ( e . g . 01/05/09 ) as the year . If ` ` True ` ` , the first number is taken
 to be the year , otherwise the last number is taken to be the year .
 If this is set to ` ` None ` ` , the value is retrieved from the current
 : class : ` parserinfo ` object ( which itself defaults to ` ` False ` ` ) .
 : param fuzzy :
 Whether to allow fuzzy parsing , allowing for string like " Today is
 January 1 , 2047 at 8:21:00AM " .
 : param fuzzy _ with _ tokens :
 If ` ` True ` ` , ` ` fuzzy ` ` is automatically set to True , and the parser
 will return a tuple where the first element is the parsed
 : class : ` datetime . datetime ` datetimestamp and the second element is
 a tuple containing the portions of the string which were ignored :
 . . doctest : :
 > > > from dateutil . parser import parse
 > > > parse ( " Today is January 1 , 2047 at 8:21:00AM " , fuzzy _ with _ tokens = True )
 ( datetime . datetime ( 2047 , 1 , 1 , 8 , 21 ) , ( u ' Today is ' , u ' ' , u ' at ' ) )"""

implementation:
if fuzzy_with_tokens :
    fuzzy = True
info = self . info
if dayfirst is None :
    dayfirst = info . dayfirst
if yearfirst is None :
    yearfirst = info . yearfirst
res = self . _result ( )
l = _timelex . split ( timestr )
# Splits the timestr into tokens
# keep up with the last token skipped so we can recombine
# consecutively skipped tokens ( - 2 for when i begins at 0 ) .
last_skipped_token_i = - 2
skipped_tokens = list ( )
try : # year / month / day list
    ymd = _ymd ( timestr )
    # Index of the month string in ymd
    mstridx = - 1
    len_l = len ( l )
    i = 0
    while i < len_l : # Check if it ' s a number
        try :
            value_repr = l [ i ]
            value = float ( value_repr )
        except ValueError :
            value = None
        if value is not None : # Token is a number
            len_li = len ( l [ i ] )
            i += 1
            if ( len ( ymd ) == 3 and len_li in ( 2 , 4 ) and res . hour is None and ( i >= len_l or ( l [ i ] != ':' and info . hms ( l [ i ] ) is None ) ) ) : # 19990101T23[59]
                s = l [ i - 1 ]
                res . hour = int ( s [ : 2 ] )
                if len_li == 4 :
                    res . minute = int ( s [ 2 : ] )
            elif len_li == 6 or ( len_li > 6 and l [ i - 1 ] . find ( '.' ) == 6 ) : # YYMMDD or HHMMSS [ . ss ]
                s = l [ i - 1 ]
                if not ymd and l [ i - 1 ] . find ( '.' ) == - 1 : # ymd . append ( info . convertyear ( int ( s [ : 2 ] ) ) )
                    ymd . append ( s [ : 2 ] )
                    ymd . append ( s [ 2 : 4 ] )
                    ymd . append ( s [ 4 : ] )
                else : # 19990101T235959 [ . 59]
                    res . hour = int ( s [ : 2 ] )
                    res . minute = int ( s [ 2 : 4 ] )
                    res . second , res . microsecond = _parsems ( s [ 4 : ] )
            elif len_li in ( 8 , 12 , 14 ) : # YYYYMMDD
                s = l [ i - 1 ]
                ymd . append ( s [ : 4 ] )
                ymd . append ( s [ 4 : 6 ] )
                ymd . append ( s [ 6 : 8 ] )
                if len_li > 8 :
                    res . hour = int ( s [ 8 : 10 ] )
                    res . minute = int ( s [ 10 : 12 ] )
                    if len_li > 12 :
                        res . second = int ( s [ 12 : ] )
            elif ( ( i < len_l and info . hms ( l [ i ] ) is not None ) or ( i + 1 < len_l and l [ i ] == ' ' and info . hms ( l [ i + 1 ] ) is not None ) ) : # HH [ ] h or MM [ ] m or SS [ . ss ] [ ] s
                if l [ i ] == ' ' :
                    i += 1
                idx = info . hms ( l [ i ] )
                while True :
                    if idx == 0 :
                        res . hour = int ( value )
                        if value % 1 :
                            res . minute = int ( 60 * ( value % 1 ) )
                    elif idx == 1 :
                        res . minute = int ( value )
                        if value % 1 :
                            res . second = int ( 60 * ( value % 1 ) )
                    elif idx == 2 :
                        res . second , res . microsecond = _parsems ( value_repr )
                    i += 1
                    if i >= len_l or idx == 2 :
                        break
                    # 12h00
                    try :
                        value_repr = l [ i ]
                        value = float ( value_repr )
                    except ValueError :
                        break
                    else :
                        i += 1
                        idx += 1
                        if i < len_l :
                            newidx = info . hms ( l [ i ] )
                            if newidx is not None :
                                idx = newidx
            elif ( i == len_l and l [ i - 2 ] == ' ' and info . hms ( l [ i - 3 ] ) is not None ) : # X h MM or X m SS
                idx = info . hms ( l [ i - 3 ] )
                if idx == 0 :
                    res . minute = int ( value )
                    sec_remainder = value % 1
                    if sec_remainder :
                        res . second = int ( 60 * sec_remainder )
                elif idx == 1 :
                    res . second , res . microsecond = _parsems ( value_repr )
                # We don ' t need to advance the tokens here because the
                # i = = len _ l call indicates that we ' re looking at all
                # the tokens already .
            elif i + 1 < len_l and l [ i ] == ':' : # HH : MM [ : SS [ . ss ] ]
                res . hour = int ( value )
                i += 1
                value = float ( l [ i ] )
                res . minute = int ( value )
                if value % 1 :
                    res . second = int ( 60 * ( value % 1 ) )
                i += 1
                if i < len_l and l [ i ] == ':' :
                    res . second , res . microsecond = _parsems ( l [ i + 1 ] )
                    i += 2
            elif i < len_l and l [ i ] in ( '-' , '/' , '.' ) :
                sep = l [ i ]
                ymd . append ( value_repr )
                i += 1
                if i < len_l and not info . jump ( l [ i ] ) :
                    try : # 01-01 [ - 01]
                        ymd . append ( l [ i ] )
                    except ValueError : # 01 - Jan [ - 01]
                        value = info . month ( l [ i ] )
                        if value is not None :
                            ymd . append ( value )
                            assert mstridx == - 1
                            mstridx = len ( ymd ) - 1
                        else :
                            return None , None
                    i += 1
                    if i < len_l and l [ i ] == sep : # We have three members
                        i += 1
                        value = info . month ( l [ i ] )
                        if value is not None :
                            ymd . append ( value )
                            mstridx = len ( ymd ) - 1
                            assert mstridx == - 1
                        else :
                            ymd . append ( l [ i ] )
                        i += 1
            elif i >= len_l or info . jump ( l [ i ] ) :
                if i + 1 < len_l and info . ampm ( l [ i + 1 ] ) is not None : # 12 am
                    res . hour = int ( value )
                    if res . hour < 12 and info . ampm ( l [ i + 1 ] ) == 1 :
                        res . hour += 12
                    elif res . hour == 12 and info . ampm ( l [ i + 1 ] ) == 0 :
                        res . hour = 0
                    i += 1
                else : # Year , month or day
                    ymd . append ( value )
                i += 1
            elif info . ampm ( l [ i ] ) is not None : # 12am
                res . hour = int ( value )
                if res . hour < 12 and info . ampm ( l [ i ] ) == 1 :
                    res . hour += 12
                elif res . hour == 12 and info . ampm ( l [ i ] ) == 0 :
                    res . hour = 0
                i += 1
            elif not fuzzy :
                return None , None
            else :
                i += 1
            continue
        # Check weekday
        value = info . weekday ( l [ i ] )
        if value is not None :
            res . weekday = value
            i += 1
            continue
        # Check month name
        value = info . month ( l [ i ] )
        if value is not None :
            ymd . append ( value )
            assert mstridx == - 1
            mstridx = len ( ymd ) - 1
            i += 1
            if i < len_l :
                if l [ i ] in ( '-' , '/' ) : # Jan - 01 [ - 99]
                    sep = l [ i ]
                    i += 1
                    ymd . append ( l [ i ] )
                    i += 1
                    if i < len_l and l [ i ] == sep : # Jan - 01-99
                        i += 1
                        ymd . append ( l [ i ] )
                        i += 1
                elif ( i + 3 < len_l and l [ i ] == l [ i + 2 ] == ' ' and info . pertain ( l [ i + 1 ] ) ) : # Jan of 01
                # In this case , 01 is clearly year
                    try :
                        value = int ( l [ i + 3 ] )
                    except ValueError : # Wrong guess
                        pass
                    else : # Convert it here to become unambiguous
                        ymd . append ( str ( info . convertyear ( value ) ) )
                    i += 4
            continue
        # Check am / pm
        value = info . ampm ( l [ i ] )
        if value is not None : # For fuzzy parsing , ' a ' or ' am ' ( both valid English words )
        # may erroneously trigger the AM / PM flag . Deal with that
        # here .
            val_is_ampm = True
            # If there ' s already an AM / PM flag , this one isn ' t one .
            if fuzzy and res . ampm is not None :
                val_is_ampm = False
            # If AM / PM is found and hour is not , raise a ValueError
            if res . hour is None :
                if fuzzy :
                    val_is_ampm = False
                else :
                    raise ValueError ( 'No hour specified with ' + 'AM or PM flag.' )
            elif not 0 <= res . hour <= 12 : # If AM / PM is found , it ' s a 12 hour clock , so raise
            # an error for invalid range
                if fuzzy :
                    val_is_ampm = False
                else :
                    raise ValueError ( 'Invalid hour specified for ' + '12-hour clock.' )
            if val_is_ampm :
                if value == 1 and res . hour < 12 :
                    res . hour += 12
                elif value == 0 and res . hour == 12 :
                    res . hour = 0
                res . ampm = value
            elif fuzzy :
                last_skipped_token_i = self . _skip_token ( skipped_tokens , last_skipped_token_i , i , l )
            i += 1
            continue
        # Check for a timezone name
        if ( res . hour is not None and len ( l [ i ] ) <= 5 and res . tzname is None and res . tzoffset is None and not [ x for x in l [ i ] if x not in string . ascii_uppercase ] ) :
            res . tzname = l [ i ]
            res . tzoffset = info . tzoffset ( res . tzname )
            i += 1
            # Check for something like GMT + 3 , or BRST + 3 . Notice
            # that it doesn ' t mean " I am 3 hours after GMT " , but
            # " my time + 3 is GMT " . If found , we reverse the
            # logic so that timezone parsing code will get it
            # right .
            if i < len_l and l [ i ] in ( '+' , '-' ) :
                l [ i ] = ( '+' , '-' ) [ l [ i ] == '+' ]
                res . tzoffset = None
                if info . utczone ( res . tzname ) : # With something like GMT + 3 , the timezone
                # is * not * GMT .
                    res . tzname = None
            continue
        # Check for a numbered timezone
        if res . hour is not None and l [ i ] in ( '+' , '-' ) :
            signal = ( - 1 , 1 ) [ l [ i ] == '+' ]
            i += 1
            len_li = len ( l [ i ] )
            if len_li == 4 : # -0300
                res . tzoffset = int ( l [ i ] [ : 2 ] ) * 3600 + int ( l [ i ] [ 2 : ] ) * 60
            elif i + 1 < len_l and l [ i + 1 ] == ':' : # -03:00
                res . tzoffset = int ( l [ i ] ) * 3600 + int ( l [ i + 2 ] ) * 60
                i += 2
            elif len_li <= 2 : # - [ 0]3
                res . tzoffset = int ( l [ i ] [ : 2 ] ) * 3600
            else :
                return None , None
            i += 1
            res . tzoffset *= signal
            # Look for a timezone name between parenthesis
            if ( i + 3 < len_l and info . jump ( l [ i ] ) and l [ i + 1 ] == '(' and l [ i + 3 ] == ')' and 3 <= len ( l [ i + 2 ] ) <= 5 and not [ x for x in l [ i + 2 ] if x not in string . ascii_uppercase ] ) : # -0300 ( BRST )
                res . tzname = l [ i + 2 ]
                i += 4
            continue
        # Check jumps
        if not ( info . jump ( l [ i ] ) or fuzzy ) :
            return None , None
        last_skipped_token_i = self . _skip_token ( skipped_tokens , last_skipped_token_i , i , l )
        i += 1
    # Process year / month / day
    year , month , day = ymd . resolve_ymd ( mstridx , yearfirst , dayfirst )
    if year is not None :
        res . year = year
        res . century_specified = ymd . century_specified
    if month is not None :
        res . month = month
    if day is not None :
        res . day = day
except ( IndexError , ValueError , AssertionError ) :
    return None , None
if not info . validate ( res ) :
    return None , None
if fuzzy_with_tokens :
    return res , tuple ( skipped_tokens )
else :
    return res , None


signature:
def tzname_in_python2 ( namefunc ) :
    """Change unicode output into bytestrings in Python 2
 tzname ( ) API changed in Python 3 . It used to return bytes , but was changed
 to unicode strings"""

implementation:
def adjust_encoding ( * args , ** kwargs ) :
    name = namefunc ( * args , ** kwargs )
    if name is not None and not PY3 :
        name = name . encode ( )
    return name
return adjust_encoding


signature:
def _validate_fromutc_inputs ( f ) :
    """The CPython version of ` ` fromutc ` ` checks that the input is a ` ` datetime ` `
 object and that ` ` self ` ` is attached as its ` ` tzinfo ` ` ."""

implementation:
@ wraps ( f )
def fromutc ( self , dt ) :
    if not isinstance ( dt , datetime ) :
        raise TypeError ( "fromutc() requires a datetime argument" )
    if dt . tzinfo is not self :
        raise ValueError ( "dt.tzinfo is not self" )
    return f ( self , dt )
return fromutc


signature:
def is_ambiguous ( self , dt ) :
    """Whether or not the " wall time " of a given datetime is ambiguous in this
 zone .
 : param dt :
 A : py : class : ` datetime . datetime ` , naive or time zone aware .
 : return :
 Returns ` ` True ` ` if ambiguous , ` ` False ` ` otherwise .
 . . versionadded : : 2.6.0"""

implementation:
dt = dt . replace ( tzinfo = self )
wall_0 = enfold ( dt , fold = 0 )
wall_1 = enfold ( dt , fold = 1 )
same_offset = wall_0 . utcoffset ( ) == wall_1 . utcoffset ( )
same_dt = wall_0 . replace ( tzinfo = None ) == wall_1 . replace ( tzinfo = None )
return same_dt and not same_offset


signature:
def _fold_status ( self , dt_utc , dt_wall ) :
    """Determine the fold status of a " wall " datetime , given a representation
 of the same datetime as a ( naive ) UTC datetime . This is calculated based
 on the assumption that ` ` dt . utcoffset ( ) - dt . dst ( ) ` ` is constant for all
 datetimes , and that this offset is the actual number of hours separating
 ` ` dt _ utc ` ` and ` ` dt _ wall ` ` .
 : param dt _ utc :
 Representation of the datetime as UTC
 : param dt _ wall :
 Representation of the datetime as " wall time " . This parameter must
 either have a ` fold ` attribute or have a fold - naive
 : class : ` datetime . tzinfo ` attached , otherwise the calculation may
 fail ."""

implementation:
if self . is_ambiguous ( dt_wall ) :
    delta_wall = dt_wall - dt_utc
    _fold = int ( delta_wall == ( dt_utc . utcoffset ( ) - dt_utc . dst ( ) ) )
else :
    _fold = 0
return _fold


signature:
def _fromutc ( self , dt ) :
    """Given a timezone - aware datetime in a given timezone , calculates a
 timezone - aware datetime in a new timezone .
 Since this is the one time that we * know * we have an unambiguous
 datetime object , we take this opportunity to determine whether the
 datetime is ambiguous and in a " fold " state ( e . g . if it ' s the first
 occurence , chronologically , of the ambiguous datetime ) .
 : param dt :
 A timezone - aware : class : ` datetime . datetime ` object ."""

implementation:
# Re - implement the algorithm from Python ' s datetime . py
dtoff = dt . utcoffset ( )
if dtoff is None :
    raise ValueError ( "fromutc() requires a non-None utcoffset() " "result" )
# The original datetime . py code assumes that ` dst ( ) ` defaults to
# zero during ambiguous times . PEP 495 inverts this presumption , so
# for pre - PEP 495 versions of python , we need to tweak the algorithm .
dtdst = dt . dst ( )
if dtdst is None :
    raise ValueError ( "fromutc() requires a non-None dst() result" )
delta = dtoff - dtdst
dt += delta
# Set fold = 1 so we can default to being in the fold for
# ambiguous dates .
dtdst = enfold ( dt , fold = 1 ) . dst ( )
if dtdst is None :
    raise ValueError ( "fromutc(): dt.dst gave inconsistent " "results; cannot convert" )
return dt + dtdst


signature:
def fromutc ( self , dt ) :
    """Given a timezone - aware datetime in a given timezone , calculates a
 timezone - aware datetime in a new timezone .
 Since this is the one time that we * know * we have an unambiguous
 datetime object , we take this opportunity to determine whether the
 datetime is ambiguous and in a " fold " state ( e . g . if it ' s the first
 occurance , chronologically , of the ambiguous datetime ) .
 : param dt :
 A timezone - aware : class : ` datetime . datetime ` object ."""

implementation:
dt_wall = self . _fromutc ( dt )
# Calculate the fold status given the two datetimes .
_fold = self . _fold_status ( dt , dt_wall )
# Set the default fold value for ambiguous dates
return enfold ( dt_wall , fold = _fold )


signature:
def fromutc ( self , dt ) :
    """Given a datetime in UTC , return local time"""

implementation:
if not isinstance ( dt , datetime ) :
    raise TypeError ( "fromutc() requires a datetime argument" )
if dt . tzinfo is not self :
    raise ValueError ( "dt.tzinfo is not self" )
# Get transitions - if there are none , fixed offset
transitions = self . transitions ( dt . year )
if transitions is None :
    return dt + self . utcoffset ( dt )
# Get the transition times in UTC
dston , dstoff = transitions
dston -= self . _std_offset
dstoff -= self . _std_offset
utc_transitions = ( dston , dstoff )
dt_utc = dt . replace ( tzinfo = None )
isdst = self . _naive_isdst ( dt_utc , utc_transitions )
if isdst :
    dt_wall = dt + self . _dst_offset
else :
    dt_wall = dt + self . _std_offset
_fold = int ( not isdst and self . is_ambiguous ( dt_wall ) )
return enfold ( dt_wall , fold = _fold )


implementation:
if not self . hasdst :
    return False
start , end = self . transitions ( dt . year )
dt = dt . replace ( tzinfo = None )
return ( end <= dt < end + self . _dst_base_offset )


signature:
def strip_comment_line_with_symbol ( line , start ) :
    """Strip comments from line string ."""

implementation:
parts = line . split ( start )
counts = [ len ( findall ( r'(?:^|[^"\\]|(?:\\\\|\\")+)(")' , part ) ) for part in parts ]
total = 0
for nr , count in enumerate ( counts ) :
    total += count
    if total % 2 == 0 :
        return start . join ( parts [ : nr + 1 ] ) . rstrip ( )
else : # pragma : no cover
    return line . rstrip ( )


signature:
def strip_comments ( string , comment_symbols = frozenset ( ( '#' , '//' ) ) ) :
    """Strip comments from json string .
 : param string : A string containing json with comments started by comment _ symbols .
 : param comment _ symbols : Iterable of symbols that start a line comment ( default # or / / ) .
 : return : The string with the comments removed ."""

implementation:
lines = string . splitlines ( )
for k in range ( len ( lines ) ) :
    for symbol in comment_symbols :
        lines [ k ] = strip_comment_line_with_symbol ( lines [ k ] , start = symbol )
return '\n' . join ( lines )


signature:
def rebuild ( filename , tag = None , format = "gz" , zonegroups = [ ] , metadata = None ) :
    """Rebuild the internal timezone info in dateutil / zoneinfo / zoneinfo * tar *
 filename is the timezone tarball from ftp . iana . org / tz ."""

implementation:
tmpdir = tempfile . mkdtemp ( )
zonedir = os . path . join ( tmpdir , "zoneinfo" )
moduledir = os . path . dirname ( __file__ )
try :
    with tar_open ( filename ) as tf :
        for name in zonegroups :
            tf . extract ( name , tmpdir )
        filepaths = [ os . path . join ( tmpdir , n ) for n in zonegroups ]
        try :
            check_call ( [ "zic" , "-d" , zonedir ] + filepaths )
        except OSError as e :
            _print_on_nosuchfile ( e )
            raise
    # write metadata file
    with open ( os . path . join ( zonedir , METADATA_FN ) , 'w' ) as f :
        json . dump ( metadata , f , indent = 4 , sort_keys = True )
    target = os . path . join ( moduledir , ZONEFILENAME )
    with tar_open ( target , "w:%s" % format ) as tf :
        for entry in os . listdir ( zonedir ) :
            entrypath = os . path . join ( zonedir , entry )
            tf . add ( entrypath , entry )
finally :
    shutil . rmtree ( tmpdir )


signature:
def picknthweekday ( year , month , dayofweek , hour , minute , whichweek ) :
    """dayofweek = = 0 means Sunday , whichweek 5 means last instance"""

implementation:
first = datetime . datetime ( year , month , 1 , hour , minute )
# This will work if dayofweek is ISO weekday ( 1-7 ) or Microsoft - style ( 0-6 ) ,
# Because 7 % 7 = 0
weekdayone = first . replace ( day = ( ( dayofweek - first . isoweekday ( ) ) % 7 ) + 1 )
wd = weekdayone + ( ( whichweek - 1 ) * ONEWEEK )
if ( wd . month != month ) :
    wd -= ONEWEEK
return wd


signature:
def valuestodict ( key ) :
    """Convert a registry key ' s values to a dictionary ."""

implementation:
dout = { }
size = winreg . QueryInfoKey ( key ) [ 1 ]
tz_res = None
for i in range ( size ) :
    key_name , value , dtype = winreg . EnumValue ( key , i )
    if dtype == winreg . REG_DWORD or dtype == winreg . REG_DWORD_LITTLE_ENDIAN : # If it ' s a DWORD ( 32 - bit integer ) , it ' s stored as unsigned - convert
    # that to a proper signed integer
        if value & ( 1 << 31 ) :
            value = value - ( 1 << 32 )
    elif dtype == winreg . REG_SZ : # If it ' s a reference to the tzres DLL , load the actual string
        if value . startswith ( '@tzres' ) :
            tz_res = tz_res or tzres ( )
            value = tz_res . name_from_string ( value )
        value = value . rstrip ( '\x00' )
    # Remove trailing nulls
    dout [ key_name ] = value
return dout


signature:
def load_name ( self , offset ) :
    """Load a timezone name from a DLL offset ( integer ) .
 > > > from dateutil . tzwin import tzres
 > > > tzr = tzres ( )
 > > > print ( tzr . load _ name ( 112 ) )
 ' Eastern Standard Time '
 : param offset :
 A positive integer value referring to a string from the tzres dll .
 . . note :
 Offsets found in the registry are generally of the form
 ` @ tzres . dll , - 114 ` . The offset in this case if 114 , not - 114."""

implementation:
resource = self . p_wchar ( )
lpBuffer = ctypes . cast ( ctypes . byref ( resource ) , wintypes . LPWSTR )
nchar = self . LoadStringW ( self . _tzres . _handle , offset , lpBuffer , 0 )
return resource [ : nchar ]


signature:
def name_from_string ( self , tzname_str ) :
    """Parse strings as returned from the Windows registry into the time zone
 name as defined in the registry .
 > > > from dateutil . tzwin import tzres
 > > > tzr = tzres ( )
 > > > print ( tzr . name _ from _ string ( ' @ tzres . dll , - 251 ' ) )
 ' Dateline Daylight Time '
 > > > print ( tzr . name _ from _ string ( ' Eastern Standard Time ' ) )
 ' Eastern Standard Time '
 : param tzname _ str :
 A timezone name string as returned from a Windows registry key .
 : return :
 Returns the localized timezone string from tzres . dll if the string
 is of the form ` @ tzres . dll , - offset ` , else returns the input string ."""

implementation:
if not tzname_str . startswith ( '@' ) :
    return tzname_str
name_splt = tzname_str . split ( ',-' )
try :
    offset = int ( name_splt [ 1 ] )
except :
    raise ValueError ( "Malformed timezone string." )
return self . load_name ( offset )