signature:
def parse_description ( ) :
    """Parse the description in the README file
 CommandLine :
 pandoc - - from = markdown - - to = rst - - output = README . rst README . md
 python - c " import setup ; print ( setup . parse _ description ( ) ) " """

implementation:
from os . path import dirname , join , exists
readme_fpath = join ( dirname ( __file__ ) , 'README.rst' )
# This breaks on pip install , so check that it exists .
if exists ( readme_fpath ) :
    with open ( readme_fpath , 'r' ) as f :
        text = f . read ( )
    return text
return ''


signature:
def parse_requirements_alt ( fname = 'requirements.txt' ) :
    """pip install requirements - parser
 fname = ' requirements . txt '"""

implementation:
import requirements
from os . path import dirname , join , exists
require_fpath = join ( dirname ( __file__ ) , fname )
if exists ( require_fpath ) : # Dont use until this handles platform specific dependencies
    with open ( require_fpath , 'r' ) as file :
        requires = list ( requirements . parse ( file ) )
    packages = [ r . name for r in requires ]
    return packages
return [ ]


signature:
def inject_method ( self , func , name = None ) :
    """Injects a function into an object instance as a bound method
 The main use case of this function is for monkey patching . While monkey
 patching is sometimes necessary it should generally be avoided . Thus , we
 simply remind the developer that there might be a better way .
 Args :
 self ( object ) : instance to inject a function into
 func ( func ) : the function to inject ( must contain an arg for self )
 name ( str ) : name of the method . optional . If not specified the name
 of the function is used .
 Example :
 > > > class Foo ( object ) :
 > > > def bar ( self ) :
 > > > return ' bar '
 > > > def baz ( self ) :
 > > > return ' baz '
 > > > self = Foo ( )
 > > > assert self . bar ( ) = = ' bar '
 > > > assert not hasattr ( self , ' baz ' )
 > > > inject _ method ( self , baz )
 > > > assert not hasattr ( Foo , ' baz ' ) , ' should only change one instance '
 > > > assert self . baz ( ) = = ' baz '
 > > > inject _ method ( self , baz , ' bar ' )
 > > > assert self . bar ( ) = = ' baz '"""

implementation:
# TODO : if func is a bound method we should probably unbind it
new_method = func . __get__ ( self , self . __class__ )
if name is None :
    name = func . __name__
setattr ( self , name , new_method )


signature:
def writeto ( fpath , to_write , aslines = False , verbose = None ) :
    r"""Writes ( utf8 ) text to a file .
 Args :
 fpath ( PathLike ) : file path
 to _ write ( str ) : text to write ( must be unicode text )
 aslines ( bool ) : if True to _ write is assumed to be a list of lines
 verbose ( bool ) : verbosity flag
 CommandLine :
 python - m ubelt . util _ io writeto - - verbose
 Example :
 > > > import ubelt as ub
 > > > dpath = ub . ensure _ app _ cache _ dir ( ' ubelt ' )
 > > > fpath = dpath + ' / ' + ' testwrite . txt '
 > > > if exists ( fpath ) :
 > > > os . remove ( fpath )
 > > > to _ write = ' utf - 8 symbols Δ , Й , ק , م , ๗ , あ , 叶 , 葉 , and 말 . '
 > > > writeto ( fpath , to _ write )
 > > > read _ = ub . readfrom ( fpath )
 > > > print ( ' read _ = ' + read _ )
 > > > print ( ' to _ write = ' + to _ write )
 > > > assert read _ = = to _ write
 Example :
 > > > import ubelt as ub
 > > > dpath = ub . ensure _ app _ cache _ dir ( ' ubelt ' )
 > > > fpath = dpath + ' / ' + ' testwrite2 . txt '
 > > > if exists ( fpath ) :
 > > > os . remove ( fpath )
 > > > to _ write = [ ' a \ n ' , ' b \ n ' , ' c \ n ' , ' d \ n ' ]
 > > > writeto ( fpath , to _ write , aslines = True )
 > > > read _ = ub . readfrom ( fpath , aslines = True )
 > > > print ( ' read _ = { } ' . format ( read _ ) )
 > > > print ( ' to _ write = { } ' . format ( to _ write ) )
 > > > assert read _ = = to _ write"""

implementation:
if verbose :
    print ( 'Writing to text file: %r ' % ( fpath , ) )
with open ( fpath , 'wb' ) as file :
    if aslines :
        to_write = map ( _ensure_bytes , to_write )
        file . writelines ( to_write )
    else : # convert to bytes for writing
        bytes = _ensure_bytes ( to_write )
        file . write ( bytes )


signature:
def readfrom ( fpath , aslines = False , errors = 'replace' , verbose = None ) :
    """Reads ( utf8 ) text from a file .
 Args :
 fpath ( PathLike ) : file path
 aslines ( bool ) : if True returns list of lines
 verbose ( bool ) : verbosity flag
 Returns :
 str : text from fpath ( this is unicode )"""

implementation:
if verbose :
    print ( 'Reading text file: %r ' % ( fpath , ) )
if not exists ( fpath ) :
    raise IOError ( 'File %r does not exist' % ( fpath , ) )
with open ( fpath , 'rb' ) as file :
    if aslines :
        text = [ line . decode ( 'utf8' , errors = errors ) for line in file . readlines ( ) ]
        if sys . platform . startswith ( 'win32' ) : # nocover
        # fix line endings on windows
            text = [ line [ : - 2 ] + '\n' if line . endswith ( '\r\n' ) else line for line in text ]
    else :
        text = file . read ( ) . decode ( 'utf8' , errors = errors )
return text


signature:
def touch ( fpath , mode = 0o666 , dir_fd = None , verbose = 0 , ** kwargs ) :
    """change file timestamps
 Works like the touch unix utility
 Args :
 fpath ( PathLike ) : name of the file
 mode ( int ) : file permissions ( python3 and unix only )
 dir _ fd ( file ) : optional directory file descriptor . If specified , fpath
 is interpreted as relative to this descriptor ( python 3 only ) .
 verbose ( int ) : verbosity
 * * kwargs : extra args passed to ` os . utime ` ( python 3 only ) .
 Returns :
 PathLike : path to the file
 References :
 https : / / stackoverflow . com / questions / 1158076 / implement - touch - using - python
 Example :
 > > > import ubelt as ub
 > > > dpath = ub . ensure _ app _ cache _ dir ( ' ubelt ' )
 > > > fpath = join ( dpath , ' touch _ file ' )
 > > > assert not exists ( fpath )
 > > > ub . touch ( fpath )
 > > > assert exists ( fpath )
 > > > os . unlink ( fpath )"""

implementation:
if verbose :
    print ( 'Touching file {}' . format ( fpath ) )
if six . PY2 : # nocover
    with open ( fpath , 'a' ) :
        os . utime ( fpath , None )
else :
    flags = os . O_CREAT | os . O_APPEND
    with os . fdopen ( os . open ( fpath , flags = flags , mode = mode , dir_fd = dir_fd ) ) as f :
        os . utime ( f . fileno ( ) if os . utime in os . supports_fd else fpath , dir_fd = None if os . supports_fd else dir_fd , ** kwargs )
return fpath


signature:
def delete ( path , verbose = False ) :
    """Removes a file or recursively removes a directory .
 If a path does not exist , then this is does nothing .
 Args :
 path ( PathLike ) : file or directory to remove
 verbose ( bool ) : if True prints what is being done
 SeeAlso :
 send2trash - A cross - platform Python package for sending files
 to the trash instead of irreversibly deleting them .
 https : / / github . com / hsoft / send2trash
 Doctest :
 > > > import ubelt as ub
 > > > base = ub . ensure _ app _ cache _ dir ( ' ubelt ' , ' delete _ test ' )
 > > > dpath1 = ub . ensuredir ( join ( base , ' dir ' ) )
 > > > ub . ensuredir ( join ( base , ' dir ' , ' subdir ' ) )
 > > > ub . touch ( join ( base , ' dir ' , ' to _ remove1 . txt ' ) )
 > > > fpath1 = join ( base , ' dir ' , ' subdir ' , ' to _ remove3 . txt ' )
 > > > fpath2 = join ( base , ' dir ' , ' subdir ' , ' to _ remove2 . txt ' )
 > > > ub . touch ( fpath1)
 > > > ub . touch ( fpath2)
 > > > assert all ( map ( exists , ( dpath1 , fpath1 , fpath2 ) ) )
 > > > ub . delete ( fpath1)
 > > > assert all ( map ( exists , ( dpath1 , fpath2 ) ) )
 > > > assert not exists ( fpath1)
 > > > ub . delete ( dpath1)
 > > > assert not any ( map ( exists , ( dpath1 , fpath1 , fpath2 ) ) )
 Doctest :
 > > > import ubelt as ub
 > > > dpath = ub . ensure _ app _ cache _ dir ( ' ubelt ' , ' delete _ test2 ' )
 > > > dpath1 = ub . ensuredir ( join ( dpath , ' dir ' ) )
 > > > fpath1 = ub . touch ( join ( dpath1 , ' to _ remove . txt ' ) )
 > > > assert exists ( fpath1)
 > > > ub . delete ( dpath )
 > > > assert not exists ( fpath1)"""

implementation:
if not os . path . exists ( path ) : # if the file does exists and is not a broken link
    if os . path . islink ( path ) :
        if verbose : # nocover
            print ( 'Deleting broken link="{}"' . format ( path ) )
        os . unlink ( path )
    elif os . path . isdir ( path ) : # nocover
    # Only on windows will a file be a directory and not exist
        if verbose :
            print ( 'Deleting broken directory link="{}"' . format ( path ) )
        os . rmdir ( path )
    elif os . path . isfile ( path ) : # nocover
    # This is a windows only case
        if verbose :
            print ( 'Deleting broken file link="{}"' . format ( path ) )
        os . unlink ( path )
    else :
        if verbose : # nocover
            print ( 'Not deleting non-existant path="{}"' . format ( path ) )
else :
    if os . path . islink ( path ) :
        if verbose : # nocover
            print ( 'Deleting symbolic link="{}"' . format ( path ) )
        os . unlink ( path )
    elif os . path . isfile ( path ) :
        if verbose : # nocover
            print ( 'Deleting file="{}"' . format ( path ) )
        os . unlink ( path )
    elif os . path . isdir ( path ) :
        if verbose : # nocover
            print ( 'Deleting directory="{}"' . format ( path ) )
        if sys . platform . startswith ( 'win32' ) : # nocover
        # Workaround bug that prevents shutil from working if
        # the directory contains junctions
            from ubelt import _win32_links
            _win32_links . _win32_rmtree ( path , verbose = verbose )
        else :
            import shutil
            shutil . rmtree ( path )


signature:
def repr2 ( data , ** kwargs ) :
    """Makes a pretty and easy - to - doctest string representation !
 This is an alternative to repr , and ` pprint . pformat ` that attempts to be
 both more configurable and generate output that is consistent between
 python versions .
 Notes :
 This function has many keyword arguments that can be used to customize
 the final representation . For convinience some of the more frequently
 used kwargs have short aliases . See ` Args ` for more details .
 Args :
 data ( object ) : an arbitrary python object
 * * kwargs : see ` the Kwargs ` section
 Kwargs :
 si , stritems , ( bool ) :
 dict / list items use str instead of repr
 strkeys , sk ( bool ) :
 dict keys use str instead of repr
 strvals , sv ( bool ) :
 dict values use str instead of repr
 nl , newlines ( int | bool ) :
 number of top level nestings to place a newline after . If true all
 items are followed by newlines regardless of nesting level .
 Defaults to 1 for lists and True for dicts .
 nobr , nobraces ( bool , default = False ) :
 if True , text will not contain outer braces for containers
 cbr , compact _ brace ( bool , default = False ) :
 if True , braces are compactified ( i . e . they will not have newlines
 placed directly after them , think java / K & R / 1TBS )
 trailsep , trailing _ sep ( bool ) :
 if True , a separator is placed after the last item in a sequence .
 By default this is True if there are any ` nl > 0 ` .
 explicit ( bool , default = False ) :
 changes dict representation from ` { k1 : v1 , . . . } ` to
 ` dict ( k1 = v1 , . . . ) ` .
 precision ( int , default = None ) :
 if specified floats are formatted with this precision
 kvsep ( str , default = ' : ' ) :
 separator between keys and values
 itemsep ( str , default = ' ' ) :
 separator between items
 sort ( bool ) :
 if True , attempts to sort all unordered collections in the returned
 text . NOTE : currently if True this will sort lists , this may not be
 a correct thing to do , as such the behavior of this arg is subject
 to change .
 suppress _ small ( bool ) :
 passed to ` numpy . array2string ` for ndarrays
 max _ line _ width ( int ) :
 passed to ` numpy . array2string ` for ndarrays
 with _ dtype ( bool ) :
 only relevant to ndarrays . if True includes the dtype .
 Returns :
 str : outstr : output string
 Notes :
 There are also internal kwargs , which should not be used :
 _ return _ info ( bool ) : return information about child context
 _ root _ info ( depth ) : information about parent context
 CommandLine :
 python - m ubelt . util _ format repr2:0
 python - m ubelt . util _ format repr2:1
 Example :
 > > > from ubelt . util _ format import *
 > > > import ubelt as ub
 > > > dict _ = {
 . . . ' custom _ types ' : [ slice ( 0 , 1 , None ) , 1/3 ] ,
 . . . ' nest _ dict ' : { ' k1 ' : [ 1 , 2 , { 3 : { 4 , 5 } } ] ,
 . . . ' key2 ' : [ 1 , 2 , { 3 : { 4 , 5 } } ] ,
 . . . ' key3 ' : [ 1 , 2 , { 3 : { 4 , 5 } } ] ,
 . . . ' nest _ dict2 ' : { ' k ' : [ 1 , 2 , { 3 : { 4 , 5 } } ] } ,
 . . . ' nested _ tuples ' : [ tuple ( [ 1 ] ) , tuple ( [ 2 , 3 ] ) , frozenset ( [ 4 , 5 , 6 ] ) ] ,
 . . . ' one _ tup ' : tuple ( [ 1 ] ) ,
 . . . ' simple _ dict ' : { ' spam ' : ' eggs ' , ' ham ' : ' jam ' } ,
 . . . ' simple _ list ' : [ 1 , 2 , ' red ' , ' blue ' ] ,
 . . . ' odict ' : ub . odict ( [ ( 1 , ' 1 ' ) , ( 2 , ' 2 ' ) ] ) ,
 > > > result = repr2 ( dict _ , nl = 3 , precision = 2 ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 2 , precision = 2 ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 1 , precision = 2 ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 1 , precision = 2 , itemsep = ' ' , explicit = True ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 1 , precision = 2 , nobr = 1 , itemsep = ' ' , explicit = True ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 3 , precision = 2 , cbr = True ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 3 , precision = 2 , si = True ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 3 , sort = True ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 3 , sort = False , trailing _ sep = False ) ; print ( result )
 > > > result = repr2 ( dict _ , nl = 3 , sort = False , trailing _ sep = False , nobr = True ) ; print ( result )
 Example :
 > > > from ubelt . util _ format import *
 > > > def _ nest ( d , w ) :
 . . . if d = = 0:
 . . . return { }
 . . . else :
 . . . return { ' n { } ' . format ( d ) : _ nest ( d - 1 , w + 1 ) , ' m { } ' . format ( d ) : _ nest ( d - 1 , w + 1 ) }
 > > > dict _ = _ nest ( d = 4 , w = 1)
 > > > result = repr2 ( dict _ , nl = 6 , precision = 2 , cbr = 1)
 > > > print ( ' - - - ' )
 > > > print ( result )
 > > > result = repr2 ( dict _ , nl = - 1 , precision = 2)
 > > > print ( ' - - - ' )
 > > > print ( result )"""

implementation:
custom_extensions = kwargs . get ( 'extensions' , None )
_return_info = kwargs . get ( '_return_info' , False )
kwargs [ '_root_info' ] = _rectify_root_info ( kwargs . get ( '_root_info' , None ) )
outstr = None
_leaf_info = None
if custom_extensions :
    func = custom_extensions . lookup ( data )
    if func is not None :
        outstr = func ( data , ** kwargs )
if outstr is None :
    if isinstance ( data , dict ) :
        outstr , _leaf_info = _format_dict ( data , ** kwargs )
    elif isinstance ( data , ( list , tuple , set , frozenset ) ) :
        outstr , _leaf_info = _format_list ( data , ** kwargs )
if outstr is None : # check any globally registered functions for special formatters
    func = _FORMATTER_EXTENSIONS . lookup ( data )
    if func is not None :
        outstr = func ( data , ** kwargs )
    else :
        outstr = _format_object ( data , ** kwargs )
if _return_info :
    _leaf_info = _rectify_leaf_info ( _leaf_info )
    return outstr , _leaf_info
else :
    return outstr


signature:
def _format_list ( list_ , ** kwargs ) :
    """Makes a pretty printable / human - readable string representation of a
 sequence . In most cases this string could be evaled .
 Args :
 list _ ( list ) : input list
 * * kwargs : nl , newlines , packed , nobr , nobraces , itemsep , trailing _ sep ,
 strvals indent _ , precision , use _ numpy , with _ dtype , force _ dtype ,
 stritems , strkeys , explicit , sort , key _ order , maxlen
 Returns :
 Tuple [ str , Dict ] : retstr , _ leaf _ info
 Example :
 > > > print ( _ format _ list ( [ ] ) [ 0 ] )
 > > > print ( _ format _ list ( [ ] , nobr = True ) [ 0 ] )
 > > > print ( _ format _ list ( [ 1 ] , nl = 0 ) [ 0 ] )
 > > > print ( _ format _ list ( [ 1 ] , nobr = True ) [ 0 ] )"""

implementation:
kwargs [ '_root_info' ] = _rectify_root_info ( kwargs . get ( '_root_info' , None ) )
kwargs [ '_root_info' ] [ 'depth' ] += 1
newlines = kwargs . pop ( 'nl' , kwargs . pop ( 'newlines' , 1 ) )
kwargs [ 'nl' ] = _rectify_countdown_or_bool ( newlines )
nobraces = kwargs . pop ( 'nobr' , kwargs . pop ( 'nobraces' , False ) )
itemsep = kwargs . get ( 'itemsep' , ' ' )
compact_brace = kwargs . get ( 'cbr' , kwargs . get ( 'compact_brace' , False ) )
# kwargs [ ' cbr ' ] = _ rectify _ countdown _ or _ bool ( compact _ brace )
itemstrs , _leaf_info = _list_itemstrs ( list_ , ** kwargs )
if len ( itemstrs ) == 0 :
    nobraces = False
# force braces to prevent empty output
is_tuple = isinstance ( list_ , tuple )
is_set = isinstance ( list_ , ( set , frozenset , ) )
if nobraces :
    lbr , rbr = '' , ''
elif is_tuple :
    lbr , rbr = '(' , ')'
elif is_set :
    lbr , rbr = '{' , '}'
else :
    lbr , rbr = '[' , ']'
# Doesn ' t actually put in trailing comma if on same line
trailing_sep = kwargs . get ( 'trailsep' , kwargs . get ( 'trailing_sep' , newlines > 0 and len ( itemstrs ) ) )
# The trailing separator is always needed for single item tuples
if is_tuple and len ( list_ ) <= 1 :
    trailing_sep = True
if len ( itemstrs ) == 0 :
    newlines = False
retstr = _join_itemstrs ( itemstrs , itemsep , newlines , _leaf_info , nobraces , trailing_sep , compact_brace , lbr , rbr )
return retstr , _leaf_info


signature:
def _format_dict ( dict_ , ** kwargs ) :
    """Makes a pretty printable / human - readable string representation of a
 dictionary . In most cases this string could be evaled .
 Args :
 dict _ ( dict ) : a dictionary
 * * kwargs : si , stritems , strkeys , strvals , sk , sv , nl , newlines , nobr ,
 nobraces , cbr , compact _ brace , trailing _ sep ,
 explicit , itemsep , precision , kvsep , sort
 Returns :
 Tuple [ str , Dict ] : retstr , _ leaf _ info
 Kwargs :
 sort ( None ) : if True , sorts ALL collections and subcollections ,
 note , collections with undefined orders ( e . g . dicts , sets ) are
 sorted by default . ( default = None )
 nl ( int ) : preferred alias for newline . can be a countdown variable
 ( default = None )
 explicit ( int ) : can be a countdown variable . if True , uses
 dict ( a = b ) syntax instead of { ' a ' : b }
 nobr ( bool ) : removes outer braces ( default = False )"""

implementation:
kwargs [ '_root_info' ] = _rectify_root_info ( kwargs . get ( '_root_info' , None ) )
kwargs [ '_root_info' ] [ 'depth' ] += 1
stritems = kwargs . pop ( 'si' , kwargs . pop ( 'stritems' , False ) )
if stritems :
    kwargs [ 'strkeys' ] = True
    kwargs [ 'strvals' ] = True
kwargs [ 'strkeys' ] = kwargs . pop ( 'sk' , kwargs . pop ( 'strkeys' , False ) )
kwargs [ 'strvals' ] = kwargs . pop ( 'sv' , kwargs . pop ( 'strvals' , False ) )
newlines = kwargs . pop ( 'nl' , kwargs . pop ( 'newlines' , True ) )
kwargs [ 'nl' ] = _rectify_countdown_or_bool ( newlines )
nobraces = kwargs . pop ( 'nobr' , kwargs . pop ( 'nobraces' , False ) )
compact_brace = kwargs . get ( 'cbr' , kwargs . get ( 'compact_brace' , False ) )
# kwargs [ ' cbr ' ] = _ rectify _ countdown _ or _ bool ( compact _ brace )
# Doesn ' t actually put in trailing comma if on same line
trailing_sep = kwargs . get ( 'trailsep' , kwargs . get ( 'trailing_sep' , newlines > 0 ) )
explicit = kwargs . get ( 'explicit' , False )
itemsep = kwargs . get ( 'itemsep' , ' ' )
if len ( dict_ ) == 0 :
    retstr = 'dict()' if explicit else '{}'
    _leaf_info = None
else :
    itemstrs , _leaf_info = _dict_itemstrs ( dict_ , ** kwargs )
    if nobraces :
        lbr , rbr = '' , ''
    elif explicit :
        lbr , rbr = 'dict(' , ')'
    else :
        lbr , rbr = '{' , '}'
    retstr = _join_itemstrs ( itemstrs , itemsep , newlines , _leaf_info , nobraces , trailing_sep , compact_brace , lbr , rbr )
return retstr , _leaf_info


signature:
def _join_itemstrs ( itemstrs , itemsep , newlines , _leaf_info , nobraces , trailing_sep , compact_brace , lbr , rbr ) :
    """Joins string - ified items with separators newlines and container - braces ."""

implementation:
# positive newlines means start counting from the root
use_newline = newlines > 0
# negative countdown values mean start counting from the leafs
# if compact _ brace < 0:
# compact _ brace = ( - compact _ brace ) > = _ leaf _ info [ ' max _ height ' ]
if newlines < 0 :
    use_newline = ( - newlines ) < _leaf_info [ 'max_height' ]
if use_newline :
    sep = ',\n'
    if nobraces :
        body_str = sep . join ( itemstrs )
        if trailing_sep and len ( itemstrs ) > 0 :
            body_str += ','
        retstr = body_str
    else :
        if compact_brace : # Why must we modify the indentation below and not here ?
        # prefix = ' '
        # rest = [ ub . indent ( s , prefix ) for s in itemstrs [ 1 : ] ]
        # indented = itemstrs [ 0:1 ] + rest
            indented = itemstrs
        else :
            import ubelt as ub
            prefix = ' ' * 4
            indented = [ ub . indent ( s , prefix ) for s in itemstrs ]
        body_str = sep . join ( indented )
        if trailing_sep and len ( itemstrs ) > 0 :
            body_str += ','
        if compact_brace : # Why can we modify the indentation here but not above ?
            braced_body_str = ( lbr + body_str . replace ( '\n' , '\n ' ) + rbr )
        else :
            braced_body_str = ( lbr + '\n' + body_str + '\n' + rbr )
        retstr = braced_body_str
else :
    sep = ',' + itemsep
    body_str = sep . join ( itemstrs )
    if trailing_sep and len ( itemstrs ) > 0 :
        body_str += ','
    retstr = ( lbr + body_str + rbr )
return retstr


signature:
def _dict_itemstrs ( dict_ , ** kwargs ) :
    """Create a string representation for each item in a dict .
 Example :
 > > > from ubelt . util _ format import *
 > > > dict _ = { ' b ' : . 1 , ' l ' : ' st ' , ' g ' : 1.0 , ' s ' : 10 , ' m ' : 0.9 , ' w ' : . 5}
 > > > kwargs = { ' strkeys ' : True }
 > > > itemstrs , _ = _ dict _ itemstrs ( dict _ , * * kwargs )
 > > > char _ order = [ p [ 0 ] for p in itemstrs ]
 > > > assert char _ order = = [ ' b ' , ' g ' , ' l ' , ' m ' , ' s ' , ' w ' ]"""

implementation:
import ubelt as ub
explicit = kwargs . get ( 'explicit' , False )
kwargs [ 'explicit' ] = _rectify_countdown_or_bool ( explicit )
precision = kwargs . get ( 'precision' , None )
kvsep = kwargs . get ( 'kvsep' , ': ' )
if explicit :
    kvsep = '='
def make_item_str ( key , val ) :
    if explicit or kwargs . get ( 'strkeys' , False ) :
        key_str = six . text_type ( key )
    else :
        key_str = repr2 ( key , precision = precision , newlines = 0 )
    prefix = key_str + kvsep
    kwargs [ '_return_info' ] = True
    val_str , _leaf_info = repr2 ( val , ** kwargs )
    # If the first line does not end with an open nest char
    # ( e . g . for ndarrays ) , otherwise we need to worry about
    # residual indentation .
    pos = val_str . find ( '\n' )
    first_line = val_str if pos == - 1 else val_str [ : pos ]
    compact_brace = kwargs . get ( 'cbr' , kwargs . get ( 'compact_brace' , False ) )
    if compact_brace or not first_line . rstrip ( ) . endswith ( tuple ( '([{<' ) ) :
        rest = '' if pos == - 1 else val_str [ pos : ]
        val_str = first_line . lstrip ( ) + rest
        if '\n' in prefix : # Fix issue with keys that span new lines
            item_str = prefix + val_str
        else :
            item_str = ub . hzcat ( [ prefix , val_str ] )
    else :
        item_str = prefix + val_str
    return item_str , _leaf_info
items = list ( six . iteritems ( dict_ ) )
_tups = [ make_item_str ( key , val ) for ( key , val ) in items ]
itemstrs = [ t [ 0 ] for t in _tups ]
max_height = max ( [ t [ 1 ] [ 'max_height' ] for t in _tups ] ) if _tups else 0
_leaf_info = { 'max_height' : max_height + 1 , }
sort = kwargs . get ( 'sort' , None )
if sort is None : # Force ordering on unordered dicts
    sort = True
if isinstance ( dict_ , collections . OrderedDict ) : # never sort ordered dicts ; they are perfect just the way they are !
    sort = False
if sort :
    itemstrs = _sort_itemstrs ( items , itemstrs )
return itemstrs , _leaf_info


signature:
def _list_itemstrs ( list_ , ** kwargs ) :
    """Create a string representation for each item in a list ."""

implementation:
items = list ( list_ )
kwargs [ '_return_info' ] = True
_tups = [ repr2 ( item , ** kwargs ) for item in items ]
itemstrs = [ t [ 0 ] for t in _tups ]
max_height = max ( [ t [ 1 ] [ 'max_height' ] for t in _tups ] ) if _tups else 0
_leaf_info = { 'max_height' : max_height + 1 , }
sort = kwargs . get ( 'sort' , None )
if sort is None : # Force orderings on sets .
    sort = isinstance ( list_ , ( set , frozenset ) )
if sort :
    itemstrs = _sort_itemstrs ( items , itemstrs )
return itemstrs , _leaf_info


signature:
def _sort_itemstrs ( items , itemstrs ) :
    """Equivalent to ` sorted ( items ) ` except if ` items ` are unorderable , then
 string values are used to define an ordering ."""

implementation:
# First try to sort items by their normal values
# If that doesnt work , then sort by their string values
import ubelt as ub
try : # Set ordering is not unique . Sort by strings values instead .
    if _peek_isinstance ( items , ( set , frozenset ) ) :
        raise TypeError
    sortx = ub . argsort ( items )
except TypeError :
    sortx = ub . argsort ( itemstrs )
itemstrs = [ itemstrs [ x ] for x in sortx ]
return itemstrs


signature:
def _rectify_countdown_or_bool ( count_or_bool ) :
    """used by recursive functions to specify which level to turn a bool on in
 counting down yields True , True , . . . , False
 counting up yields False , False , False , . . . True
 Args :
 count _ or _ bool ( bool or int ) : if positive and an integer , it will count
 down , otherwise it will remain the same .
 Returns :
 int or bool : count _ or _ bool _
 CommandLine :
 python - m utool . util _ str - - test - _ rectify _ countdown _ or _ bool
 Example :
 > > > from ubelt . util _ format import _ rectify _ countdown _ or _ bool # NOQA
 > > > count _ or _ bool = True
 > > > a1 = ( _ rectify _ countdown _ or _ bool ( 2 ) )
 > > > a2 = ( _ rectify _ countdown _ or _ bool ( 1 ) )
 > > > a3 = ( _ rectify _ countdown _ or _ bool ( 0 ) )
 > > > a4 = ( _ rectify _ countdown _ or _ bool ( - 1 ) )
 > > > a5 = ( _ rectify _ countdown _ or _ bool ( - 2 ) )
 > > > a6 = ( _ rectify _ countdown _ or _ bool ( True ) )
 > > > a7 = ( _ rectify _ countdown _ or _ bool ( False ) )
 > > > a8 = ( _ rectify _ countdown _ or _ bool ( None ) )
 > > > result = [ a1 , a2 , a3 , a4 , a5 , a6 , a7 , a8]
 > > > print ( result )
 [1 , 0 , 0 , - 1 , - 2 , True , False , False ]"""

implementation:
if count_or_bool is True or count_or_bool is False :
    count_or_bool_ = count_or_bool
elif isinstance ( count_or_bool , int ) :
    if count_or_bool == 0 :
        return 0
    elif count_or_bool > 0 :
        count_or_bool_ = count_or_bool - 1
    else : # We dont countup negatives anymore
        count_or_bool_ = count_or_bool
else :
    count_or_bool_ = False
return count_or_bool_


signature:
def register ( self , type ) :
    """Registers a custom formatting function with ub . repr2"""

implementation:
def _decorator ( func ) :
    if isinstance ( type , tuple ) :
        for t in type :
            self . func_registry [ t ] = func
    else :
        self . func_registry [ type ] = func
    return func
return _decorator


signature:
def lookup ( self , data ) :
    """Returns an appropriate function to format ` data ` if one has been
 registered ."""

implementation:
for func in self . lazy_init :
    func ( )
for type , func in self . func_registry . items ( ) :
    if isinstance ( data , type ) :
        return func


signature:
def _register_numpy_extensions ( self ) :
    """CommandLine :
 python - m ubelt . util _ format FormatterExtensions . _ register _ numpy _ extensions
 Example :
 > > > import sys
 > > > import pytest
 > > > import ubelt as ub
 > > > if not ub . modname _ to _ modpath ( ' numpy ' ) :
 . . . raise pytest . skip ( )
 > > > # xdoctest : + IGNORE _ WHITESPACE
 > > > import numpy as np
 > > > data = np . array ( [ [ . 2 , 42 , 5 ] , [ 21.2 , 3 , . 4 ] ] )
 > > > print ( ub . repr2 ( data ) )
 np . array ( [ [ 0.2 , 42 . , 5 . ] ,
 [21.2 , 3 . , 0.4 ] ] , dtype = np . float64)
 > > > print ( ub . repr2 ( data , with _ dtype = False ) )
 np . array ( [ [ 0.2 , 42 . , 5 . ] ,
 [21.2 , 3 . , 0.4 ] ] )
 > > > print ( ub . repr2 ( data , strvals = True ) )
 [ [ 0.2 , 42 . , 5 . ] ,
 [21.2 , 3 . , 0.4 ] ]
 > > > data = np . empty ( ( 0 , 10 ) , dtype = np . float64)
 > > > print ( ub . repr2 ( data , strvals = False ) )
 np . empty ( ( 0 , 10 ) , dtype = np . float64)
 > > > print ( ub . repr2 ( data , strvals = True ) )
 > > > data = np . ma . empty ( ( 0 , 10 ) , dtype = np . float64)
 > > > print ( ub . repr2 ( data , strvals = False ) )
 np . ma . empty ( ( 0 , 10 ) , dtype = np . float64)"""

implementation:
import numpy as np
@ self . register ( np . ndarray )
def format_ndarray ( data , ** kwargs ) :
    import re
    strvals = kwargs . get ( 'sv' , kwargs . get ( 'strvals' , False ) )
    itemsep = kwargs . get ( 'itemsep' , ' ' )
    precision = kwargs . get ( 'precision' , None )
    suppress_small = kwargs . get ( 'supress_small' , None )
    max_line_width = kwargs . get ( 'max_line_width' , None )
    with_dtype = kwargs . get ( 'with_dtype' , kwargs . get ( 'dtype' , not strvals ) )
    newlines = kwargs . pop ( 'nl' , kwargs . pop ( 'newlines' , 1 ) )
    # if with _ dtype and strvals :
    # raise ValueError ( ' cannot format with strvals and dtype ' )
    separator = ',' + itemsep
    if strvals :
        prefix = ''
        suffix = ''
    else :
        modname = type ( data ) . __module__
        # substitute shorthand for numpy module names
        np_nice = 'np'
        modname = re . sub ( '\\bnumpy\\b' , np_nice , modname )
        modname = re . sub ( '\\bma.core\\b' , 'ma' , modname )
        class_name = type ( data ) . __name__
        if class_name == 'ndarray' :
            class_name = 'array'
        prefix = modname + '.' + class_name + '('
        if with_dtype :
            dtype_repr = data . dtype . name
            # dtype _ repr = np . core . arrayprint . dtype _ short _ repr ( data . dtype )
            suffix = ',{}dtype={}.{})' . format ( itemsep , np_nice , dtype_repr )
        else :
            suffix = ')'
    if not strvals and data . size == 0 and data . shape != ( 0 , ) : # Special case for displaying empty data
        prefix = modname + '.empty('
        body = repr ( tuple ( map ( int , data . shape ) ) )
    else :
        body = np . array2string ( data , precision = precision , separator = separator , suppress_small = suppress_small , prefix = prefix , max_line_width = max_line_width )
    if not newlines : # remove newlines if we need to
        body = re . sub ( '\n *' , '' , body )
    formatted = prefix + body + suffix
    return formatted
# Hack , make sure we also register numpy floats
self . register ( np . float32 ) ( self . func_registry [ float ] )


signature:
def _rectify_hasher ( hasher ) :
    """Convert a string - based key into a hasher class
 Notes :
 In terms of speed on 64bit systems , sha1 is the fastest followed by md5
 and sha512 . The slowest algorithm is sha256 . If xxhash is installed
 the fastest algorithm is xxh64.
 Example :
 > > > assert _ rectify _ hasher ( NoParam ) is DEFAULT _ HASHER
 > > > assert _ rectify _ hasher ( ' sha1 ' ) is hashlib . sha1
 > > > assert _ rectify _ hasher ( ' sha256 ' ) is hashlib . sha256
 > > > assert _ rectify _ hasher ( ' sha512 ' ) is hashlib . sha512
 > > > assert _ rectify _ hasher ( ' md5 ' ) is hashlib . md5
 > > > assert _ rectify _ hasher ( hashlib . sha1 ) is hashlib . sha1
 > > > assert _ rectify _ hasher ( hashlib . sha1 ( ) ) ( ) . name = = ' sha1'
 > > > import pytest
 > > > assert pytest . raises ( KeyError , _ rectify _ hasher , ' 42 ' )
 > > > # assert pytest . raises ( TypeError , _ rectify _ hasher , object )
 > > > if xxhash :
 > > > assert _ rectify _ hasher ( ' xxh64 ' ) is xxhash . xxh64
 > > > assert _ rectify _ hasher ( ' xxh32 ' ) is xxhash . xxh32"""

implementation:
if xxhash is not None : # pragma : nobranch
    if hasher in { 'xxh32' , 'xx32' , 'xxhash' } :
        return xxhash . xxh32
    if hasher in { 'xxh64' , 'xx64' } :
        return xxhash . xxh64
if hasher is NoParam or hasher == 'default' :
    hasher = DEFAULT_HASHER
elif isinstance ( hasher , six . string_types ) :
    if hasher not in hashlib . algorithms_available :
        raise KeyError ( 'unknown hasher: {}' . format ( hasher ) )
    else :
        hasher = getattr ( hashlib , hasher )
elif isinstance ( hasher , HASH ) : # by default the result of this function is a class we will make an
# instance of , if we already have an instance , wrap it in a callable
# so the external syntax does not need to change .
    return lambda : hasher
return hasher


signature:
def _rectify_base ( base ) :
    """transforms base shorthand into the full list representation
 Example :
 > > > assert _ rectify _ base ( NoParam ) is DEFAULT _ ALPHABET
 > > > assert _ rectify _ base ( ' hex ' ) is _ ALPHABET _ 16
 > > > assert _ rectify _ base ( ' abc ' ) is _ ALPHABET _ 26
 > > > assert _ rectify _ base ( 10 ) is _ ALPHABET _ 10
 > > > assert _ rectify _ base ( [ ' 1 ' , ' 2 ' ] ) = = [ ' 1 ' , ' 2 ' ]
 > > > import pytest
 > > > assert pytest . raises ( TypeError , _ rectify _ base , ' uselist ' )"""

implementation:
if base is NoParam or base == 'default' :
    return DEFAULT_ALPHABET
elif base in [ 26 , 'abc' , 'alpha' ] :
    return _ALPHABET_26
elif base in [ 16 , 'hex' ] :
    return _ALPHABET_16
elif base in [ 10 , 'dec' ] :
    return _ALPHABET_10
else :
    if not isinstance ( base , ( list , tuple ) ) :
        raise TypeError ( 'Argument `base` must be a key, list, or tuple; not {}' . format ( type ( base ) ) )
    return base


signature:
def _hashable_sequence ( data , types = True ) :
    r"""Extracts the sequence of bytes that would be hashed by hash _ data
 Example :
 > > > data = [ 2 , ( 3 , 4 ) ]
 > > > result1 = ( b ' ' . join ( _ hashable _ sequence ( data , types = False ) ) )
 > > > result2 = ( b ' ' . join ( _ hashable _ sequence ( data , types = True ) ) )
 > > > assert result1 = = b ' _ [ _ \ x02 _ , _ _ [ _ \ x03 _ , _ \ x04 _ , _ _ ] _ _ ] _ '
 > > > assert result2 = = b ' _ [ _ INT \ x02 _ , _ _ [ _ INT \ x03 _ , _ INT \ x04 _ , _ _ ] _ _ ] _ '"""

implementation:
hasher = _HashTracer ( )
_update_hasher ( hasher , data , types = types )
return hasher . sequence


signature:
def _convert_to_hashable ( data , types = True ) :
    r"""Converts ` data ` into a hashable byte representation if an appropriate
 hashing function is known .
 Args :
 data ( object ) : ordered data with structure
 types ( bool ) : include type prefixes in the hash
 Returns :
 tuple ( bytes , bytes ) : prefix , hashable :
 a prefix hinting the original data type and the byte representation
 of ` data ` .
 Raises :
 TypeError : if data has no registered hash methods
 Example :
 > > > assert _ convert _ to _ hashable ( None ) = = ( b ' NULL ' , b ' NONE ' )
 > > > assert _ convert _ to _ hashable ( ' string ' ) = = ( b ' TXT ' , b ' string ' )
 > > > assert _ convert _ to _ hashable ( 1 ) = = ( b ' INT ' , b ' \ x01 ' )
 > > > assert _ convert _ to _ hashable ( 1.0 ) = = ( b ' FLT ' , b ' \ x01 / \ x01 ' )
 > > > assert _ convert _ to _ hashable ( _ intlike [ - 1 ] ( 1 ) ) = = ( b ' INT ' , b ' \ x01 ' )"""

implementation:
# HANDLE MOST COMMON TYPES FIRST
if data is None :
    hashable = b'NONE'
    prefix = b'NULL'
elif isinstance ( data , six . binary_type ) :
    hashable = data
    prefix = b'TXT'
elif isinstance ( data , six . text_type ) : # convert unicode into bytes
    hashable = data . encode ( 'utf-8' )
    prefix = b'TXT'
elif isinstance ( data , _intlike ) : # warnings . warn ( ' Hashing ints is slow , numpy is prefered ' )
    hashable = _int_to_bytes ( data )
    # hashable = data . to _ bytes ( 8 , byteorder = ' big ' )
    prefix = b'INT'
elif isinstance ( data , float ) :
    a , b = float ( data ) . as_integer_ratio ( )
    hashable = _int_to_bytes ( a ) + b'/' + _int_to_bytes ( b )
    prefix = b'FLT'
else : # Then dynamically look up any other type
    hash_func = _HASHABLE_EXTENSIONS . lookup ( data )
    prefix , hashable = hash_func ( data )
if types :
    return prefix , hashable
else :
    return b'' , hashable


signature:
def _update_hasher ( hasher , data , types = True ) :
    """Converts ` data ` into a byte representation and calls update on the hasher
 ` hashlib . HASH ` algorithm .
 Args :
 hasher ( HASH ) : instance of a hashlib algorithm
 data ( object ) : ordered data with structure
 types ( bool ) : include type prefixes in the hash
 Example :
 > > > hasher = hashlib . sha512 ( )
 > > > data = [ 1 , 2 , [ ' a ' , 2 , ' c ' ] ]
 > > > _ update _ hasher ( hasher , data )
 > > > print ( hasher . hexdigest ( ) [ 0:8 ] )
 e2c67675
 2ba8d82b"""

implementation:
# Determine if the data should be hashed directly or iterated through
if isinstance ( data , ( tuple , list , zip ) ) :
    needs_iteration = True
else :
    needs_iteration = any ( check ( data ) for check in _HASHABLE_EXTENSIONS . iterable_checks )
if needs_iteration : # Denote that we are hashing over an iterable
# Multiple structure bytes makes it harder accidently make conflicts
    SEP = b'_,_'
    ITER_PREFIX = b'_[_'
    ITER_SUFFIX = b'_]_'
    iter_ = iter ( data )
    hasher . update ( ITER_PREFIX )
    # first , try to nest quickly without recursive calls
    # ( this works if all data in the sequence is a non - iterable )
    try :
        for item in iter_ :
            prefix , hashable = _convert_to_hashable ( item , types )
            binary_data = prefix + hashable + SEP
            hasher . update ( binary_data )
    except TypeError : # need to use recursive calls
    # Update based on current item
        _update_hasher ( hasher , item , types )
        for item in iter_ : # Ensure the items have a spacer between them
            _update_hasher ( hasher , item , types )
            hasher . update ( SEP )
    hasher . update ( ITER_SUFFIX )
else :
    prefix , hashable = _convert_to_hashable ( data , types )
    binary_data = prefix + hashable
    hasher . update ( binary_data )


signature:
def _convert_hexstr_base ( hexstr , base ) :
    r"""Packs a long hexstr into a shorter length string with a larger base .
 Args :
 hexstr ( str ) : string of hexidecimal symbols to convert
 base ( list ) : symbols of the conversion base
 Example :
 > > > print ( _ convert _ hexstr _ base ( ' fffff ' , _ ALPHABET _ 26 ) )
 nxmrlxv
 > > > print ( _ convert _ hexstr _ base ( ' 0 ' , _ ALPHABET _ 26 ) )
 > > > print ( _ convert _ hexstr _ base ( ' - fffff ' , _ ALPHABET _ 26 ) )
 - nxmrlxv
 > > > print ( _ convert _ hexstr _ base ( ' aafffff1 ' , _ ALPHABET _ 16 ) )
 aafffff1
 Sympy :
 > > > import sympy as sy
 > > > # Determine the length savings with lossless conversion
 > > > consts = dict ( hexbase = 16 , hexlen = 256 , baselen = 27)
 > > > symbols = sy . symbols ( ' hexbase , hexlen , baselen , newlen ' )
 > > > haexbase , hexlen , baselen , newlen = symbols
 > > > eqn = sy . Eq ( 16 * * hexlen , baselen * * newlen )
 > > > newlen _ ans = sy . solve ( eqn , newlen ) [ 0 ] . subs ( consts ) . evalf ( )
 > > > print ( ' newlen _ ans = % r ' % ( newlen _ ans , ) )
 > > > # for a 26 char base we can get 216
 > > > print ( ' Required length for lossless conversion len2 = % r ' % ( len2 , ) )
 > > > def info ( base , len ) :
 . . . bits = base * * len
 . . . print ( ' base = % r ' % ( base , ) )
 . . . print ( ' len = % r ' % ( len , ) )
 . . . print ( ' bits = % r ' % ( bits , ) )
 > > > info ( 16 , 256)
 > > > info ( 27 , 16)
 > > > info ( 27 , 64)
 > > > info ( 27 , 216)"""

implementation:
if base is _ALPHABET_16 : # already in hex , no conversion needed
    return hexstr
baselen = len ( base )
x = int ( hexstr , 16 )
# first convert to base 16
if x == 0 :
    return '0'
sign = 1 if x > 0 else - 1
x *= sign
digits = [ ]
while x :
    digits . append ( base [ x % baselen ] )
    x //= baselen
if sign < 0 :
    digits . append ( '-' )
digits . reverse ( )
newbase_str = '' . join ( digits )
return newbase_str


signature:
def _digest_hasher ( hasher , hashlen , base ) :
    """counterpart to _ update _ hasher"""

implementation:
# Get a 128 character hex string
hex_text = hasher . hexdigest ( )
# Shorten length of string ( by increasing base )
base_text = _convert_hexstr_base ( hex_text , base )
# Truncate
text = base_text [ : hashlen ]
return text


signature:
def hash_data ( data , hasher = NoParam , base = NoParam , types = False , hashlen = NoParam , convert = False ) :
    """Get a unique hash depending on the state of the data .
 Args :
 data ( object ) :
 Any sort of loosely organized data
 hasher ( str or HASHER ) :
 Hash algorithm from hashlib , defaults to ` sha512 ` .
 base ( str or List [ str ] ) :
 Shorthand key or a list of symbols . Valid keys are : ' abc ' , ' hex ' ,
 and ' dec ' . Defaults to ' hex ' .
 types ( bool ) :
 If True data types are included in the hash , otherwise only the raw
 data is hashed . Defaults to False .
 hashlen ( int ) :
 Maximum number of symbols in the returned hash . If not specified ,
 all are returned . DEPRECATED . Use slice syntax instead .
 convert ( bool , optional , default = True ) :
 if True , try and convert the data to json an the json is hashed
 instead . This can improve runtime in some instances , however the
 hash may differ from the case where convert = False .
 Notes :
 alphabet26 is a pretty nice base , I recommend it .
 However we default to hex because it is standard .
 This means the output of hashdata with base = sha1 will be the same as
 the output of ` sha1sum ` .
 Returns :
 str : text - hash string
 Example :
 > > > import ubelt as ub
 > > > print ( ub . hash _ data ( [ 1 , 2 , ( 3 , ' 4 ' ) ] , convert = False ) )
 60b758587f599663931057e6ebdf185a . . .
 > > > print ( ub . hash _ data ( [ 1 , 2 , ( 3 , ' 4 ' ) ] , base = ' abc ' , hasher = ' sha512 ' ) [ : 32 ] )
 hsrgqvfiuxvvhcdnypivhhthmrolkzej"""

implementation:
if convert and isinstance ( data , six . string_types ) : # nocover
    try :
        data = json . dumps ( data )
    except TypeError as ex : # import warnings
    # warnings . warn ( ' Unable to encode input as json due to : { ! r } ' . format ( ex ) )
        pass
base = _rectify_base ( base )
hashlen = _rectify_hashlen ( hashlen )
hasher = _rectify_hasher ( hasher ) ( )
# Feed the data into the hasher
_update_hasher ( hasher , data , types = types )
# Get the hashed representation
text = _digest_hasher ( hasher , hashlen , base )
return text


signature:
def hash_file ( fpath , blocksize = 65536 , stride = 1 , hasher = NoParam , hashlen = NoParam , base = NoParam ) :
    """Hashes the data in a file on disk .
 Args :
 fpath ( PathLike ) : file path string
 blocksize ( int ) : 2 * * 16 . Affects speed of reading file
 stride ( int ) : strides > 1 skip data to hash , useful for faster
 hashing , but less accurate , also makes hash dependant on
 blocksize .
 hasher ( HASH ) : hash algorithm from hashlib , defaults to ` sha512 ` .
 hashlen ( int ) : maximum number of symbols in the returned hash . If
 not specified , all are returned .
 base ( list , str ) : list of symbols or shorthand key . Valid keys are
 ' abc ' , ' hex ' , and ' dec ' . Defaults to ' hex ' .
 Notes :
 For better hashes keep stride = 1
 For faster hashes set stride > 1
 blocksize matters when stride > 1
 References :
 http : / / stackoverflow . com / questions / 3431825 / md5 - checksum - of - a - file
 http : / / stackoverflow . com / questions / 5001893 / when - to - use - sha - 1 - vs - sha - 2
 Example :
 > > > import ubelt as ub
 > > > from os . path import join
 > > > fpath = join ( ub . ensure _ app _ cache _ dir ( ' ubelt ' ) , ' tmp . txt ' )
 > > > ub . writeto ( fpath , ' foobar ' )
 > > > print ( ub . hash _ file ( fpath , hasher = ' sha1 ' , base = ' hex ' ) )
 8843d7f92416211de9ebb963ff4ce28125932878
 Example :
 > > > import ubelt as ub
 > > > from os . path import join
 > > > fpath = ub . touch ( join ( ub . ensure _ app _ cache _ dir ( ' ubelt ' ) , ' empty _ file ' ) )
 > > > # Test that the output is the same as sha1sum
 > > > if ub . find _ exe ( ' sha1sum ' ) :
 > > > want = ub . cmd ( [ ' sha1sum ' , fpath ] , verbose = 2 ) [ ' out ' ] . split ( ' ' ) [ 0]
 > > > got = ub . hash _ file ( fpath , hasher = ' sha1 ' )
 > > > print ( ' want = { ! r } ' . format ( want ) )
 > > > print ( ' got = { ! r } ' . format ( got ) )
 > > > assert want . endswith ( got )
 > > > # Do the same for sha512 sum and md5sum
 > > > if ub . find _ exe ( ' sha512sum ' ) :
 > > > want = ub . cmd ( [ ' sha512sum ' , fpath ] , verbose = 2 ) [ ' out ' ] . split ( ' ' ) [ 0]
 > > > got = ub . hash _ file ( fpath , hasher = ' sha512 ' )
 > > > print ( ' want = { ! r } ' . format ( want ) )
 > > > print ( ' got = { ! r } ' . format ( got ) )
 > > > assert want . endswith ( got )
 > > > if ub . find _ exe ( ' md5sum ' ) :
 > > > want = ub . cmd ( [ ' md5sum ' , fpath ] , verbose = 2 ) [ ' out ' ] . split ( ' ' ) [ 0]
 > > > got = ub . hash _ file ( fpath , hasher = ' md5 ' )
 > > > print ( ' want = { ! r } ' . format ( want ) )
 > > > print ( ' got = { ! r } ' . format ( got ) )
 > > > assert want . endswith ( got )"""

implementation:
base = _rectify_base ( base )
hashlen = _rectify_hashlen ( hashlen )
hasher = _rectify_hasher ( hasher ) ( )
with open ( fpath , 'rb' ) as file :
    buf = file . read ( blocksize )
    if stride > 1 : # skip blocks when stride is greater than 1
        while len ( buf ) > 0 :
            hasher . update ( buf )
            file . seek ( blocksize * ( stride - 1 ) , 1 )
            buf = file . read ( blocksize )
    else : # otherwise hash the entire file
        while len ( buf ) > 0 :
            hasher . update ( buf )
            buf = file . read ( blocksize )
# Get the hashed representation
text = _digest_hasher ( hasher , hashlen , base )
return text


signature:
def register ( self , hash_types ) :
    """Registers a function to generate a hash for data of the appropriate
 types . This can be used to register custom classes . Internally this is
 used to define how to hash non - builtin objects like ndarrays and uuids .
 The registered function should return a tuple of bytes . First a small
 prefix hinting at the data type , and second the raw bytes that can be
 hashed .
 Args :
 hash _ types ( class or tuple of classes ) :
 Returns :
 func : closure to be used as the decorator
 Example :
 > > > # xdoctest : + SKIP
 > > > # Skip this doctest because we dont want tests to modify
 > > > # the global state .
 > > > import ubelt as ub
 > > > import pytest
 > > > class MyType ( object ) :
 . . . def _ _ init _ _ ( self , id ) :
 . . . self . id = id
 > > > data = MyType ( 1)
 > > > # Custom types wont work with ub . hash _ data by default
 > > > with pytest . raises ( TypeError ) :
 . . . ub . hash _ data ( data )
 > > > # You can register your functions with ubelt ' s internal
 > > > # hashable _ extension registery .
 > > > @ ub . util _ hash . _ HASHABLE _ EXTENSIONS . register ( MyType )
 > > > def hash _ my _ type ( data ) :
 . . . return b ' mytype ' , six . b ( ub . hash _ data ( data . id ) )
 > > > # TODO : allow hash _ data to take an new instance of
 > > > # HashableExtensions , so we dont have to modify the global
 > > > # ubelt state when we run tests .
 > > > my _ instance = MyType ( 1)
 > > > ub . hash _ data ( my _ instance )"""

implementation:
# ensure iterable
if not isinstance ( hash_types , ( list , tuple ) ) :
    hash_types = [ hash_types ]
def _decor_closure ( hash_func ) :
    for hash_type in hash_types :
        key = ( hash_type . __module__ , hash_type . __name__ )
        self . keyed_extensions [ key ] = ( hash_type , hash_func )
    return hash_func
return _decor_closure


signature:
def lookup ( self , data ) :
    """Returns an appropriate function to hash ` data ` if one has been
 registered .
 Raises :
 TypeError : if data has no registered hash methods
 Example :
 > > > import ubelt as ub
 > > > import pytest
 > > > if not ub . modname _ to _ modpath ( ' numpy ' ) :
 . . . raise pytest . skip ( ' numpy is optional ' )
 > > > self = HashableExtensions ( )
 > > > self . _ register _ numpy _ extensions ( )
 > > > self . _ register _ builtin _ class _ extensions ( )
 > > > import numpy as np
 > > > data = np . array ( [ 1 , 2 , 3 ] )
 > > > self . lookup ( data [ 0 ] )
 > > > class Foo ( object ) :
 > > > def _ _ init _ _ ( f ) :
 > > > f . attr = 1
 > > > data = Foo ( )
 > > > assert pytest . raises ( TypeError , self . lookup , data )
 > > > # If ub . hash _ data doesnt support your object ,
 > > > # then you can register it .
 > > > @ self . register ( Foo )
 > > > def _ hashfoo ( data ) :
 > > > return b ' FOO ' , data . attr
 > > > func = self . lookup ( data )
 > > > assert func ( data ) [ 1 ] = = 1
 > > > data = uuid . uuid4 ( )
 > > > self . lookup ( data )"""

implementation:
# Maybe try using functools . singledispatch instead ?
# First try O ( 1 ) lookup
query_hash_type = data . __class__
key = ( query_hash_type . __module__ , query_hash_type . __name__ )
try :
    hash_type , hash_func = self . keyed_extensions [ key ]
except KeyError :
    raise TypeError ( 'No registered hash func for hashable type=%r' % ( query_hash_type ) )
return hash_func


signature:
def _textio_iterlines ( stream ) :
    """Iterates over lines in a TextIO stream until an EOF is encountered .
 This is the iterator version of stream . readlines ( )"""

implementation:
line = stream . readline ( )
while line != '' :
    yield line
    line = stream . readline ( )


signature:
def _proc_async_iter_stream ( proc , stream , buffersize = 1 ) :
    """Reads output from a process in a separate thread"""

implementation:
from six . moves import queue
from threading import Thread
def enqueue_output ( proc , stream , stream_queue ) :
    while proc . poll ( ) is None :
        line = stream . readline ( )
        # print ( ' ENQUEUE LIVE { ! r } { ! r } ' . format ( stream , line ) )
        stream_queue . put ( line )
    for line in _textio_iterlines ( stream ) : # print ( ' ENQUEUE FINAL { ! r } { ! r } ' . format ( stream , line ) )
        stream_queue . put ( line )
    # print ( " STREAM IS DONE { ! r } " . format ( stream ) )
    stream_queue . put ( None )
    # signal that the stream is finished
    # stream . close ( )
stream_queue = queue . Queue ( maxsize = buffersize )
_thread = Thread ( target = enqueue_output , args = ( proc , stream , stream_queue ) )
_thread . daemon = True
# thread dies with the program
_thread . start ( )
return stream_queue


signature:
def _proc_iteroutput_thread ( proc ) :
    """Iterates over output from a process line by line
 Note :
 WARNING . Current implementation might have bugs with other threads .
 This behavior was seen when using earlier versions of tqdm . I ' m not
 sure if this was our bug or tqdm ' s . Newer versions of tqdm fix this ,
 but I cannot guarantee that there isn ' t an issue on our end .
 Yields :
 Tuple [ str , str ] : oline , eline : stdout and stderr line
 References :
 https : / / stackoverflow . com / questions / 375427 / non - blocking - read - subproc"""

implementation:
from six . moves import queue
# Create threads that read stdout / stderr and queue up the output
stdout_queue = _proc_async_iter_stream ( proc , proc . stdout )
stderr_queue = _proc_async_iter_stream ( proc , proc . stderr )
stdout_live = True
stderr_live = True
# read from the output asynchronously until
while stdout_live or stderr_live :
    if stdout_live : # pragma : nobranch
        try :
            oline = stdout_queue . get_nowait ( )
            stdout_live = oline is not None
        except queue . Empty :
            oline = None
    if stderr_live :
        try :
            eline = stderr_queue . get_nowait ( )
            stderr_live = eline is not None
        except queue . Empty :
            eline = None
    if oline is not None or eline is not None :
        yield oline , eline


signature:
def _proc_iteroutput_select ( proc ) :
    """Iterates over output from a process line by line
 UNIX only . Use ` _ proc _ iteroutput _ thread ` instead for a cross platform
 solution based on threads .
 Yields :
 Tuple [ str , str ] : oline , eline : stdout and stderr line"""

implementation:
from six . moves import zip_longest
# Read output while the external program is running
while proc . poll ( ) is None :
    reads = [ proc . stdout . fileno ( ) , proc . stderr . fileno ( ) ]
    ret = select . select ( reads , [ ] , [ ] )
    oline = eline = None
    for fd in ret [ 0 ] :
        if fd == proc . stdout . fileno ( ) :
            oline = proc . stdout . readline ( )
        if fd == proc . stderr . fileno ( ) :
            eline = proc . stderr . readline ( )
    yield oline , eline
# Grab any remaining data in stdout and stderr after the process finishes
oline_iter = _textio_iterlines ( proc . stdout )
eline_iter = _textio_iterlines ( proc . stderr )
for oline , eline in zip_longest ( oline_iter , eline_iter ) :
    yield oline , eline


signature:
def _tee_output ( make_proc , stdout = None , stderr = None , backend = 'auto' ) :
    """Simultaneously reports and captures stdout and stderr from a process
 subprocess must be created using ( stdout = subprocess . PIPE ,
 stderr = subprocess . PIPE )"""

implementation:
logged_out = [ ]
logged_err = [ ]
if backend == 'auto' : # backend = ' select ' if POSIX else ' thread '
    backend = 'thread'
if backend == 'select' :
    if not POSIX : # nocover
        raise NotImplementedError ( 'select is only available on posix' )
    # the select - based version is stable , but slow
    _proc_iteroutput = _proc_iteroutput_select
elif backend == 'thread' : # the thread version is fast , but might run into issues .
    _proc_iteroutput = _proc_iteroutput_thread
else :
    raise ValueError ( 'backend must be select, thread, or auto' )
proc = make_proc ( )
for oline , eline in _proc_iteroutput ( proc ) :
    if oline :
        if stdout : # pragma : nobranch
            stdout . write ( oline )
            stdout . flush ( )
        logged_out . append ( oline )
    if eline :
        if stderr : # pragma : nobranch
            stderr . write ( eline )
            stderr . flush ( )
        logged_err . append ( eline )
return proc , logged_out , logged_err


signature:
def cmd ( command , shell = False , detach = False , verbose = 0 , tee = None , cwd = None , env = None , tee_backend = 'auto' , verbout = None , ** kwargs ) :
    """Executes a command in a subprocess .
 The advantage of this wrapper around subprocess is that
 (1 ) you control if the subprocess prints to stdout ,
 (2 ) the text written to stdout and stderr is returned for parsing ,
 (3 ) cross platform behavior that lets you specify the command as a string
 or tuple regardless of whether or not shell = True .
 (4 ) ability to detach , return the process object and allow the process to
 run in the background ( eventually we may return a Future object instead ) .
 Args :
 command ( str or Sequence ) : bash - like command string or tuple of
 executable and args
 shell ( bool ) : if True , process is run in shell , defaults to False .
 detach ( bool ) : if True , process is detached and run in background ,
 defaults to False .
 verbose ( int ) : verbosity mode . Can be 0 , 1 , 2 , or 3 . Defaults to 0.
 tee ( bool , optional ) : if True , simultaneously writes to stdout while
 capturing output from the command . If not specified , defaults to
 True if verbose > 0 . If detech is True , then this argument is
 ignored .
 cwd ( PathLike , optional ) : path to run command
 env ( str , optional ) : environment passed to Popen
 tee _ backend ( str , optional ) : backend for tee output .
 Valid choices are : " auto " , " select " ( POSIX only ) , and " thread " .
 * * kwargs : only used to support deprecated arguments
 Returns :
 dict : info - information about command status .
 if detach is False ` info ` contains captured standard out ,
 standard error , and the return code
 if detach is False ` info ` contains a reference to the process .
 Notes :
 Inputs can either be text or tuple based . On UNIX we ensure conversion
 to text if shell = True , and to tuple if shell = False . On windows , the
 input is always text based . See [ 3 ] for a potential cross - platform
 shlex solution for windows .
 CommandLine :
 python - m ubelt . util _ cmd cmd
 python - c " import ubelt as ub ; ub . cmd ( ' ping localhost - c 2 ' , verbose = 2 ) "
 References :
 [1 ] https : / / stackoverflow . com / questions / 11495783 / redirect - subprocess - stderr - to - stdout
 [2 ] https : / / stackoverflow . com / questions / 7729336 / how - can - i - print - and - display - subprocess - stdout - and - stderr - output - without - distorti
 [3 ] https : / / stackoverflow . com / questions / 33560364 / python - windows - parsing - command - lines - with - shlex
 Example :
 > > > info = cmd ( ( ' echo ' , ' simple cmdline interface ' ) , verbose = 1)
 simple cmdline interface
 > > > assert info [ ' ret ' ] = = 0
 > > > assert info [ ' out ' ] . strip ( ) = = ' simple cmdline interface '
 > > > assert info [ ' err ' ] . strip ( ) = = ' '
 Doctest :
 > > > info = cmd ( ' echo str noshell ' , verbose = 0)
 > > > assert info [ ' out ' ] . strip ( ) = = ' str noshell '
 Doctest :
 > > > # windows echo will output extra single quotes
 > > > info = cmd ( ( ' echo ' , ' tuple noshell ' ) , verbose = 0)
 > > > assert info [ ' out ' ] . strip ( ) . strip ( " ' " ) = = ' tuple noshell '
 Doctest :
 > > > # Note this command is formatted to work on win32 and unix
 > > > info = cmd ( ' echo str & & echo shell ' , verbose = 0 , shell = True )
 > > > assert info [ ' out ' ] . strip ( ) = = ' str ' + chr ( 10 ) + ' shell '
 Doctest :
 > > > info = cmd ( ( ' echo ' , ' tuple shell ' ) , verbose = 0 , shell = True )
 > > > assert info [ ' out ' ] . strip ( ) . strip ( " ' " ) = = ' tuple shell '
 Doctest :
 > > > import ubelt as ub
 > > > from os . path import join , exists
 > > > fpath1 = join ( ub . get _ app _ cache _ dir ( ' ubelt ' ) , ' cmdout1 . txt ' )
 > > > fpath2 = join ( ub . get _ app _ cache _ dir ( ' ubelt ' ) , ' cmdout2 . txt ' )
 > > > ub . delete ( fpath1)
 > > > ub . delete ( fpath2)
 > > > info1 = ub . cmd ( ( ' touch ' , fpath1 ) , detach = True )
 > > > info2 = ub . cmd ( ' echo writing2 > ' + fpath2 , shell = True , detach = True )
 > > > while not exists ( fpath1 ) :
 . . . pass
 > > > while not exists ( fpath2 ) :
 . . . pass
 > > > assert ub . readfrom ( fpath1 ) = = ' '
 > > > assert ub . readfrom ( fpath2 ) . strip ( ) = = ' writing2'
 > > > info1 [ ' proc ' ] . wait ( )
 > > > info2 [ ' proc ' ] . wait ( )"""

implementation:
if kwargs : # nocover
    if 'verbout' in kwargs :
        warnings . warn ( '`verbout` is deprecated and will be removed. ' 'Use `tee` instead' , DeprecationWarning )
        tee = kwargs . pop ( 'verbout' )
    if 'detatch' in kwargs :
        warnings . warn ( '`detatch` is deprecated (misspelled) and will be removed. ' 'Use `detach` instead' , DeprecationWarning )
        detach = kwargs . pop ( 'detatch' )
    if kwargs :
        raise ValueError ( 'Unknown kwargs: {}' . format ( list ( kwargs . keys ) ) )
# Determine if command is specified as text or a tuple
if isinstance ( command , six . string_types ) :
    command_text = command
    command_tup = None
else :
    import pipes
    command_tup = command
    command_text = ' ' . join ( list ( map ( pipes . quote , command_tup ) ) )
if shell or sys . platform . startswith ( 'win32' ) : # When shell = True , args is sent to the shell ( e . g . bin / sh ) as text
    args = command_text
else : # When shell = False , args is a list of executable and arguments
    if command_tup is None : # parse this out of the string
    # NOTE : perhaps use the solution from [ 3 ] here ?
        import shlex
        command_tup = shlex . split ( command_text )
        # command _ tup = shlex . split ( command _ text , posix = not WIN32)
    args = command_tup
if tee is None :
    tee = verbose > 0
if verbose > 1 :
    import os
    import platform
    import getpass
    from ubelt import util_path
    if verbose > 2 :
        try :
            print ( '┌─── START CMD ───' )
        except Exception : # nocover
            print ( '+=== START CMD ===' )
    cwd_ = os . getcwd ( ) if cwd is None else cwd
    compname = platform . node ( )
    username = getpass . getuser ( )
    cwd_ = util_path . compressuser ( cwd_ )
    ps1 = '[ubelt.cmd] {}@{}:{}$ ' . format ( username , compname , cwd_ )
    print ( ps1 + command_text )
# Create a new process to execute the command
def make_proc ( ) : # delay the creation of the process until we validate all args
    import subprocess
    proc = subprocess . Popen ( args , stdout = subprocess . PIPE , stderr = subprocess . PIPE , shell = shell , universal_newlines = True , cwd = cwd , env = env )
    return proc
if detach :
    info = { 'proc' : make_proc ( ) , 'command' : command_text }
    if verbose > 0 : # nocover
        print ( '...detaching' )
else :
    if tee : # we need to tee output and start threads if tee is False ?
        stdout , stderr = sys . stdout , sys . stderr
        proc , logged_out , logged_err = _tee_output ( make_proc , stdout , stderr , backend = tee_backend )
        try :
            out = '' . join ( logged_out )
        except UnicodeDecodeError : # nocover
            out = '\n' . join ( _ . decode ( 'utf-8' ) for _ in logged_out )
        try :
            err = '' . join ( logged_err )
        except UnicodeDecodeError : # nocover
            err = '\n' . join ( _ . decode ( 'utf-8' ) for _ in logged_err )
        ( out_ , err_ ) = proc . communicate ( )
    else :
        proc = make_proc ( )
        ( out , err ) = proc . communicate ( )
    # calling wait means that the process will terminate and it is safe to
    # return a reference to the process object .
    ret = proc . wait ( )
    info = { 'out' : out , 'err' : err , 'ret' : ret , 'proc' : proc , 'cwd' : cwd , 'command' : command_text }
    if verbose > 2 : # https : / / en . wikipedia . org / wiki / Box - drawing _ character
        try :
            print ( '└─── END CMD ───' )
        except Exception : # nocover
            print ( 'L___ END CMD ___' )
return info


signature:
def timestamp ( method = 'iso8601' ) :
    """make an iso8601 timestamp
 Args :
 method ( str ) : type of timestamp
 Example :
 > > > stamp = timestamp ( )
 > > > print ( ' stamp = { ! r } ' . format ( stamp ) )
 stamp = . . . - . . . - . . . T . . ."""

implementation:
if method == 'iso8601' : # ISO 8601
# datetime . datetime . utcnow ( ) . isoformat ( )
# datetime . datetime . now ( ) . isoformat ( )
# utcnow
    tz_hour = time . timezone // 3600
    utc_offset = str ( tz_hour ) if tz_hour < 0 else '+' + str ( tz_hour )
    stamp = time . strftime ( '%Y-%m-%dT%H%M%S' ) + utc_offset
    return stamp
else :
    raise ValueError ( 'only iso8601 is accepted for now' )


signature:
def benchmark_hash_data ( ) :
    """CommandLine :
 python ~ / code / ubelt / dev / bench _ hash . py - - convert = True - - show
 python ~ / code / ubelt / dev / bench _ hash . py - - convert = False - - show"""

implementation:
import ubelt as ub
# ITEM = ' JUST A STRING ' * 100
ITEM = [ 0 , 1 , 'a' , 'b' , [ 'JUST A STRING' ] * 4 ]
HASHERS = [ 'sha1' , 'sha512' , 'xxh32' , 'xxh64' ]
scales = list ( range ( 5 , 13 ) )
results = ub . AutoDict ( )
# Use json is faster or at least as fast it most cases
# xxhash is also significantly faster than sha512
convert = ub . argval ( '--convert' , default = 'True' ) . lower ( ) == 'True'
print ( 'convert = {!r}' . format ( convert ) )
ti = ub . Timerit ( 9 , bestof = 3 , verbose = 1 , unit = 'ms' )
for s in ub . ProgIter ( scales , desc = 'benchmark' , verbose = 3 ) :
    N = 2 ** s
    print ( ' --- s={s}, N={N} --- ' . format ( s = s , N = N ) )
    data = [ ITEM ] * N
    for hasher in HASHERS :
        for timer in ti . reset ( hasher ) :
            ub . hash_data ( data , hasher = hasher , convert = convert )
        results [ hasher ] . update ( { N : ti . mean ( ) } )
    col = { h : results [ h ] [ N ] for h in HASHERS }
    sortx = ub . argsort ( col )
    ranking = ub . dict_subset ( col , sortx )
    print ( 'walltime: ' + ub . repr2 ( ranking , precision = 9 , nl = 0 ) )
    best = next ( iter ( ranking ) )
    # pairs = list ( ub . iter _ window ( 2 ) )
    pairs = [ ( k , best ) for k in ranking ]
    ratios = [ ranking [ k1 ] / ranking [ k2 ] for k1 , k2 in pairs ]
    nicekeys = [ '{}/{}' . format ( k1 , k2 ) for k1 , k2 in pairs ]
    relratios = ub . odict ( zip ( nicekeys , ratios ) )
    print ( 'speedup: ' + ub . repr2 ( relratios , precision = 4 , nl = 0 ) )
# xdoc + REQUIRES ( - - show )
# import pytest
# pytest . skip ( )
import pandas as pd
df = pd . DataFrame . from_dict ( results )
df . columns . name = 'hasher'
df . index . name = 'N'
ratios = df . copy ( ) . drop ( columns = df . columns )
for k1 , k2 in [ ( 'sha512' , 'xxh32' ) , ( 'sha1' , 'xxh32' ) , ( 'xxh64' , 'xxh32' ) ] :
    ratios [ '{}/{}' . format ( k1 , k2 ) ] = df [ k1 ] / df [ k2 ]
print ( )
print ( 'Seconds per iteration' )
print ( df . to_string ( float_format = '%.9f' ) )
print ( )
print ( 'Ratios of seconds' )
print ( ratios . to_string ( float_format = '%.2f' ) )
print ( )
print ( 'Average Ratio (over all N)' )
print ( 'convert = {!r}' . format ( convert ) )
print ( ratios . mean ( ) . sort_values ( ) )
if ub . argflag ( '--show' ) :
    import netharn . util as kwel
    kwel . autompl ( )
    xdata = sorted ( ub . peek ( results . values ( ) ) . keys ( ) )
    ydata = ub . map_vals ( lambda d : [ d [ x ] for x in xdata ] , results )
    kwel . multi_plot ( xdata , ydata , xlabel = 'N' , ylabel = 'seconds' , title = 'convert = {}' . format ( convert ) )
    kwel . show_if_requested ( )


signature:
def import_module_from_path ( modpath , index = - 1 ) :
    """Imports a module via its path
 Args :
 modpath ( PathLike ) : path to the module on disk or within a zipfile .
 Returns :
 module : the imported module
 References :
 https : / / stackoverflow . com / questions / 67631 / import - module - given - path
 Notes :
 If the module is part of a package , the package will be imported first .
 These modules may cause problems when reloading via IPython magic
 This can import a module from within a zipfile . To do this modpath
 should specify the path to the zipfile and the path to the module
 within that zipfile separated by a colon or pathsep .
 E . g . ` / path / to / archive . zip : mymodule . py `
 Warning :
 It is best to use this with paths that will not conflict with
 previously existing modules .
 If the modpath conflicts with a previously existing module name . And
 the target module does imports of its own relative to this conflicting
 path . In this case , the module that was loaded first will win .
 For example if you try to import ' / foo / bar / pkg / mod . py ' from the folder
 structure :
 - foo /
 + - bar /
 + - pkg /
 + _ _ init _ _ . py
 | - mod . py
 | - helper . py
 If there exists another module named ` pkg ` already in sys . modules
 and mod . py does something like ` from . import helper ` , Python will
 assume helper belongs to the ` pkg ` module already in sys . modules .
 This can cause a NameError or worse - - - a incorrect helper module .
 Example :
 > > > import xdoctest
 > > > modpath = xdoctest . _ _ file _ _
 > > > module = import _ module _ from _ path ( modpath )
 > > > assert module is xdoctest
 Example :
 > > > # Test importing a module from within a zipfile
 > > > import zipfile
 > > > from xdoctest import utils
 > > > from os . path import join , expanduser
 > > > dpath = expanduser ( ' ~ / . cache / xdoctest ' )
 > > > dpath = utils . ensuredir ( dpath )
 > > > # dpath = utils . TempDir ( ) . ensure ( )
 > > > # Write to an external module named bar
 > > > external _ modpath = join ( dpath , ' bar . py ' )
 > > > open ( external _ modpath , ' w ' ) . write ( ' testvar = 1 ' )
 > > > internal = ' folder / bar . py '
 > > > # Move the external bar module into a zipfile
 > > > zippath = join ( dpath , ' myzip . zip ' )
 > > > with zipfile . ZipFile ( zippath , ' w ' ) as myzip :
 > > > myzip . write ( external _ modpath , internal )
 > > > # Import the bar module from within the zipfile
 > > > modpath = zippath + ' : ' + internal
 > > > modpath = zippath + os . path . sep + internal
 > > > module = import _ module _ from _ path ( modpath )
 > > > assert module . _ _ name _ _ = = os . path . normpath ( ' folder / bar ' )
 > > > assert module . testvar = = 1
 Doctest :
 > > > import pytest
 > > > with pytest . raises ( IOError ) :
 > > > import _ module _ from _ path ( ' does - not - exist ' )
 > > > with pytest . raises ( IOError ) :
 > > > import _ module _ from _ path ( ' does - not - exist . zip / ' )"""

implementation:
import os
if not os . path . exists ( modpath ) :
    import re
    import zipimport
    # We allow ( if not prefer or force ) the colon to be a path . sep in order
    # to agree with the mod . _ _ name _ _ attribute that will be produced
    # zip followed by colon or slash
    pat = '(.zip[' + re . escape ( os . path . sep ) + '/:])'
    parts = re . split ( pat , modpath , flags = re . IGNORECASE )
    if len ( parts ) > 2 :
        archivepath = '' . join ( parts [ : - 1 ] ) [ : - 1 ]
        internal = parts [ - 1 ]
        modname = os . path . splitext ( internal ) [ 0 ]
        modname = os . path . normpath ( modname )
        if os . path . exists ( archivepath ) :
            zimp_file = zipimport . zipimporter ( archivepath )
            module = zimp_file . load_module ( modname )
            return module
    raise IOError ( 'modpath={} does not exist' . format ( modpath ) )
else : # the importlib version doesnt work in pytest
    module = _custom_import_modpath ( modpath )
    # TODO : use this implementation once pytest fixes importlib
    # module = _ pkgutil _ import _ modpath ( modpath )
    return module


signature:
def import_module_from_name ( modname ) :
    """Imports a module from its string name ( _ _ name _ _ )
 Args :
 modname ( str ) : module name
 Returns :
 module : module
 Example :
 > > > # test with modules that wont be imported in normal circumstances
 > > > # todo write a test where we gaurentee this
 > > > modname _ list = [
 > > > ' pickletools ' ,
 > > > ' lib2to3 . fixes . fix _ apply ' ,
 > > > # assert not any ( m in sys . modules for m in modname _ list )
 > > > modules = [ import _ module _ from _ name ( modname ) for modname in modname _ list ]
 > > > assert [ m . _ _ name _ _ for m in modules ] = = modname _ list
 > > > assert all ( m in sys . modules for m in modname _ list )"""

implementation:
if True : # See if this fixes the Docker issue we saw but were unable to
# reproduce on another environment . Either way its better to use the
# standard importlib implementation than the one I wrote a long time
# ago .
    import importlib
    module = importlib . import_module ( modname )
else : # The _ _ import _ _ statment is weird
    if '.' in modname :
        fromlist = modname . split ( '.' ) [ - 1 ]
        fromlist_ = list ( map ( str , fromlist ) )
        # needs to be ascii for python2.7
        module = __import__ ( modname , { } , { } , fromlist_ , 0 )
    else :
        module = __import__ ( modname , { } , { } , [ ] , 0 )
return module


signature:
def _extension_module_tags ( ) :
    """Returns valid tags an extension module might have"""

implementation:
import sysconfig
tags = [ ]
if six . PY2 : # see also ' SHLIB _ EXT '
    multiarch = sysconfig . get_config_var ( 'MULTIARCH' )
    if multiarch is not None :
        tags . append ( multiarch )
else : # handle PEP 3149 - - ABI version tagged . so files
# ABI = application binary interface
    tags . append ( sysconfig . get_config_var ( 'SOABI' ) )
    tags . append ( 'abi3' )
    # not sure why this one is valid but it is
tags = [ t for t in tags if t ]
return tags


signature:
def _platform_pylib_exts ( ) : # nocover
    """Returns . so , . pyd , or . dylib depending on linux , win or mac .
 On python3 return the previous with and without abi ( e . g .
 . cpython - 35m - x86_64 - linux - gnu ) flags . On python2 returns with
 and without multiarch ."""

implementation:
import sysconfig
valid_exts = [ ]
if six . PY2 : # see also ' SHLIB _ EXT '
    base_ext = '.' + sysconfig . get_config_var ( 'SO' ) . split ( '.' ) [ - 1 ]
else : # return with and without API flags
# handle PEP 3149 - - ABI version tagged . so files
    base_ext = '.' + sysconfig . get_config_var ( 'EXT_SUFFIX' ) . split ( '.' ) [ - 1 ]
for tag in _extension_module_tags ( ) :
    valid_exts . append ( '.' + tag + base_ext )
valid_exts . append ( base_ext )
return tuple ( valid_exts )


signature:
def _syspath_modname_to_modpath ( modname , sys_path = None , exclude = None ) :
    """syspath version of modname _ to _ modpath
 Args :
 modname ( str ) : name of module to find
 sys _ path ( List [ PathLike ] , default = None ) :
 if specified overrides ` sys . path `
 exclude ( List [ PathLike ] , default = None ) :
 list of directory paths . if specified prevents these directories
 from being searched .
 Notes :
 This is much slower than the pkgutil mechanisms .
 CommandLine :
 python - m xdoctest . static _ analysis _ syspath _ modname _ to _ modpath
 Example :
 > > > print ( _ syspath _ modname _ to _ modpath ( ' xdoctest . static _ analysis ' ) )
 . . . static _ analysis . py
 > > > print ( _ syspath _ modname _ to _ modpath ( ' xdoctest ' ) )
 . . . xdoctest
 > > > print ( _ syspath _ modname _ to _ modpath ( ' _ ctypes ' ) )
 . . . _ ctypes . . .
 > > > assert _ syspath _ modname _ to _ modpath ( ' xdoctest ' , sys _ path = [ ] ) is None
 > > > assert _ syspath _ modname _ to _ modpath ( ' xdoctest . static _ analysis ' , sys _ path = [ ] ) is None
 > > > assert _ syspath _ modname _ to _ modpath ( ' _ ctypes ' , sys _ path = [ ] ) is None
 > > > assert _ syspath _ modname _ to _ modpath ( ' this ' , sys _ path = [ ] ) is None
 Example :
 > > > # test what happens when the module is not visible in the path
 > > > modname = ' xdoctest . static _ analysis '
 > > > modpath = _ syspath _ modname _ to _ modpath ( modname )
 > > > exclude = [ split _ modpath ( modpath ) [ 0 ] ]
 > > > found = _ syspath _ modname _ to _ modpath ( modname , exclude = exclude )
 > > > # this only works if installed in dev mode , pypi fails
 > > > assert found is None , ' should not have found { } ' . format ( found )"""

implementation:
def _isvalid ( modpath , base ) : # every directory up to the module , should have an init
    subdir = dirname ( modpath )
    while subdir and subdir != base :
        if not exists ( join ( subdir , '__init__.py' ) ) :
            return False
        subdir = dirname ( subdir )
    return True
_fname_we = modname . replace ( '.' , os . path . sep )
candidate_fnames = [ _fname_we + '.py' , # _ fname _ we + ' . pyc ' ,
# _ fname _ we + ' . pyo ' ,
]
# Add extension library suffixes
candidate_fnames += [ _fname_we + ext for ext in _platform_pylib_exts ( ) ]
if sys_path is None :
    sys_path = sys . path
# the empty string in sys . path indicates cwd . Change this to a ' . '
candidate_dpaths = [ '.' if p == '' else p for p in sys_path ]
if exclude :
    def normalize ( p ) :
        if sys . platform . startswith ( 'win32' ) : # nocover
            return realpath ( p ) . lower ( )
        else :
            return realpath ( p )
    # Keep only the paths not in exclude
    real_exclude = { normalize ( p ) for p in exclude }
    candidate_dpaths = [ p for p in candidate_dpaths if normalize ( p ) not in real_exclude ]
for dpath in candidate_dpaths : # Check for directory - based modules ( has presidence over files )
    modpath = join ( dpath , _fname_we )
    if exists ( modpath ) :
        if isfile ( join ( modpath , '__init__.py' ) ) :
            if _isvalid ( modpath , dpath ) :
                return modpath
    # If that fails , check for file - based modules
    for fname in candidate_fnames :
        modpath = join ( dpath , fname )
        if isfile ( modpath ) :
            if _isvalid ( modpath , dpath ) :
                return modpath


signature:
def modname_to_modpath ( modname , hide_init = True , hide_main = False , sys_path = None ) :
    """Finds the path to a python module from its name .
 Determines the path to a python module without directly import it
 Converts the name of a module ( _ _ name _ _ ) to the path ( _ _ file _ _ ) where it is
 located without importing the module . Returns None if the module does not
 exist .
 Args :
 modname ( str ) : module filepath
 hide _ init ( bool ) : if False , _ _ init _ _ . py will be returned for packages
 hide _ main ( bool ) : if False , and hide _ init is True , _ _ main _ _ . py will be
 returned for packages , if it exists .
 sys _ path ( list ) : if specified overrides ` sys . path ` ( default None )
 Returns :
 str : modpath - path to the module , or None if it doesn ' t exist
 CommandLine :
 python - m xdoctest . static _ analysis modname _ to _ modpath : 0
 pytest / home / joncrall / code / xdoctest / xdoctest / static _ analysis . py : : modname _ to _ modpath : 0
 Example :
 > > > modname = ' xdoctest . _ _ main _ _ '
 > > > modpath = modname _ to _ modpath ( modname , hide _ main = False )
 > > > assert modpath . endswith ( ' _ _ main _ _ . py ' )
 > > > modname = ' xdoctest '
 > > > modpath = modname _ to _ modpath ( modname , hide _ init = False )
 > > > assert modpath . endswith ( ' _ _ init _ _ . py ' )
 > > > modpath = basename ( modname _ to _ modpath ( ' _ ctypes ' ) )
 > > > assert ' ctypes ' in modpath"""

implementation:
modpath = _syspath_modname_to_modpath ( modname , sys_path )
if modpath is None :
    return None
modpath = normalize_modpath ( modpath , hide_init = hide_init , hide_main = hide_main )
return modpath


signature:
def normalize_modpath ( modpath , hide_init = True , hide_main = False ) :
    """Normalizes _ _ init _ _ and _ _ main _ _ paths .
 Notes :
 Adds _ _ init _ _ if reasonable , but only removes _ _ main _ _ by default
 Args :
 hide _ init ( bool ) : if True , always return package modules
 as _ _ init _ _ . py files otherwise always return the dpath .
 hide _ init ( bool ) : if True , always strip away main files otherwise
 ignore _ _ main _ _ . py .
 CommandLine :
 xdoctest - m xdoctest . static _ analysis normalize _ modpath
 Example :
 > > > import xdoctest . static _ analysis as static
 > > > modpath = static . _ _ file _ _
 > > > assert static . normalize _ modpath ( modpath ) = = modpath . replace ( ' . pyc ' , ' . py ' )
 > > > dpath = dirname ( modpath )
 > > > res0 = static . normalize _ modpath ( dpath , hide _ init = 0 , hide _ main = 0)
 > > > res1 = static . normalize _ modpath ( dpath , hide _ init = 0 , hide _ main = 1)
 > > > res2 = static . normalize _ modpath ( dpath , hide _ init = 1 , hide _ main = 0)
 > > > res3 = static . normalize _ modpath ( dpath , hide _ init = 1 , hide _ main = 1)
 > > > assert res0 . endswith ( ' _ _ init _ _ . py ' )
 > > > assert res1 . endswith ( ' _ _ init _ _ . py ' )
 > > > assert not res2 . endswith ( ' . py ' )
 > > > assert not res3 . endswith ( ' . py ' )"""

implementation:
if six . PY2 :
    if modpath . endswith ( '.pyc' ) :
        modpath = modpath [ : - 1 ]
if hide_init :
    if basename ( modpath ) == '__init__.py' :
        modpath = dirname ( modpath )
        hide_main = True
else : # add in init , if reasonable
    modpath_with_init = join ( modpath , '__init__.py' )
    if exists ( modpath_with_init ) :
        modpath = modpath_with_init
if hide_main : # We can remove main , but dont add it
    if basename ( modpath ) == '__main__.py' : # corner case where main might just be a module name not in a pkg
        parallel_init = join ( dirname ( modpath ) , '__init__.py' )
        if exists ( parallel_init ) :
            modpath = dirname ( modpath )
return modpath


signature:
def modpath_to_modname ( modpath , hide_init = True , hide_main = False , check = True , relativeto = None ) :
    """Determines importable name from file path
 Converts the path to a module ( _ _ file _ _ ) to the importable python name
 ( _ _ name _ _ ) without importing the module .
 The filename is converted to a module name , and parent directories are
 recursively included until a directory without an _ _ init _ _ . py file is
 encountered .
 Args :
 modpath ( str ) : module filepath
 hide _ init ( bool ) : removes the _ _ init _ _ suffix ( default True )
 hide _ main ( bool ) : removes the _ _ main _ _ suffix ( default False )
 check ( bool ) : if False , does not raise an error if modpath is a dir
 and does not contain an _ _ init _ _ file .
 relativeto ( str , optional ) : if specified , all checks are ignored and
 this is considered the path to the root module .
 Returns :
 str : modname
 Raises :
 ValueError : if check is True and the path does not exist
 CommandLine :
 xdoctest - m xdoctest . static _ analysis modpath _ to _ modname
 Example :
 > > > from xdoctest import static _ analysis
 > > > modpath = static _ analysis . _ _ file _ _ . replace ( ' . pyc ' , ' . py ' )
 > > > modpath = modpath . replace ( ' . pyc ' , ' . py ' )
 > > > modname = modpath _ to _ modname ( modpath )
 > > > assert modname = = ' xdoctest . static _ analysis '
 Example :
 > > > import xdoctest
 > > > assert modpath _ to _ modname ( xdoctest . _ _ file _ _ . replace ( ' . pyc ' , ' . py ' ) ) = = ' xdoctest '
 > > > assert modpath _ to _ modname ( dirname ( xdoctest . _ _ file _ _ . replace ( ' . pyc ' , ' . py ' ) ) ) = = ' xdoctest '
 Example :
 > > > modpath = modname _ to _ modpath ( ' _ ctypes ' )
 > > > modname = modpath _ to _ modname ( modpath )
 > > > assert modname = = ' _ ctypes '"""

implementation:
if check and relativeto is None :
    if not exists ( modpath ) :
        raise ValueError ( 'modpath={} does not exist' . format ( modpath ) )
modpath_ = abspath ( expanduser ( modpath ) )
modpath_ = normalize_modpath ( modpath_ , hide_init = hide_init , hide_main = hide_main )
if relativeto :
    dpath = dirname ( abspath ( expanduser ( relativeto ) ) )
    rel_modpath = relpath ( modpath_ , dpath )
else :
    dpath , rel_modpath = split_modpath ( modpath_ , check = check )
modname = splitext ( rel_modpath ) [ 0 ]
if '.' in modname :
    modname , abi_tag = modname . split ( '.' )
modname = modname . replace ( '/' , '.' )
modname = modname . replace ( '\\' , '.' )
return modname


signature:
def split_modpath ( modpath , check = True ) :
    """Splits the modpath into the dir that must be in PYTHONPATH for the module
 to be imported and the modulepath relative to this directory .
 Args :
 modpath ( str ) : module filepath
 check ( bool ) : if False , does not raise an error if modpath is a
 directory and does not contain an ` _ _ init _ _ . py ` file .
 Returns :
 tuple : ( directory , rel _ modpath )
 Raises :
 ValueError : if modpath does not exist or is not a package
 Example :
 > > > from xdoctest import static _ analysis
 > > > modpath = static _ analysis . _ _ file _ _ . replace ( ' . pyc ' , ' . py ' )
 > > > modpath = abspath ( modpath )
 > > > dpath , rel _ modpath = split _ modpath ( modpath )
 > > > recon = join ( dpath , rel _ modpath )
 > > > assert recon = = modpath
 > > > assert rel _ modpath = = join ( ' xdoctest ' , ' static _ analysis . py ' )"""

implementation:
if six . PY2 :
    if modpath . endswith ( '.pyc' ) :
        modpath = modpath [ : - 1 ]
modpath_ = abspath ( expanduser ( modpath ) )
if check :
    if not exists ( modpath_ ) :
        if not exists ( modpath ) :
            raise ValueError ( 'modpath={} does not exist' . format ( modpath ) )
        raise ValueError ( 'modpath={} is not a module' . format ( modpath ) )
    if isdir ( modpath_ ) and not exists ( join ( modpath , '__init__.py' ) ) : # dirs without inits are not modules
        raise ValueError ( 'modpath={} is not a module' . format ( modpath ) )
full_dpath , fname_ext = split ( modpath_ )
_relmod_parts = [ fname_ext ]
# Recurse down directories until we are out of the package
dpath = full_dpath
while exists ( join ( dpath , '__init__.py' ) ) :
    dpath , dname = split ( dpath )
    _relmod_parts . append ( dname )
relmod_parts = _relmod_parts [ : : - 1 ]
rel_modpath = os . path . sep . join ( relmod_parts )
return dpath , rel_modpath


signature:
def argval ( key , default = util_const . NoParam , argv = None ) :
    """Get the value of a keyword argument specified on the command line .
 Values can be specified as ` < key > < value > ` or ` < key > = < value > `
 Args :
 key ( str or tuple ) : string or tuple of strings . Each key should be
 prefixed with two hyphens ( i . e . ` - - ` )
 default ( Optional [ object ] ) : value to return if not specified
 argv ( Optional [ list ] ) : overrides ` sys . argv ` if specified
 Returns :
 str : value : the value specified after the key . It they key is
 specified multiple times , then the first value is returned .
 TODO :
 - [ ] Can we handle the case where the value is a list of long paths ?
 - [ ] Should we default the first or last specified instance of the flag .
 Example :
 > > > import ubelt as ub
 > > > argv = [ ' - - ans ' , ' 42 ' , ' - - quest = the grail ' , ' - - ans = 6 ' , ' - - bad ' ]
 > > > assert ub . argval ( ' - - spam ' , argv = argv ) = = ub . NoParam
 > > > assert ub . argval ( ' - - quest ' , argv = argv ) = = ' the grail '
 > > > assert ub . argval ( ' - - ans ' , argv = argv ) = = ' 42'
 > > > assert ub . argval ( ' - - bad ' , argv = argv ) = = ub . NoParam
 > > > assert ub . argval ( ( ' - - bad ' , ' - - bar ' ) , argv = argv ) = = ub . NoParam
 Example :
 > > > # Test fix for GH Issue # 41
 > > > import ubelt as ub
 > > > argv = [ ' - - path = / path / with / k = 3 ' ]
 > > > ub . argval ( ' - - path ' , argv = argv ) = = ' / path / with / k = 3'"""

implementation:
if argv is None : # nocover
    argv = sys . argv
keys = [ key ] if isinstance ( key , six . string_types ) else key
n_max = len ( argv ) - 1
for argx , item in enumerate ( argv ) :
    for key_ in keys :
        if item == key_ :
            if argx < n_max :
                value = argv [ argx + 1 ]
                return value
        elif item . startswith ( key_ + '=' ) :
            value = '=' . join ( item . split ( '=' ) [ 1 : ] )
            return value
value = default
return value


signature:
def argflag ( key , argv = None ) :
    """Determines if a key is specified on the command line
 Args :
 key ( str or tuple ) : string or tuple of strings . Each key should be
 prefixed with two hyphens ( i . e . ` - - ` )
 argv ( Optional [ list ] ) : overrides ` sys . argv ` if specified
 Returns :
 bool : flag : True if the key ( or any of the keys ) was specified
 Example :
 > > > import ubelt as ub
 > > > argv = [ ' - - spam ' , ' - - eggs ' , ' foo ' ]
 > > > assert ub . argflag ( ' - - eggs ' , argv = argv ) is True
 > > > assert ub . argflag ( ' - - ans ' , argv = argv ) is False
 > > > assert ub . argflag ( ' foo ' , argv = argv ) is True
 > > > assert ub . argflag ( ( ' bar ' , ' - - spam ' ) , argv = argv ) is True"""

implementation:
if argv is None : # nocover
    argv = sys . argv
keys = [ key ] if isinstance ( key , six . string_types ) else key
flag = any ( k in argv for k in keys )
return flag


signature:
def hzcat ( args , sep = '' ) :
    """Horizontally concatenates strings preserving indentation
 Concatenates a list of objects ensuring that the next item in the list is
 all the way to the right of any previous items .
 Args :
 args ( List [ str ] ) : strings to concatenate
 sep ( str ) : separator ( defaults to ' ' )
 CommandLine :
 python - m ubelt . util _ str hzcat
 Example1:
 > > > import ubelt as ub
 > > > B = ub . repr2 ( [ [ 1 , 2 ] , [ 3 , 457 ] ] , nl = 1 , cbr = True , trailsep = False )
 > > > C = ub . repr2 ( [ [ 5 , 6 ] , [ 7 , 8 ] ] , nl = 1 , cbr = True , trailsep = False )
 > > > args = [ ' A = ' , B , ' * ' , C ]
 > > > print ( ub . hzcat ( args ) )
 A = [ [ 1 , 2 ] , * [ [ 5 , 6 ] ,
 [3 , 457 ] ] [ 7 , 8 ] ]
 Example2:
 > > > from ubelt . util _ str import *
 > > > import ubelt as ub
 > > > import unicodedata
 > > > aa = unicodedata . normalize ( ' NFD ' , ' á ' ) # a unicode char with len2
 > > > B = ub . repr2 ( [ [ ' θ ' , aa ] , [ aa , aa , aa ] ] , nl = 1 , si = True , cbr = True , trailsep = False )
 > > > C = ub . repr2 ( [ [ 5 , 6 ] , [ 7 , ' θ ' ] ] , nl = 1 , si = True , cbr = True , trailsep = False )
 > > > args = [ ' A ' , ' = ' , B , ' * ' , C ]
 > > > print ( ub . hzcat ( args , sep = ' | ' ) )
 A | = | [ [ θ , á ] , | * | [ [ 5 , 6 ] ,
 | | [ á , á , á ] ] | | [ 7 , θ ] ]"""

implementation:
import unicodedata
if '\n' in sep or '\r' in sep :
    raise ValueError ( '`sep` cannot contain newline characters' )
# TODO : ensure unicode data works correctly for python2
args = [ unicodedata . normalize ( 'NFC' , ensure_unicode ( val ) ) for val in args ]
arglines = [ a . split ( '\n' ) for a in args ]
height = max ( map ( len , arglines ) )
# Do vertical padding
arglines = [ lines + [ '' ] * ( height - len ( lines ) ) for lines in arglines ]
# Initialize output
all_lines = [ '' for _ in range ( height ) ]
width = 0
n_args = len ( args )
for sx , lines in enumerate ( arglines ) : # Concatenate the new string
    for lx , line in enumerate ( lines ) :
        all_lines [ lx ] += line
    # Find the new maximum horizontal width
    width = max ( width , max ( map ( len , all_lines ) ) )
    if sx < n_args - 1 : # Horizontal padding on all but last iter
        for lx , line in list ( enumerate ( all_lines ) ) :
            residual = width - len ( line )
            all_lines [ lx ] = line + ( ' ' * residual ) + sep
        width += len ( sep )
# Clean up trailing whitespace
all_lines = [ line . rstrip ( ' ' ) for line in all_lines ]
ret = '\n' . join ( all_lines )
return ret


signature:
def ensure_unicode ( text ) :
    r"""Casts bytes into utf8 ( mostly for python2 compatibility )
 References :
 http : / / stackoverflow . com / questions / 12561063 / extract - data - from - file
 Example :
 > > > from ubelt . util _ str import *
 > > > import codecs # NOQA
 > > > assert ensure _ unicode ( ' my ünicôdé strįng ' ) = = ' my ünicôdé strįng '
 > > > assert ensure _ unicode ( ' text1 ' ) = = ' text1'
 > > > assert ensure _ unicode ( ' text1 ' . encode ( ' utf8 ' ) ) = = ' text1'
 > > > assert ensure _ unicode ( ' ï » ¿ text1 ' . encode ( ' utf8 ' ) ) = = ' ï » ¿ text1'
 > > > assert ( codecs . BOM _ UTF8 + ' text » ¿ ' . encode ( ' utf8 ' ) ) . decode ( ' utf8 ' )"""

implementation:
if isinstance ( text , six . text_type ) :
    return text
elif isinstance ( text , six . binary_type ) :
    return text . decode ( 'utf8' )
else : # nocover
    raise ValueError ( 'unknown input type {!r}' . format ( text ) )


signature:
def symlink ( real_path , link_path , overwrite = False , verbose = 0 ) :
    """Create a symbolic link .
 This will work on linux or windows , however windows does have some corner
 cases . For more details see notes in ` ubelt . _ win32 _ links ` .
 Args :
 path ( PathLike ) : path to real file or directory
 link _ path ( PathLike ) : path to desired location for symlink
 overwrite ( bool ) : overwrite existing symlinks .
 This will not overwrite real files on systems with proper symlinks .
 However , on older versions of windows junctions are
 indistinguishable from real files , so we cannot make this
 guarantee . ( default = False )
 verbose ( int ) : verbosity level ( default = 0)
 Returns :
 PathLike : link path
 CommandLine :
 python - m ubelt . util _ links symlink : 0
 Example :
 > > > import ubelt as ub
 > > > dpath = ub . ensure _ app _ cache _ dir ( ' ubelt ' , ' test _ symlink0 ' )
 > > > real _ path = join ( dpath , ' real _ file . txt ' )
 > > > link _ path = join ( dpath , ' link _ file . txt ' )
 > > > [ ub . delete ( p ) for p in [ real _ path , link _ path ] ]
 > > > ub . writeto ( real _ path , ' foo ' )
 > > > result = symlink ( real _ path , link _ path )
 > > > assert ub . readfrom ( result ) = = ' foo '
 > > > [ ub . delete ( p ) for p in [ real _ path , link _ path ] ]
 Example :
 > > > import ubelt as ub
 > > > from os . path import dirname
 > > > dpath = ub . ensure _ app _ cache _ dir ( ' ubelt ' , ' test _ symlink1 ' )
 > > > ub . delete ( dpath )
 > > > ub . ensuredir ( dpath )
 > > > _ dirstats ( dpath )
 > > > real _ dpath = ub . ensuredir ( ( dpath , ' real _ dpath ' ) )
 > > > link _ dpath = ub . augpath ( real _ dpath , base = ' link _ dpath ' )
 > > > real _ path = join ( dpath , ' afile . txt ' )
 > > > link _ path = join ( dpath , ' afile . txt ' )
 > > > [ ub . delete ( p ) for p in [ real _ path , link _ path ] ]
 > > > ub . writeto ( real _ path , ' foo ' )
 > > > result = symlink ( real _ dpath , link _ dpath )
 > > > assert ub . readfrom ( link _ path ) = = ' foo ' , ' read should be same '
 > > > ub . writeto ( link _ path , ' bar ' )
 > > > _ dirstats ( dpath )
 > > > assert ub . readfrom ( link _ path ) = = ' bar ' , ' very bad bar '
 > > > assert ub . readfrom ( real _ path ) = = ' bar ' , ' changing link did not change real '
 > > > ub . writeto ( real _ path , ' baz ' )
 > > > _ dirstats ( dpath )
 > > > assert ub . readfrom ( real _ path ) = = ' baz ' , ' very bad baz '
 > > > assert ub . readfrom ( link _ path ) = = ' baz ' , ' changing real did not change link '
 > > > ub . delete ( link _ dpath , verbose = 1)
 > > > _ dirstats ( dpath )
 > > > assert not exists ( link _ dpath ) , ' link should not exist '
 > > > assert exists ( real _ path ) , ' real path should exist '
 > > > _ dirstats ( dpath )
 > > > ub . delete ( dpath , verbose = 1)
 > > > _ dirstats ( dpath )
 > > > assert not exists ( real _ path )"""

implementation:
path = normpath ( real_path )
link = normpath ( link_path )
if not os . path . isabs ( path ) : # if path is not absolute it must be specified relative to link
    if _can_symlink ( ) :
        path = os . path . relpath ( path , os . path . dirname ( link ) )
    else : # nocover
    # On windows , we need to use absolute paths
        path = os . path . abspath ( path )
if verbose :
    print ( 'Symlink: {path} -> {link}' . format ( path = path , link = link ) )
if islink ( link ) :
    if verbose :
        print ( '... already exists' )
    pointed = _readlink ( link )
    if pointed == path :
        if verbose > 1 :
            print ( '... and points to the right place' )
        return link
    if verbose > 1 :
        if not exists ( link ) :
            print ( '... but it is broken and points somewhere else: {}' . format ( pointed ) )
        else :
            print ( '... but it points somewhere else: {}' . format ( pointed ) )
    if overwrite :
        util_io . delete ( link , verbose = verbose > 1 )
elif exists ( link ) :
    if _win32_links is None :
        if verbose :
            print ( '... already exists, but its a file. This will error.' )
        raise FileExistsError ( 'cannot overwrite a physical path: "{}"' . format ( path ) )
    else : # nocover
        if verbose :
            print ( '... already exists, and is either a file or hard link. ' 'Assuming it is a hard link. ' 'On non-win32 systems this would error.' )
if _win32_links is None :
    os . symlink ( path , link )
else : # nocover
    _win32_links . _symlink ( path , link , overwrite = overwrite , verbose = verbose )
return link


signature:
def _dirstats ( dpath = None ) : # nocover
    """Testing helper for printing directory information
 ( mostly for investigating windows weirdness )
 CommandLine :
 python - m ubelt . util _ links _ dirstats"""

implementation:
from ubelt import util_colors
if dpath is None :
    dpath = os . getcwd ( )
print ( '===============' )
print ( 'Listing for dpath={}' . format ( dpath ) )
print ( 'E L F D J - path' )
print ( '--------------' )
if not os . path . exists ( dpath ) :
    print ( '... does not exist' )
    return
paths = sorted ( os . listdir ( dpath ) )
for path in paths :
    full_path = join ( dpath , path )
    E = os . path . exists ( full_path )
    L = os . path . islink ( full_path )
    F = os . path . isfile ( full_path )
    D = os . path . isdir ( full_path )
    J = util_platform . WIN32 and _win32_links . _win32_is_junction ( full_path )
    ELFDJ = [ E , L , F , D , J ]
    if ELFDJ == [ 1 , 0 , 0 , 1 , 0 ] : # A directory
        path = util_colors . color_text ( path , 'green' )
    elif ELFDJ == [ 1 , 0 , 1 , 0 , 0 ] : # A file ( or a hard link they are indistinguishable with one query )
        path = util_colors . color_text ( path , 'white' )
    elif ELFDJ == [ 1 , 0 , 0 , 1 , 1 ] : # A directory junction
        path = util_colors . color_text ( path , 'yellow' )
    elif ELFDJ == [ 1 , 1 , 1 , 0 , 0 ] : # A file link
        path = util_colors . color_text ( path , 'turquoise' )
    elif ELFDJ == [ 1 , 1 , 0 , 1 , 0 ] : # A directory link
        path = util_colors . color_text ( path , 'teal' )
    elif ELFDJ == [ 0 , 1 , 0 , 0 , 0 ] : # A broken file link
        path = util_colors . color_text ( path , 'red' )
    elif ELFDJ == [ 0 , 1 , 0 , 1 , 0 ] : # A broken directory link
        path = util_colors . color_text ( path , 'darkred' )
    elif ELFDJ == [ 0 , 0 , 0 , 1 , 1 ] : # A broken directory junction
        path = util_colors . color_text ( path , 'purple' )
    elif ELFDJ == [ 1 , 0 , 1 , 0 , 1 ] : # A file junction ? Thats not good .
    # I guess this is a windows 7 thing ?
        path = util_colors . color_text ( path , 'red' )
    elif ELFDJ == [ 1 , 1 , 0 , 0 , 0 ] : # Windows ? Why ? What does this mean ! ?
    # A directory link that cant be resolved ?
        path = util_colors . color_text ( path , 'red' )
    else :
        print ( 'dpath = {!r}' . format ( dpath ) )
        print ( 'path = {!r}' . format ( path ) )
        raise AssertionError ( str ( ELFDJ ) + str ( path ) )
    line = '{E:d} {L:d} {F:d} {D:d} {J:d} - {path}' . format ( ** locals ( ) )
    if os . path . islink ( full_path ) :
        line += ' -> ' + os . readlink ( full_path )
    elif _win32_links is not None :
        if _win32_links . _win32_is_junction ( full_path ) :
            line += ' => ' + _win32_links . _win32_read_junction ( full_path )
    print ( line )