signature:
def no_error_extract_data_from_file ( self , file_path : str ) -> Iterable [ DataSourceType ] :
    """Proxy for ` extract _ data _ from _ file ` that suppresses any errors and instead just returning an empty list .
 : param file _ path : see ` extract _ data _ from _ file `
 : return : see ` extract _ data _ from _ file `"""

implementation:
try :
    return self . extract_data_from_file ( file_path )
except Exception as e :
    logging . warning ( e )
    return [ ]


signature:
def _load_all_in_directory ( self ) -> Dict [ str , Iterable [ DataSourceType ] ] :
    """Loads all of the data from the files in directory location .
 : return : a origin map of all the loaded data"""

implementation:
origin_mapped_data = dict ( )
# type : Dict [ str , Iterable [ DataSourceType ] ]
for file_path in glob . iglob ( "%s/**/*" % self . _directory_location , recursive = True ) :
    if self . is_data_file ( file_path ) :
        origin_mapped_data [ file_path ] = self . no_error_extract_data_from_file ( file_path )
return origin_mapped_data


signature:
def _extract_data_from_origin_map ( origin_mapped_data : Dict [ str , Iterable [ DataSourceType ] ] ) -> Iterable [ DataSourceType ] :
    """Extracts the data from a data origin map .
 : param origin _ mapped _ data : a map containing the origin of the data as the key string and the data as the value
 : return : the data contained within the map"""

implementation:
data = [ ]
for _ , data_item in origin_mapped_data . items ( ) :
    data . extend ( data_item )
return data


signature:
def start ( self ) :
    """Monitors data kept in files in the predefined directory in a new thread .
 Note : Due to the underlying library , it may take a few milliseconds after this method is started for changes to
 start to being noticed ."""

implementation:
with self . _status_lock :
    if self . _running :
        raise RuntimeError ( "Already running" )
    self . _running = True
# Cannot re - use Observer after stopped
self . _observer = Observer ( )
self . _observer . schedule ( self . _event_handler , self . _directory_location , recursive = True )
self . _observer . start ( )
# Load all in directory afterwards to ensure no undetected changes between loading all and observing
self . _origin_mapped_data = self . _load_all_in_directory ( )


signature:
def stop ( self ) :
    """Stops monitoring the predefined directory ."""

implementation:
with self . _status_lock :
    if self . _running :
        assert self . _observer is not None
        self . _observer . stop ( )
        self . _running = False
        self . _origin_mapped_data = dict ( )


signature:
def _on_file_created ( self , event : FileSystemEvent ) :
    """Called when a file in the monitored directory has been created .
 : param event : the file system event"""

implementation:
if not event . is_directory and self . is_data_file ( event . src_path ) :
    assert event . src_path not in self . _origin_mapped_data
    self . _origin_mapped_data [ event . src_path ] = self . no_error_extract_data_from_file ( event . src_path )
    self . notify_listeners ( FileSystemChange . CREATE )


signature:
def _on_file_modified ( self , event : FileSystemEvent ) :
    """Called when a file in the monitored directory has been modified .
 : param event : the file system event"""

implementation:
if not event . is_directory and self . is_data_file ( event . src_path ) :
    assert event . src_path in self . _origin_mapped_data
    self . _origin_mapped_data [ event . src_path ] = self . no_error_extract_data_from_file ( event . src_path )
    self . notify_listeners ( FileSystemChange . MODIFY )


signature:
def _on_file_deleted ( self , event : FileSystemEvent ) :
    """Called when a file in the monitored directory has been deleted .
 : param event : the file system event"""

implementation:
if not event . is_directory and self . is_data_file ( event . src_path ) :
    assert event . src_path in self . _origin_mapped_data
    del ( self . _origin_mapped_data [ event . src_path ] )
    self . notify_listeners ( FileSystemChange . DELETE )


signature:
def _on_file_moved ( self , event : FileSystemMovedEvent ) :
    """Called when a file in the monitored directory has been moved .
 Breaks move down into a delete and a create ( which it is sometimes detected as ! ) .
 : param event : the file system event"""

implementation:
if not event . is_directory and self . is_data_file ( event . src_path ) :
    delete_event = FileSystemEvent ( event . src_path )
    delete_event . event_type = EVENT_TYPE_DELETED
    self . _on_file_deleted ( delete_event )
    create_event = FileSystemEvent ( event . dest_path )
    create_event . event_type = EVENT_TYPE_CREATED
    self . _on_file_created ( create_event )


signature:
def tear_down ( self ) :
    """Tears down all temp files and directories ."""

implementation:
while len ( self . _temp_directories ) > 0 :
    directory = self . _temp_directories . pop ( )
    shutil . rmtree ( directory , ignore_errors = True )
while len ( self . _temp_files ) > 0 :
    file = self . _temp_files . pop ( )
    try :
        os . remove ( file )
    except OSError :
        pass


signature:
def create_temp_directory ( self , ** mkdtemp_kwargs ) -> str :
    """Creates a temp directory .
 : param mkdtemp _ kwargs : named arguments to be passed to ` tempfile . mkdtemp `
 : return : the location of the temp directory"""

implementation:
kwargs = { ** self . default_mkdtemp_kwargs , ** mkdtemp_kwargs }
location = tempfile . mkdtemp ( ** kwargs )
self . _temp_directories . add ( location )
return location


signature:
def create_temp_file ( self , ** mkstemp_kwargs ) -> Tuple [ int , str ] :
    """Creates a temp file .
 : param mkstemp _ kwargs : named arguments to be passed to ` tempfile . mkstemp `
 : return : tuple where the first element is the file handle and the second is the location of the temp file"""

implementation:
kwargs = { ** self . default_mkstemp_kwargs , ** mkstemp_kwargs }
handle , location = tempfile . mkstemp ( ** kwargs )
self . _temp_files . add ( location )
return handle , location


signature:
def change ( self , new_abspath = None , new_dirpath = None , new_dirname = None , new_basename = None , new_fname = None , new_ext = None ) :
    """Return a new : class : ` pathlib _ mate . pathlib2 . Path ` object with updated information ."""

implementation:
if new_abspath is not None :
    p = self . __class__ ( new_abspath )
    return p
if ( new_dirpath is None ) and ( new_dirname is not None ) :
    new_dirpath = os . path . join ( self . parent . dirpath , new_dirname )
elif ( new_dirpath is not None ) and ( new_dirname is None ) :
    new_dirpath = new_dirpath
elif ( new_dirpath is None ) and ( new_dirname is None ) :
    new_dirpath = self . dirpath
elif ( new_dirpath is not None ) and ( new_dirname is not None ) :
    raise ValueError ( "Cannot having both new_dirpath and new_dirname!" )
if new_basename is None :
    if new_fname is None :
        new_fname = self . fname
    if new_ext is None :
        new_ext = self . ext
    new_basename = new_fname + new_ext
else :
    if new_fname is not None or new_ext is not None :
        raise ValueError ( "Cannot having both new_basename, " "new_fname, new_ext!" )
return self . __class__ ( new_dirpath , new_basename )


signature:
def is_not_exist_or_allow_overwrite ( self , overwrite = False ) :
    """Test whether a file target is not exists or it exists but allow
 overwrite ."""

implementation:
if self . exists ( ) and overwrite is False :
    return False
else : # pragma : no cover
    return True


signature:
def moveto ( self , new_abspath = None , new_dirpath = None , new_dirname = None , new_basename = None , new_fname = None , new_ext = None , overwrite = False , makedirs = False ) :
    """An advanced : meth : ` pathlib _ mate . pathlib2 . Path . rename ` method provide ability to rename by
 each components of a path . A new ` ` Path ` ` instance will returns .
 * * 中文文档 * *
 高级重命名函数 , 允许用于根据路径的各个组成部分进行重命名 。 但和os . rename
 方法一样 , 需要保证母文件夹存在 。"""

implementation:
self . assert_exists ( )
p = self . change ( new_abspath = new_abspath , new_dirpath = new_dirpath , new_dirname = new_dirname , new_basename = new_basename , new_fname = new_fname , new_ext = new_ext , )
if p . is_not_exist_or_allow_overwrite ( overwrite = overwrite ) : # 如果两个路径不同 , 才进行move
    if self . abspath != p . abspath :
        if makedirs :
            parent = p . parent
            if not parent . exists ( ) :
                os . makedirs ( parent . abspath )
        self . rename ( p )
return p


signature:
def copyto ( self , new_abspath = None , new_dirpath = None , new_dirname = None , new_basename = None , new_fname = None , new_ext = None , overwrite = False , makedirs = False ) :
    """Copy this file to other place ."""

implementation:
self . assert_exists ( )
p = self . change ( new_abspath = new_abspath , new_dirpath = new_dirpath , new_dirname = new_dirname , new_basename = new_basename , new_fname = new_fname , new_ext = new_ext , )
if p . is_not_exist_or_allow_overwrite ( overwrite = overwrite ) : # 如果两个路径不同 , 才进行copy
    if self . abspath != p . abspath :
        try :
            shutil . copy ( self . abspath , p . abspath )
        except IOError as e :
            if makedirs :
                os . makedirs ( p . parent . abspath )
                shutil . copy ( self . abspath , p . abspath )
            else :
                raise e
return p


signature:
def _create_client ( base_url : str , tls : TLSConfig = False ) -> Optional [ APIClient ] :
    """Creates a Docker client with the given details .
 : param base _ url : the base URL of the Docker daemon
 : param tls : the Docker daemon ' s TLS config ( if any )
 : return : the created client else None if unable to connect the client to the daemon"""

implementation:
try :
    client = APIClient ( base_url = base_url , tls = tls , version = "auto" )
    return client if client . ping ( ) else None
except :
    return None


signature:
def create_client ( ) -> APIClient :
    """Clients a Docker client .
 Will raise a ` ConnectionError ` if the Docker daemon is not accessible .
 : return : the Docker client"""

implementation:
global _client
client = _client ( )
if client is None : # First try looking at the environment variables for specification of the daemon ' s location
    docker_environment = kwargs_from_env ( assert_hostname = False )
    if "base_url" in docker_environment :
        client = _create_client ( docker_environment . get ( "base_url" ) , docker_environment . get ( "tls" ) )
        if client is None :
            raise ConnectionError ( "Could not connect to the Docker daemon specified by the `DOCKER_X` environment variables: %s" % docker_environment )
        else :
            logging . info ( "Connected to Docker daemon specified by the environment variables" )
    else : # Let ' s see if the Docker daemon is accessible via the UNIX socket
        client = _create_client ( "unix://var/run/docker.sock" )
        if client is not None :
            logging . info ( "Connected to Docker daemon running on UNIX socket" )
        else :
            raise ConnectionError ( "Cannot connect to Docker - is the Docker daemon running? `$DOCKER_HOST` should be set or the " "daemon should be accessible via the standard UNIX socket." )
    _client = weakref . ref ( client )
assert isinstance ( client , APIClient )
return client


signature:
def get_dump_method ( dump , protocol = - 1 ) :
    """Get dump function code string"""

implementation:
if dump is None :
    dump = 'pickle'
if dump . startswith ( 'pickle' ) :
    if dump == 'pickle' :
        proto = protocol
    else :
        proto = dump . strip ( 'pickle' )
    try :
        proto = int ( proto )
        assert proto >= - 1
    except :
        raise Exception ( "protocol must be an integer >=-1" )
    code = """
try:
    import cPickle as pickle
except:
    import pickle
with open('$FILE_PATH', 'wb') as fd:
    pickle.dump( value, fd, protocol=%i )
    fd.flush()
    os.fsync(fd.fileno())
""" % proto
elif dump . startswith ( 'dill' ) :
    if dump == 'dill' :
        proto = 2
    else :
        proto = dump . strip ( 'dill' )
        try :
            proto = int ( proto )
            assert proto >= - 1
        except :
            raise Exception ( "protocol must be an integer >=-1" )
    code = """
import dill
with open('$FILE_PATH', 'wb') as fd:
    dill.dump( value, fd, protocol=%i )
    fd.flush()
    os.fsync(fd.fileno())
""" % proto
elif dump == 'json' :
    code = """
import json
with open('$FILE_PATH', 'wb') as fd:
    json.dump( value,fd, ensure_ascii=True, indent=4 )
    fd.flush()
    os.fsync(fd.fileno())
"""
elif dump == 'numpy' :
    code = """
import numpy
with open('$FILE_PATH', 'wb') as fd:
    numpy.save(file=fd, arr=value)
    fd.flush()
    os.fsync(fd.fileno())
"""
elif dump == 'numpy_text' :
    code = """
import numpy
numpy.savetxt(fname='$FILE_PATH', X=value, fmt='%.6e')
"""
else :
    assert isinstance ( dump , basestring ) , "dump must be None or a string"
    assert '$FILE_PATH' in dump , "string dump code must inlcude '$FILE_PATH'"
    code = dump
# return
return code


signature:
def get_pull_method ( pull ) :
    """Get pull function code string"""

implementation:
if pull is None or pull . startswith ( 'pickle' ) :
    code = """
import os
try:
    import cPickle as pickle
except:
    import pickle
with open('$FILE_PATH', 'rb') as fd:
    PULLED_DATA = pickle.load( fd )
"""
elif pull . startswith ( 'dill' ) :
    code = """
import dill
with open('$FILE_PATH', 'rb') as fd:
    PULLED_DATA = dill.load( fd )
"""
elif pull == 'json' :
    code = """
import json
with open('$FILE_PATH', 'rb') as fd:
    PULLED_DATA = json.load(fd)
"""
elif pull == 'numpy' :
    code = """
import numpy
with open('$FILE_PATH', 'rb') as fd:
    PULLED_DATA=numpy.load(file=fd)

"""
elif pull == 'numpy_text' :
    code = """
import numpy
with open('$FILE_PATH', 'rb') as fd:
    PULLED_DATA=numpy.loadtxt(fname=fd)
"""
else :
    assert isinstance ( pull , basestring ) , "pull must be None or a string"
    assert 'PULLED_DATA' in pull , "string pull code must inlcude 'PULLED_DATA'"
    assert '$FILE_PATH' in pull , "string pull code must inlcude '$FILE_PATH'"
    code = pull
# return
return code


signature:
def path_required ( func ) :
    """Decorate methods when repository path is required ."""

implementation:
@ wraps ( func )
def wrapper ( self , * args , ** kwargs ) :
    if self . path is None :
        warnings . warn ( 'Must load (Repository.load_repository) or initialize (Repository.create_repository) the repository first !' )
        return
    return func ( self , * args , ** kwargs )
return wrapper


signature:
def __clean_before_after ( self , stateBefore , stateAfter , keepNoneEmptyDirectory = True ) :
    """clean repository given before and after states"""

implementation:
# prepare after for faster search
errors = [ ]
afterDict = { }
[ afterDict . setdefault ( list ( aitem ) [ 0 ] , [ ] ) . append ( aitem ) for aitem in stateAfter ]
# loop before
for bitem in reversed ( stateBefore ) :
    relaPath = list ( bitem ) [ 0 ]
    basename = os . path . basename ( relaPath )
    btype = bitem [ relaPath ] [ 'type' ]
    alist = afterDict . get ( relaPath , [ ] )
    aitem = [ a for a in alist if a [ relaPath ] [ 'type' ] == btype ]
    if len ( aitem ) > 1 :
        errors . append ( "Multiple '%s' of type '%s' where found in '%s', this should never had happened. Please report issue" % ( basename , btype , relaPath ) )
        continue
    if not len ( aitem ) :
        removeDirs = [ ]
        removeFiles = [ ]
        if btype == 'dir' :
            if not len ( relaPath ) :
                errors . append ( "Removing main repository directory is not allowed" )
                continue
            removeDirs . append ( os . path . join ( self . __path , relaPath ) )
            removeFiles . append ( os . path . join ( self . __path , relaPath , self . __dirInfo ) )
            removeFiles . append ( os . path . join ( self . __path , relaPath , self . __dirLock ) )
        elif btype == 'file' :
            removeFiles . append ( os . path . join ( self . __path , relaPath ) )
            removeFiles . append ( os . path . join ( self . __path , relaPath , self . __fileInfo % basename ) )
            removeFiles . append ( os . path . join ( self . __path , relaPath , self . __fileLock % basename ) )
        else : # # # MUST VERIFY THAT ONCE pyrepobjectdir IS IMPLEMENTED
            removeDirs . append ( os . path . join ( self . __path , relaPath ) )
            removeFiles . append ( os . path . join ( self . __path , relaPath , self . __fileInfo % basename ) )
        # remove files
        for fpath in removeFiles :
            if os . path . isfile ( fpath ) :
                try :
                    os . remove ( fpath )
                except Exception as err :
                    errors . append ( "Unable to clean file '%s' (%s)" % ( fpath , str ( err ) ) )
        # remove directories
        for dpath in removeDirs :
            if os . path . isdir ( dpath ) :
                if keepNoneEmptyDirectory or not len ( os . listdir ( dpath ) ) :
                    try :
                        shutil . rmtree ( dpath )
                    except Exception as err :
                        errors . append ( "Unable to clean directory '%s' (%s)" % ( fpath , str ( err ) ) )
# return result and errors list
return len ( errors ) == 0 , errors


signature:
def get_stats ( self ) :
    """Get repository descriptive stats
 : Returns :
 # . numberOfDirectories ( integer ) : Number of diretories in repository
 # . numberOfFiles ( integer ) : Number of files in repository"""

implementation:
if self . __path is None :
    return 0 , 0
nfiles = 0
ndirs = 0
for fdict in self . get_repository_state ( ) :
    fdname = list ( fdict ) [ 0 ]
    if fdname == '' :
        continue
    if fdict [ fdname ] . get ( 'pyrepfileinfo' , False ) :
        nfiles += 1
    elif fdict [ fdname ] . get ( 'pyrepdirinfo' , False ) :
        ndirs += 1
    else :
        raise Exception ( 'Not sure what to do next. Please report issue' )
return ndirs , nfiles


signature:
def reset ( self ) :
    """Reset repository instance ."""

implementation:
self . __path = None
self . __repo = { 'repository_unique_name' : str ( uuid . uuid1 ( ) ) , 'create_utctime' : time . time ( ) , 'last_update_utctime' : None , 'pyrep_version' : str ( __version__ ) , 'repository_information' : '' , 'walk_repo' : [ ] }


signature:
def is_repository ( self , path ) :
    """Check if there is a Repository in path .
 : Parameters :
 # . path ( string ) : The real path of the directory where to check if
 there is a repository .
 : Returns :
 # . result ( boolean ) : Whether it ' s a repository or not ."""

implementation:
if path . strip ( ) in ( '' , '.' ) :
    path = os . getcwd ( )
repoPath = os . path . realpath ( os . path . expanduser ( path ) )
if os . path . isfile ( os . path . join ( repoPath , self . __repoFile ) ) :
    return True
else :
    try :
        from . OldRepository import Repository
        REP = Repository ( )
        result = REP . is_repository ( repoPath )
    except :
        return False
    else :
        if result :
            warnings . warn ( "This is an old repository version 2.x.y! Make sure to start using repositories 3.x.y " )
        return result


signature:
def load_repository ( self , path , verbose = True , ntrials = 3 ) :
    """Load repository from a directory path and update the current instance .
 First , new repository still will be loaded . If failed , then old
 style repository load will be tried .
 : Parameters :
 # . path ( string ) : The path of the directory from where to load
 the repository from . If ' . ' or an empty string is passed ,
 the current working directory will be used .
 # . verbose ( boolean ) : Whether to be verbose about abnormalities
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . repository ( pyrep . Repository ) : returns self repository with loaded data ."""

implementation:
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
repo = None
for _trial in range ( ntrials ) :
    try :
        self . __load_repository ( path = path , verbose = True )
    except Exception as err1 :
        try :
            from . OldRepository import Repository
            REP = Repository ( path )
        except Exception as err2 : # traceback . print _ exc ( )
            error = "Unable to load repository using neiher new style (%s) nor old style (%s)" % ( err1 , err2 )
            if self . DEBUG_PRINT_FAILED_TRIALS :
                print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
        else :
            error = None
            repo = REP
            break
    else :
        error = None
        repo = self
        break
# check and return
assert error is None , error
return repo


signature:
def create_repository ( self , path , info = None , description = None , replace = True , allowNoneEmpty = True , raiseError = True ) :
    """create a repository in a directory . This method insures the creation of
 the directory in the system if it is missing . \n
 * * N . B . If replace is True and existing repository is found in path , create _ repository erases all existing files and directories in path . * *
 : Parameters :
 # . path ( string ) : The real absolute path where to create the Repository .
 If ' . ' or an empty string is passed , the current working directory will be used .
 # . description ( None , str ) : Repository main directory information .
 # . info ( None , object ) : Repository information . It can
 be None or any pickle writable type of data .
 # . replace ( boolean ) : Whether to replace existing repository .
 # . allowNoneEmpty ( boolean ) : Allow creating repository in none - empty
 directory .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 : Returns :
 # . success ( boolean ) : Whether creating repository was successful
 # . message ( None , str ) : Any returned message ."""

implementation:
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( allowNoneEmpty , bool ) , "allowNoneEmpty must be boolean"
assert isinstance ( replace , bool ) , "replace must be boolean"
assert isinstance ( path , basestring ) , "path must be string"
if info is None :
    info = ''
try :
    pickle . dumps ( info )
except Exception as err :
    raise Exception ( "info must be None or any pickle writable type of data (%s)" % str ( err ) )
# assert isinstance ( info , basestring ) , " info must be None or a string "
if description is None :
    description = ''
assert isinstance ( description , basestring ) , "description must be None or a string"
# get real path
if path . strip ( ) in ( '' , '.' ) :
    path = os . getcwd ( )
realPath = os . path . realpath ( os . path . expanduser ( path ) )
# reset if replace is set to True
message = [ ]
if self . is_repository ( realPath ) :
    if not replace :
        message . append ( "A pyrep Repository already exists in the given path '%s' set replace to True if you need to proceed." % path )
        return False , message
    else :
        message . append ( "Old existing pyrep repository existing in the given path '%s' has been replaced." % path )
        try :
            for _df in os . listdir ( realPath ) :
                _p = os . path . join ( realPath , _df )
                if os . path . isdir ( _p ) :
                    shutil . rmtree ( _p )
                else :
                    os . remove ( _p )
        except Exception as err :
            message . append ( "Unable to clean remove repository before create (%s)" % ( str ( err ) ) )
            return False , '\n' . join ( message )
if not os . path . isdir ( realPath ) :
    os . makedirs ( realPath )
elif len ( os . listdir ( realPath ) ) and not allowNoneEmpty :
    return False , "Not allowed to create repository in a non empty directory"
# reset repository
oldRepo = self . __repo
self . reset ( )
self . __path = realPath . rstrip ( os . sep )
self . __repo [ 'repository_information' ] = info
# save repository
saved = self . save ( description = description )
if not saved :
    self . __repo = oldRepo
    message . append ( "Absolute path and directories might be created but no pyrep Repository is created." )
    return False , '\n' . join ( message )
# return
return True , '\n' . join ( message )


signature:
def remove_repository ( self , path = None , removeEmptyDirs = True ) :
    """Remove all repository from path along with all repository tracked files .
 : Parameters :
 # . path ( None , string ) : The path the repository to remove .
 # . removeEmptyDirs ( boolean ) : Whether to remove remaining empty
 directories ."""

implementation:
assert isinstance ( removeEmptyDirs , bool ) , "removeEmptyDirs must be boolean"
if path is not None :
    if path != self . __path :
        repo = Repository ( )
        repo . load_repository ( path )
    else :
        repo = self
else :
    repo = self
assert repo . path is not None , "path is not given and repository is not initialized"
# remove repo files and directories
for fdict in reversed ( repo . get_repository_state ( ) ) :
    relaPath = list ( fdict ) [ 0 ]
    realPath = os . path . join ( repo . path , relaPath )
    path , name = os . path . split ( realPath )
    if fdict [ relaPath ] [ 'type' ] == 'file' :
        if os . path . isfile ( realPath ) :
            os . remove ( realPath )
        if os . path . isfile ( os . path . join ( repo . path , path , self . __fileInfo % name ) ) :
            os . remove ( os . path . join ( repo . path , path , self . __fileInfo % name ) )
        if os . path . isfile ( os . path . join ( repo . path , path , self . __fileLock % name ) ) :
            os . remove ( os . path . join ( repo . path , path , self . __fileLock % name ) )
        if os . path . isfile ( os . path . join ( repo . path , path , self . __fileClass % name ) ) :
            os . remove ( os . path . join ( repo . path , path , self . __fileClass % name ) )
    elif fdict [ relaPath ] [ 'type' ] == 'dir' :
        if os . path . isfile ( os . path . join ( realPath , self . __dirInfo ) ) :
            os . remove ( os . path . join ( realPath , self . __dirInfo ) )
        if os . path . isfile ( os . path . join ( realPath , self . __dirLock ) ) :
            os . remove ( os . path . join ( realPath , self . __dirLock ) )
        if not len ( os . listdir ( realPath ) ) and removeEmptyDirs :
            shutil . rmtree ( realPath )
# remove repo information file
if os . path . isfile ( os . path . join ( repo . path , self . __repoFile ) ) :
    os . remove ( os . path . join ( repo . path , self . __repoFile ) )
if os . path . isfile ( os . path . join ( repo . path , self . __repoLock ) ) :
    os . remove ( os . path . join ( repo . path , self . __repoLock ) )


signature:
def save ( self , description = None , raiseError = True , ntrials = 3 ) :
    """Save repository ' . pyreprepo ' to disk and create ( if missing ) or
 update ( if description is not None ) ' . pyrepdirinfo ' .
 : Parameters :
 # . description ( None , str ) : Repository main directory information .
 If given will be replaced .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( bool ) : Whether saving was successful .
 # . error ( None , string ) : Fail to save repository message in case
 saving is not successful . If success is True , error will be None ."""

implementation:
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
# get description
if description is not None :
    assert isinstance ( description , basestring ) , "description must be None or a string"
dirInfoPath = os . path . join ( self . __path , self . __dirInfo )
if description is None and not os . path . isfile ( dirInfoPath ) :
    description = ''
# create and acquire lock
LR = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , self . __repoLock ) )
acquired , code = LR . acquire_lock ( )
# check if acquired .
m = "code %s. Unable to aquire the lock when calling 'save'. You may try again!" % ( code , )
if not acquired :
    assert not raiseError , Exception ( m )
    return False , m
# save repository
for _trial in range ( ntrials ) :
    try : # open file
        repoInfoPath = os . path . join ( self . __path , self . __repoFile )
        error = None
        self . __save_dirinfo ( description = description , dirInfoPath = dirInfoPath )
        # load and update repository info if existing
        if os . path . isfile ( repoInfoPath ) :
            with open ( repoInfoPath , 'rb' ) as fd :
                repo = self . __load_repository_pickle_file ( os . path . join ( self . __path , self . __repoFile ) )
                self . __repo [ 'walk_repo' ] = repo [ 'walk_repo' ]
        # create repository
        with open ( repoInfoPath , 'wb' ) as fd :
            self . __repo [ "last_update_utctime" ] = time . time ( )
            pickle . dump ( self . __repo , fd , protocol = self . _DEFAULT_PICKLE_PROTOCOL )
            fd . flush ( )
            os . fsync ( fd . fileno ( ) )
    except Exception as err :
        error = "Unable to save repository (%s)" % err
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        break
# release lock
LR . release_lock ( )
# return
assert error is None or not raiseError , error
return error is None , error


signature:
def is_name_allowed ( self , path ) :
    """Get whether creating a file or a directory from the basenane of the given
 path is allowed
 : Parameters :
 # . path ( str ) : The absolute or relative path or simply the file
 or directory name .
 : Returns :
 # . allowed ( bool ) : Whether name is allowed .
 # . message ( None , str ) : Reason for the name to be forbidden ."""

implementation:
assert isinstance ( path , basestring ) , "given path must be a string"
name = os . path . basename ( path )
if not len ( name ) :
    return False , "empty name is not allowed"
# exact match
for em in [ self . __repoLock , self . __repoFile , self . __dirInfo , self . __dirLock ] :
    if name == em :
        return False , "name '%s' is reserved for pyrep internal usage" % em
# pattern match
for pm in [ self . __fileInfo , self . __fileLock ] : # , self . _ _ objectDir ] :
    if name == pm or ( name . endswith ( pm [ 3 : ] ) and name . startswith ( '.' ) ) :
        return False , "name pattern '%s' is not allowed as result may be reserved for pyrep internal usage" % pm
# name is ok
return True , None


signature:
def to_repo_relative_path ( self , path , split = False ) :
    """Given a path , return relative path to diretory
 : Parameters :
 # . path ( str ) : Path as a string
 # . split ( boolean ) : Whether to split path to its components
 : Returns :
 # . relativePath ( str , list ) : Relative path as a string or as a list
 of components if split is True"""

implementation:
path = os . path . normpath ( path )
if path == '.' :
    path = ''
path = path . split ( self . __path ) [ - 1 ] . strip ( os . sep )
if split :
    return path . split ( os . sep )
else :
    return path


signature:
def get_repository_state ( self , relaPath = None ) :
    """Get a list representation of repository state along with useful
 information . List state is ordered relativeley to directories level
 : Parameters :
 # . relaPath ( None , str ) : relative directory path from where to
 start . If None all repository representation is returned .
 : Returns :
 # . state ( list ) : List representation of the repository .
 List items are all dictionaries . Every dictionary has a single
 key which is the file or the directory name and the value is a
 dictionary of information including :
 * ' type ' : the type of the tracked whether it ' s file , dir , or objectdir
 * ' exists ' : whether file or directory actually exists on disk
 * ' pyrepfileinfo ' : In case of a file or an objectdir whether . % s _ pyrepfileinfo exists
 * ' pyrepdirinfo ' : In case of a directory whether . pyrepdirinfo exists"""

implementation:
state = [ ]
def _walk_dir ( relaPath , dirList ) :
    dirDict = { 'type' : 'dir' , 'exists' : os . path . isdir ( os . path . join ( self . __path , relaPath ) ) , 'pyrepdirinfo' : os . path . isfile ( os . path . join ( self . __path , relaPath , self . __dirInfo ) ) , }
    state . append ( { relaPath : dirDict } )
    # loop files and dirobjects
    for fname in sorted ( [ f for f in dirList if isinstance ( f , basestring ) ] ) :
        relaFilePath = os . path . join ( relaPath , fname )
        realFilePath = os . path . join ( self . __path , relaFilePath )
        # if os . path . isdir ( realFilePath ) and df . startswith ( ' . ' ) and df . endswith ( self . _ _ objectDir [ 3 : ] ) :
        # fileDict = { ' type ' : ' objectdir ' ,
        # ' exists ' : True ,
        # ' pyrepfileinfo ' : os . path . isfile ( os . path . join ( self . _ _ path , relaPath , self . _ _ fileInfo % fname ) ) ,
        # else :
        # fileDict = { ' type ' : ' file ' ,
        # ' exists ' : os . path . isfile ( realFilePath ) ,
        # ' pyrepfileinfo ' : os . path . isfile ( os . path . join ( self . _ _ path , relaPath , self . _ _ fileInfo % fname ) ) ,
        fileDict = { 'type' : 'file' , 'exists' : os . path . isfile ( realFilePath ) , 'pyrepfileinfo' : os . path . isfile ( os . path . join ( self . __path , relaPath , self . __fileInfo % fname ) ) , }
        state . append ( { relaFilePath : fileDict } )
    # loop directories
    # for ddict in sorted ( [ d for d in dirList if isinstance ( d , dict ) and len ( d ) ] , key = lambda k : list ( k ) [ 0 ] ) :
    for ddict in sorted ( [ d for d in dirList if isinstance ( d , dict ) ] , key = lambda k : list ( k ) [ 0 ] ) :
        dirname = list ( ddict ) [ 0 ]
        _walk_dir ( relaPath = os . path . join ( relaPath , dirname ) , dirList = ddict [ dirname ] )
# call recursive _ walk _ dir
if relaPath is None :
    _walk_dir ( relaPath = '' , dirList = self . __repo [ 'walk_repo' ] )
else :
    assert isinstance ( relaPath , basestring ) , "relaPath must be None or a str"
    relaPath = self . to_repo_relative_path ( path = relaPath , split = False )
    spath = relaPath . split ( os . sep )
    dirList = self . __repo [ 'walk_repo' ]
    while len ( spath ) :
        dirname = spath . pop ( 0 )
        dList = [ d for d in dirList if isinstance ( d , dict ) ]
        if not len ( dList ) :
            dirList = None
            break
        cDict = [ d for d in dList if dirname in d ]
        if not len ( cDict ) :
            dirList = None
            break
        dirList = cDict [ 0 ] [ dirname ]
    if dirList is not None :
        _walk_dir ( relaPath = relaPath , dirList = dirList )
# return state list
return state


signature:
def get_file_info ( self , relativePath ) :
    """Get file information dict from the repository given its relative path .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of
 the file .
 : Returns :
 # . info ( None , dictionary ) : The file information dictionary .
 If None , it means an error has occurred .
 # . errorMessage ( string ) : The error message if any error occurred ."""

implementation:
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
fileName = os . path . basename ( relativePath )
isRepoFile , fileOnDisk , infoOnDisk , classOnDisk = self . is_repository_file ( relativePath )
if not isRepoFile :
    return None , "file is not a registered repository file."
if not infoOnDisk :
    return None , "file is a registered repository file but info file missing"
fileInfoPath = os . path . join ( self . __path , os . path . dirname ( relativePath ) , self . __fileInfo % fileName )
try :
    with open ( fileInfoPath , 'rb' ) as fd :
        info = pickle . load ( fd )
except Exception as err :
    return None , "Unable to read file info from disk (%s)" % str ( err )
return info , ''


signature:
def is_repository_file ( self , relativePath ) :
    """Check whether a given relative path is a repository file path
 : Parameters :
 # . relativePath ( string ) : File relative path
 : Returns :
 # . isRepoFile ( boolean ) : Whether file is a repository file .
 # . isFileOnDisk ( boolean ) : Whether file is found on disk .
 # . isFileInfoOnDisk ( boolean ) : Whether file info is found on disk .
 # . isFileClassOnDisk ( boolean ) : Whether file class is found on disk ."""

implementation:
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
if relativePath == '' :
    return False , False , False , False
relaDir , name = os . path . split ( relativePath )
fileOnDisk = os . path . isfile ( os . path . join ( self . __path , relativePath ) )
infoOnDisk = os . path . isfile ( os . path . join ( self . __path , os . path . dirname ( relativePath ) , self . __fileInfo % name ) )
classOnDisk = os . path . isfile ( os . path . join ( self . __path , os . path . dirname ( relativePath ) , self . __fileClass % name ) )
cDir = self . __repo [ 'walk_repo' ]
if len ( relaDir ) :
    for dirname in relaDir . split ( os . sep ) :
        dList = [ d for d in cDir if isinstance ( d , dict ) ]
        if not len ( dList ) :
            cDir = None
            break
        cDict = [ d for d in dList if dirname in d ]
        if not len ( cDict ) :
            cDir = None
            break
        cDir = cDict [ 0 ] [ dirname ]
if cDir is None :
    return False , fileOnDisk , infoOnDisk , classOnDisk
# if name not in cDir :
if str ( name ) not in [ str ( i ) for i in cDir ] :
    return False , fileOnDisk , infoOnDisk , classOnDisk
# this is a repository registered file . check whether all is on disk
return True , fileOnDisk , infoOnDisk , classOnDisk


signature:
def walk_files_path ( self , relativePath = "" , fullPath = False , recursive = False ) :
    """Walk the repository relative path and yield file relative / full path .
 : parameters :
 # . relativePath ( string ) : The relative path from which start the walk .
 # . fullPath ( boolean ) : Whether to return full or relative path .
 # . recursive ( boolean ) : Whether walk all directories files recursively"""

implementation:
assert isinstance ( fullPath , bool ) , "fullPath must be boolean"
assert isinstance ( recursive , bool ) , "recursive must be boolean"
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
dirList = self . __get_repository_directory ( relativePath = relativePath )
assert dirList is not None , "given relative path '%s' is not a repository directory" % relativePath
# walk recursive function
def _walk ( rpath , dlist , recursive ) : # walk files
    for fname in dlist :
        if isinstance ( fname , basestring ) :
            if fullPath :
                yield os . path . join ( self . __path , rpath , fname )
            else :
                yield os . path . join ( rpath , fname )
    if recursive :
        for ddict in dlist :
            if isinstance ( ddict , dict ) :
                dname = list ( ddict ) [ 0 ]
                for p in _walk ( rpath = os . path . join ( rpath , dname ) , dlist = ddict [ dname ] , recursive = recursive ) :
                    yield p
# walk all files
return _walk ( rpath = relativePath , dlist = dirList , recursive = recursive )


signature:
def walk_files_info ( self , relativePath = "" , fullPath = False , recursive = False ) :
    """Walk the repository relative path and yield tuple of two items where
 first item is file relative / full path and second item is file info .
 If file info is not found on disk , second item will be None .
 : parameters :
 # . relativePath ( string ) : The relative path from which start the walk .
 # . fullPath ( boolean ) : Whether to return full or relative path .
 # . recursive ( boolean ) : Whether walk all directories files recursively"""

implementation:
assert isinstance ( fullPath , bool ) , "fullPath must be boolean"
assert isinstance ( recursive , bool ) , "recursive must be boolean"
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
for relaPath in self . walk_files_path ( relativePath = relativePath , fullPath = False , recursive = recursive ) :
    fpath , fname = os . path . split ( relaPath )
    fileInfoPath = os . path . join ( self . __path , fpath , self . __fileInfo % fname )
    if os . path . isfile ( fileInfoPath ) :
        with open ( fileInfoPath , 'rb' ) as fd :
            info = pickle . load ( fd )
    else :
        info = None
    if fullPath :
        yield ( os . path . join ( self . __path , relaPath ) , info )
    else :
        yield ( relaPath , info )


signature:
def walk_directories_info ( self , relativePath = "" , fullPath = False , recursive = False ) :
    """Walk the repository relative path and yield tuple of two items where
 first item is directory relative / full path and second item is directory
 info . If directory file info is not found on disk , second item will be None .
 : parameters :
 # . relativePath ( string ) : The relative path from which start the walk .
 # . fullPath ( boolean ) : Whether to return full or relative path .
 # . recursive ( boolean ) : Whether walk all directories files recursively ."""

implementation:
assert isinstance ( fullPath , bool ) , "fullPath must be boolean"
assert isinstance ( recursive , bool ) , "recursive must be boolean"
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
# walk directories
for dpath in self . walk_directories_path ( relativePath = relativePath , fullPath = False , recursive = recursive ) :
    dirInfoPath = os . path . join ( self . __path , dpath , self . __dirInfo )
    if os . path . isfile ( dirInfoPath ) :
        with open ( dirInfoPath , 'rb' ) as fd :
            info = pickle . load ( fd )
    else :
        info = None
    if fullPath :
        yield ( os . path . join ( self . __path , dpath ) , info )
    else :
        yield ( dpath , info )


signature:
def create_package ( self , path = None , name = None , mode = None ) :
    """Create a tar file package of all the repository files and directories .
 Only files and directories that are tracked in the repository
 are stored in the package tar file .
 * * N . B . On some systems packaging requires root permissions . * *
 : Parameters :
 # . path ( None , string ) : The real absolute path where to create the
 package . If None , it will be created in the same directory as
 the repository . If ' . ' or an empty string is passed , the current
 working directory will be used .
 # . name ( None , string ) : The name to give to the package file
 If None , the package directory name will be used with the
 appropriate extension added .
 # . mode ( None , string ) : The writing mode of the tarfile .
 If None , automatically the best compression mode will be chose .
 Available modes are ( ' w ' , ' w : ' , ' w : gz ' , ' w : bz2 ' )"""

implementation:
# check mode
assert mode in ( None , 'w' , 'w:' , 'w:gz' , 'w:bz2' ) , 'unkown archive mode %s' % str ( mode )
if mode is None : # mode = ' w : bz2'
    mode = 'w:'
# get root
if path is None :
    root = os . path . split ( self . __path ) [ 0 ]
elif path . strip ( ) in ( '' , '.' ) :
    root = os . getcwd ( )
else :
    root = os . path . realpath ( os . path . expanduser ( path ) )
assert os . path . isdir ( root ) , 'absolute path %s is not a valid directory' % path
# get name
if name is None :
    ext = mode . split ( ":" )
    if len ( ext ) == 2 :
        if len ( ext [ 1 ] ) :
            ext = "." + ext [ 1 ]
        else :
            ext = '.tar'
    else :
        ext = '.tar'
    name = os . path . split ( self . __path ) [ 1 ] + ext
# create tar file
tarfilePath = os . path . join ( root , name )
try :
    tarHandler = tarfile . TarFile . open ( tarfilePath , mode = mode )
except Exception as e :
    raise Exception ( "Unable to create package (%s)" % e )
# walk directory and create empty directories
for dpath in sorted ( list ( self . walk_directories_path ( recursive = True ) ) ) :
    t = tarfile . TarInfo ( dpath )
    t . type = tarfile . DIRTYPE
    tarHandler . addfile ( t )
    tarHandler . add ( os . path . join ( self . __path , dpath , self . __dirInfo ) , arcname = self . __dirInfo )
# walk files and add to tar
for fpath in self . walk_files_path ( recursive = True ) :
    relaPath , fname = os . path . split ( fpath )
    tarHandler . add ( os . path . join ( self . __path , fpath ) , arcname = fname )
    tarHandler . add ( os . path . join ( self . __path , relaPath , self . __fileInfo % fname ) , arcname = self . __fileInfo % fname )
    tarHandler . add ( os . path . join ( self . __path , relaPath , self . __fileClass % fname ) , arcname = self . __fileClass % fname )
# save repository . pyrepinfo
tarHandler . add ( os . path . join ( self . __path , self . __repoFile ) , arcname = ".pyrepinfo" )
# close tar file
tarHandler . close ( )


signature:
def add_directory ( self , relativePath , description = None , clean = False , raiseError = True , ntrials = 3 ) :
    """Add a directory in the repository and creates its attribute in the
 Repository with utc timestamp . It insures adding all the missing
 directories in the path .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path to
 where directory must be added .
 # . description ( None , string ) : Any random description about the
 added directory .
 # . clean ( boolean ) : Whether to remove existing non repository
 tracked files and folders in all created directory chain tree .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( boolean ) : Whether adding the directory was successful .
 # . message ( None , string ) : Reason why directory was not added or
 random information ."""

implementation:
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( relativePath , basestring ) , "relativePath must be a string"
if description is not None :
    assert isinstance ( description , basestring ) , "description must be None or a string"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
# normalise path
path = self . to_repo_relative_path ( path = relativePath , split = False )
# whether to replace
if self . is_repository_directory ( path ) :
    return True , "Directory is already tracked in repository"
# check whether name is allowed
allowed , reason = self . is_name_allowed ( path )
if not allowed :
    if raiseError :
        raise Exception ( reason )
    return False , reason
# lock repository and get _ _ repo updated from disk
LR = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , self . __repoLock ) )
acquired , code = LR . acquire_lock ( )
if not acquired :
    m = "code %s. Unable to aquire the lock to add directory. You may try again!" % ( code , )
    if raiseError :
        raise Exception ( m )
    return False , m
# load repository info
for _trial in range ( ntrials ) :
    try :
        repo = self . __load_repository_pickle_file ( os . path . join ( self . __path , self . __repoFile ) )
        self . __repo [ 'walk_repo' ] = repo [ 'walk_repo' ]
    except Exception as err :
        error = str ( err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        error = None
        break
if error is not None :
    _ = LR . release_lock ( )
    assert not raiseError , Exception ( error )
    return False , error
# create directories
error = None
posList = self . __repo [ 'walk_repo' ]
dirPath = self . __path
spath = path . split ( os . sep )
for idx , name in enumerate ( spath ) : # create and acquire lock .
    LD = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( dirPath , self . __dirLock ) )
    acquired , code = LD . acquire_lock ( )
    if not acquired :
        error = "Code %s. Unable to aquire the lock when adding '%s'. All prior directories were added. You may try again, to finish adding directory" % ( code , dirPath )
        break
    # add to directory
    for _trial in range ( ntrials ) :
        try :
            dirPath = os . path . join ( dirPath , name )
            riPath = os . path . join ( dirPath , self . __dirInfo )
            dList = [ d for d in posList if isinstance ( d , dict ) ]
            dList = [ d for d in dList if name in d ]
            # clean directory
            if not len ( dList ) and clean and os . path . exists ( dirPath ) :
                try :
                    shutil . rmtree ( dirPath , ignore_errors = True )
                except Exception as err :
                    error = "Unable to clean directory '%s' (%s)" % ( dirPath , err )
                    break
            # create directory
            if not os . path . exists ( dirPath ) :
                try :
                    os . mkdir ( dirPath )
                except Exception as err :
                    error = "Unable to create directory '%s' (%s)" % ( dirPath , err )
                    break
            # create and dump dirinfo
            self . __save_dirinfo ( description = [ None , description ] [ idx == len ( spath ) - 1 ] , dirInfoPath = riPath , create = True )
            # update directory list
            if not len ( dList ) :
                rsd = { name : [ ] }
                posList . append ( rsd )
                posList = rsd [ name ]
            else :
                assert len ( dList ) == 1 , "Same directory name dict is found twice. This should'n have happened. Report issue"
                posList = dList [ 0 ] [ name ]
        except Exception as err :
            LD . release_lock ( )
            error = "Unable to create directory '%s' info file (%s)" % ( dirPath , str ( err ) )
            if self . DEBUG_PRINT_FAILED_TRIALS :
                print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
        else :
            LD . release_lock ( )
            break
    if error is not None :
        break
# save _ _ repo
if error is None :
    try :
        _ , error = self . __save_repository_pickle_file ( lockFirst = False , raiseError = False )
    except Exception as err :
        error = str ( err )
        pass
try :
    LD . release_lock ( )
except :
    pass
try :
    LR . release_lock ( )
except :
    pass
# check and return
assert error is None or not raiseError , error
return error is None , error


signature:
def remove_directory ( self , relativePath , clean = False , raiseError = True , ntrials = 3 ) :
    """Remove directory from repository tracking .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the
 directory to remove from the repository .
 # . clean ( boolean ) : Whether to os remove directory . If False only
 tracked files will be removed along with left empty directories .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( boolean ) : Whether removing the directory was successful .
 # . reason ( None , string ) : Reason why directory was not removed ."""

implementation:
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( clean , bool ) , "clean must be boolean"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
# normalise path
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
parentPath , dirName = os . path . split ( relativePath )
# check if this is main repository directory
if relativePath == '' :
    return False , "Removing main repository directory is not allowed"
# check if this is a repository directory
if not self . is_repository_directory ( relativePath ) :
    return False , "Given relative path '%s' is not a repository path" % relativePath
# check if directory actually exists on disk
realPath = os . path . join ( self . __path , relativePath )
if not os . path . isdir ( realPath ) :
    error = "Repository relative directory '%s' seems to be missing. call maintain_repository to fix all issues"
    assert not raiseError , error
    return False , error
# get and acquire lock
LD = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , parentPath , self . __dirLock ) )
acquired , code = LD . acquire_lock ( )
if not acquired :
    error = "Code %s. Unable to aquire the lock when adding '%s'. All prior directories were added. You may try again, to finish adding directory" % ( code , realPath )
    assert not raiseError , error
    return False , error
# lock repository and get _ _ repo updated from disk
LR = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , self . __repoLock ) )
acquired , code = LR . acquire_lock ( )
if not acquired :
    LD . release_lock ( )
    m = "code %s. Unable to aquire the repository lock. You may try again!" % ( code , )
    assert raiseError , Exception ( m )
    return False , m
# remove directory
for _trial in range ( ntrials ) :
    error = None
    try :
        dirList = self . __get_repository_parent_directory ( relativePath = relativePath )
        assert dirList is not None , "Given relative path '%s' is not a repository directory" % ( relativePath , )
        stateBefore = self . get_repository_state ( relaPath = parentPath )
        _files = [ f for f in dirList if isinstance ( f , basestring ) ]
        _dirs = [ d for d in dirList if isinstance ( d , dict ) ]
        _dirs = [ d for d in dirList if dirName not in d ]
        _ = [ dirList . pop ( 0 ) for _ in range ( len ( dirList ) ) ]
        dirList . extend ( _files )
        dirList . extend ( _dirs )
        if clean :
            shutil . rmtree ( realPath )
        else :
            stateAfter = self . get_repository_state ( relaPath = parentPath )
            success , errors = self . __clean_before_after ( stateBefore = stateBefore , stateAfter = stateAfter , keepNoneEmptyDirectory = True )
            assert success , "\n" . join ( errors )
    except Exception as err :
        error = str ( err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        break
# return
if error is None :
    _ , error = self . __save_repository_pickle_file ( lockFirst = False , raiseError = False )
LD . release_lock ( )
LR . release_lock ( )
# check and return
assert error is None or not raiseError , "Unable to remove directory after %i trials '%s' (%s)" % ( relativePath , ntrials , error , )
return error is None , error


signature:
def rename_directory ( self , relativePath , newName , raiseError = True , ntrials = 3 ) :
    """Rename a directory in the repository . It insures renaming the directory in the system .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of
 the directory to be renamed .
 # . newName ( string ) : The new directory name .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( boolean ) : Whether renaming the directory was successful .
 # . message ( None , string ) : Some explanatory message or error reason
 why directory was not renamed ."""

implementation:
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
parentPath , dirName = os . path . split ( relativePath )
if relativePath == '' :
    error = "Renaming main repository directory is not allowed"
    assert not raiseError , error
    return False , error
realPath = os . path . join ( self . __path , relativePath )
newRealPath = os . path . join ( os . path . dirname ( realPath ) , newName )
if os . path . isdir ( newRealPath ) :
    error = "New directory path '%s' already exist" % ( newRealPath , )
    assert not raiseError , error
    return False , error
# get directory parent list
LD = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , parentPath , self . __dirLock ) )
acquired , code = LD . acquire_lock ( )
if not acquired :
    error = "Code %s. Unable to aquire repository lock when renaming '%s'. All prior directories were added. You may try again, to finish adding the directory" % ( code , dirPath )
    assert not raiseError , error
    return False , error
error = None
# lock repository and get _ _ repo updated from disk
LR = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , self . __repoLock ) )
acquired , code = LR . acquire_lock ( )
if not acquired :
    LD . release_lock ( )
    m = "Code %s. Unable to aquire directory lock when renaming '%s'. All prior directories were added. You may try again, to finish adding the directory" % ( code , dirPath )
    assert raiseError , Exception ( m )
    return False , m
# load repository info
for _trial in range ( ntrials ) :
    try :
        repo = self . __load_repository_pickle_file ( os . path . join ( self . __path , self . __repoFile ) )
        self . __repo [ 'walk_repo' ] = repo [ 'walk_repo' ]
    except Exception as err :
        error = str ( err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        error = None
        break
if error is not None :
    LD . release_lock ( )
    LR . release_lock ( )
    assert not raiseError , Exception ( error )
    return False , error
# rename directory
for _trial in range ( ntrials ) :
    error = None
    try :
        dirList = self . __get_repository_parent_directory ( relativePath = relativePath )
        assert dirList is not None , "Given relative path '%s' is not a repository directory" % ( relativePath , )
        # change dirName in dirList
        _dirDict = [ nd for nd in dirList if isinstance ( nd , dict ) ]
        _dirDict = [ nd for nd in _dirDict if dirName in nd ]
        assert len ( _dirDict ) == 1 , "This should not have happened. Directory not found in repository. Please report issue"
        # rename directory
        os . rename ( realPath , newRealPath )
        # update dirList
        _dirDict [ 0 ] [ newName ] = _dirDict [ 0 ] [ dirName ]
        _dirDict [ 0 ] . pop ( dirName )
        # update and dump dirinfo
        self . __save_dirinfo ( description = None , dirInfoPath = parentPath , create = False )
    except Exception as err :
        error = str ( err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        error = None
        break
if error is None :
    _ , error = self . __save_repository_pickle_file ( lockFirst = False , raiseError = False )
LR . release_lock ( )
LD . release_lock ( )
# check and return
assert error is None or not raiseError , "Unable to rename directory '%s' to '%s' after %i trials (%s)" % ( relativePath , newName , ntrials , error , )
return error is None , error


signature:
def copy_directory ( self , relativePath , newRelativePath , overwrite = False , raiseError = True , ntrials = 3 ) :
    """Copy a directory in the repository . New directory must not exist .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of
 the directory to be copied .
 # . newRelativePath ( string ) : The new directory relative path .
 # . overwrite ( boolean ) : Whether to overwrite existing but not tracked
 directory in repository .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( boolean ) : Whether renaming the directory was successful .
 # . message ( None , string ) : Some explanatory message or error reason
 why directory was not renamed ."""

implementation:
# from distutils . dir _ util import copy _ tree
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( overwrite , bool ) , "overwrite must be boolean"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
if relativePath == '' :
    m = "Copying to repository main directory is not possible"
    assert not raiseError , m
    return False , m
realPath = os . path . join ( self . __path , relativePath )
parentRealPath , dirName = os . path . split ( realPath )
parentRelativePath = os . path . dirname ( relativePath )
if not self . is_repository_directory ( relativePath ) :
    m = "Directory '%s' is not a tracked repository directory" % ( relativePath )
    assert not raiseError , m
    return False , m
newRelativePath = self . to_repo_relative_path ( path = newRelativePath , split = False )
newRealPath = os . path . join ( self . __path , newRelativePath )
newParentRealPath , newDirName = os . path . split ( newRealPath )
newParentRelativePath = os . path . dirname ( newRelativePath )
if realPath == newRealPath :
    m = "Copying to the same directory is not possible"
    assert not raiseError , m
    return False , m
if self . is_repository_directory ( newRelativePath ) :
    m = "Directory '%s' is a tracked repository directory" % ( newRelativePath )
    assert not raiseError , m
    return False , m
if os . path . isdir ( newRealPath ) :
    if overwrite :
        try :
            shutil . rmtree ( newRealPath )
        except Exception as err :
            assert not raiseError , str ( err )
            return False , str ( err )
    else :
        error = "New directory path '%s' already exist on disk. Set overwrite to True" % ( newRealPath , )
        assert not raiseError , error
        return False , error
# add directory
try :
    success , reason = self . add_directory ( newParentRelativePath , raiseError = False , ntrials = ntrials )
except Exception as err :
    reason = "Unable to add directory (%s)" % ( str ( err ) )
    success = False
if not success :
    assert not raiseError , reason
    return False , reason
# lock repository and get _ _ repo updated from disk
LR = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , self . __repoLock ) )
acquired , code = LR . acquire_lock ( )
if not acquired :
    m = "code %s. Unable to aquire the repository lock. You may try again!" % ( code , )
    assert raiseError , Exception ( m )
    return False , m
try :
    repo = self . __load_repository_pickle_file ( os . path . join ( self . __path , self . __repoFile ) )
    self . __repo [ 'walk_repo' ] = repo [ 'walk_repo' ]
except Exception as err :
    LR . release_lock ( )
    assert not raiseError , Exception ( str ( err ) )
    return False , m
# create locks
L0 = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( parentRealPath , self . __dirLock ) )
acquired , code = L0 . acquire_lock ( )
if not acquired :
    LR . release_lock ( )
    error = "Code %s. Unable to aquire the lock when adding '%s'. All prior directories were added. You may try again, to finish adding directory" % ( code , dirPath )
    assert not raiseError , error
    return False , error
L1 = None
if parentRealPath != newParentRealPath :
    L1 = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( newParentRealPath , self . __dirLock ) )
    acquired , code = L1 . acquire_lock ( )
    if not acquired :
        L0 . release_lock ( )
        LR . release_lock ( )
        error = "Code %s. Unable to aquire the lock when adding '%s'. All prior directories were added. You may try again, to finish adding directory" % ( code , dirPath )
        assert not raiseError , error
        return False , error
# get directory parent list
error = None
for _trial in range ( ntrials ) :
    try : # make sure again because sometimes , when multiple processes are working on the same repo things can happen in between
        assert self . is_repository_directory ( relativePath ) , "Directory '%s' is not anymore a tracked repository directory" % ( relativePath )
        assert not self . is_repository_directory ( newRelativePath ) , "Directory '%s' has become a tracked repository directory" % ( relativePath )
        dirList = self . __get_repository_parent_directory ( relativePath = relativePath )
        assert dirList is not None , "Given relative path '%s' is not a repository directory" % ( relativePath , )
        newDirList = self . __get_repository_parent_directory ( relativePath = newRelativePath )
        assert newDirList is not None , "Given new relative path '%s' parent directory is not a repository directory" % ( newRelativePath , )
        # change dirName in dirList
        _dirDict = [ nd for nd in dirList if isinstance ( nd , dict ) ]
        _dirDict = [ nd for nd in _dirDict if dirName in nd ]
        assert len ( _dirDict ) == 1 , "This should not have happened. Directory not found in repository. Please report issue"
        _newDirDict = [ nd for nd in newDirList if isinstance ( nd , dict ) ]
        _newDirDict = [ nd for nd in _newDirDict if newDirName in nd ]
        assert len ( _newDirDict ) == 0 , "This should not have happened. New directory is found in repository. Please report issue"
        # try to copy directory
        _newDirDict = copy . deepcopy ( _dirDict [ 0 ] )
        if dirName != newDirName :
            _newDirDict [ newDirName ] = _newDirDict . pop ( dirName )
        copy_tree ( realPath , newRealPath )
        # update newDirList
        newDirList . append ( _newDirDict )
        # update and dump dirinfo
        self . __save_dirinfo ( description = None , dirInfoPath = newParentRelativePath , create = False )
    except Exception as err :
        error = str ( err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        error = None
        break
if error is None :
    _ , error = self . __save_repository_pickle_file ( lockFirst = False , raiseError = False )
LR . release_lock ( )
L0 . release_lock ( )
if L1 is not None :
    L1 . release_lock ( )
# check and return
assert error is None or not raiseError , "Unable to copy directory '%s' to '%s' after %i trials (%s)" % ( relativePath , newRelativePath , ntrials , error , )
return error is None , error


signature:
def dump_file ( self , value , relativePath , description = None , dump = None , pull = None , replace = False , raiseError = True , ntrials = 3 ) :
    """Dump a file using its value to the system and creates its
 attribute in the Repository with utc timestamp .
 : Parameters :
 # . value ( object ) : The value of a file to dump and add to the
 repository . It is any python object or file .
 # . relativePath ( str ) : The relative to the repository path to where
 to dump the file .
 # . description ( None , string ) : Any description about the file .
 # . dump ( None , string ) : The dumping method .
 If None it will be set automatically to pickle and therefore the
 object must be pickleable . If a string is given , it can be a
 keyword ( ' json ' , ' pickle ' , ' dill ' ) or a string compileable code to
 dump the data . The string code must include all the necessary
 imports and a ' $ FILE _ PATH ' that replaces the absolute file path
 when the dumping will be performed . \n
 e . g . " import numpy as np ; np . savetxt ( fname = ' $ FILE _ PATH ' , X = value , fmt = ' % . 6e ' ) "
 # . pull ( None , string ) : The pulling method . If None it will be set
 automatically to pickle and therefore the object must be
 pickleable . If a string is given , it can be a keyword
 ( ' json ' , ' pickle ' , ' dill ' ) or a string compileable code to pull
 the data . The string code must include all the necessary imports ,
 a ' $ FILE _ PATH ' that replaces the absolute file path when the
 dumping will be performed and finally a PULLED _ DATA variable . \n
 e . g " import numpy as np ; PULLED _ DATA = np . loadtxt ( fname = ' $ FILE _ PATH ' ) "
 # . replace ( boolean ) : Whether to replace any existing file .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( boolean ) : Whether renaming the directory was successful .
 # . message ( None , string ) : Some explanatory message or error reason
 why directory was not dumped ."""

implementation:
# check arguments
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( replace , bool ) , "replace must be boolean"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
if description is None :
    description = ''
assert isinstance ( description , basestring ) , "description must be None or a string"
# convert dump and pull methods to strings
if pull is None and dump is not None :
    if dump . startswith ( 'pickle' ) or dump . startswith ( 'dill' ) or dump . startswith ( 'numpy' ) or dump == 'json' :
        pull = dump
dump = get_dump_method ( dump , protocol = self . _DEFAULT_PICKLE_PROTOCOL )
pull = get_pull_method ( pull )
# check name and path
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
savePath = os . path . join ( self . __path , relativePath )
fPath , fName = os . path . split ( savePath )
# check if name is allowed
success , reason = self . is_name_allowed ( savePath )
if not success :
    assert not raiseError , reason
    return False , reason
# ensure directory added
try :
    success , reason = self . add_directory ( fPath , raiseError = False , ntrials = ntrials )
except Exception as err :
    reason = "Unable to add directory (%s)" % ( str ( err ) )
    success = False
if not success :
    assert not raiseError , reason
    return False , reason
# lock repository
LR = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( self . __path , self . __repoLock ) )
acquired , code = LR . acquire_lock ( )
if not acquired :
    m = "code %s. Unable to aquire the repository lock. You may try again!" % ( code , )
    assert raiseError , Exception ( m )
    return False , m
# lock file
LF = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( fPath , self . __fileLock % fName ) )
acquired , code = LF . acquire_lock ( )
if not acquired :
    LR . release_lock ( )
    error = "Code %s. Unable to aquire the lock when adding '%s'" % ( code , relativePath )
    assert not raiseError , error
    return False , error
# load repository info
for _trial in range ( ntrials ) :
    try :
        repo = self . __load_repository_pickle_file ( os . path . join ( self . __path , self . __repoFile ) )
        self . __repo [ 'walk_repo' ] = repo [ 'walk_repo' ]
    except Exception as err :
        error = str ( err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        error = None
        break
if error is not None :
    LR . release_lock ( )
    LF . release_lock ( )
    assert not raiseError , Exception ( error )
    return False , error
# dump file
for _trial in range ( ntrials ) :
    error = None
    try :
        isRepoFile , fileOnDisk , infoOnDisk , classOnDisk = self . is_repository_file ( relativePath )
        if isRepoFile :
            assert replace , "file is a registered repository file. set replace to True to replace"
        fileInfoPath = os . path . join ( self . __path , os . path . dirname ( relativePath ) , self . __fileInfo % fName )
        if isRepoFile and fileOnDisk :
            with open ( fileInfoPath , 'rb' ) as fd :
                info = pickle . load ( fd )
            assert info [ 'repository_unique_name' ] == self . __repo [ 'repository_unique_name' ] , "it seems that file was created by another repository"
            info [ 'last_update_utctime' ] = time . time ( )
        else :
            info = { 'repository_unique_name' : self . __repo [ 'repository_unique_name' ] }
            info [ 'create_utctime' ] = info [ 'last_update_utctime' ] = time . time ( )
        info [ 'dump' ] = dump
        info [ 'pull' ] = pull
        info [ 'description' ] = description
        # get parent directory list if file is new and not being replaced
        if not isRepoFile :
            dirList = self . __get_repository_directory ( fPath )
        # dump file
        # exec ( dump . replace ( " $ FILE _ PATH " , str ( savePath ) ) )
        my_exec ( dump . replace ( "$FILE_PATH" , str ( savePath ) ) , locals = locals ( ) , globals = globals ( ) , description = 'dump' )
        # update info
        with open ( fileInfoPath , 'wb' ) as fd :
            pickle . dump ( info , fd , protocol = self . _DEFAULT_PICKLE_PROTOCOL )
            fd . flush ( )
            os . fsync ( fd . fileno ( ) )
        # update class file
        fileClassPath = os . path . join ( self . __path , os . path . dirname ( relativePath ) , self . __fileClass % fName )
        with open ( fileClassPath , 'wb' ) as fd :
            if value is None :
                klass = None
            else :
                klass = value . __class__
            pickle . dump ( klass , fd , protocol = self . _DEFAULT_PICKLE_PROTOCOL )
            fd . flush ( )
            os . fsync ( fd . fileno ( ) )
        # add to repo if file is new and not being replaced
        if not isRepoFile :
            dirList . append ( fName )
    except Exception as err :
        error = "unable to dump the file (%s)" % ( str ( err ) , )
        try :
            if 'pickle.dump(' in dump :
                mi = get_pickling_errors ( value )
                if mi is not None :
                    error += '\nmore info: %s' % str ( mi )
        except :
            pass
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        error = None
        break
# save repository
if error is None :
    _ , error = self . __save_repository_pickle_file ( lockFirst = False , raiseError = False )
# release locks
LR . release_lock ( )
LF . release_lock ( )
assert not raiseError or error is None , "unable to dump file '%s' after %i trials (%s)" % ( relativePath , ntrials , error , )
return success , error


signature:
def update_file ( self , value , relativePath , description = False , dump = False , pull = False , raiseError = True , ntrials = 3 ) :
    """Update the value of a file that is already in the Repository . \n
 If file is not registered in repository , and error will be thrown . \n
 If file is missing in the system , it will be regenerated as dump method
 is called .
 Unlike dump _ file , update _ file won ' t block the whole repository but only
 the file being updated .
 : Parameters :
 # . value ( object ) : The value of a file to update .
 # . relativePath ( str ) : The relative to the repository path of the
 file to be updated .
 # . description ( False , string ) : Any random description about the file .
 If False is given , the description info won ' t be updated ,
 otherwise it will be update to what description argument value is .
 # . dump ( False , string ) : The new dump method . If False is given ,
 the old one will be used .
 # . pull ( False , string ) : The new pull method . If False is given ,
 the old one will be used .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( boolean ) : Whether renaming the directory was successful .
 # . message ( None , string ) : Some explanatory message or error reason
 why directory was not updated ."""

implementation:
# check arguments
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert description is False or description is None or isinstance ( description , basestring ) , "description must be False, None or a string"
assert dump is False or dump is None or isinstance ( dump , basestring ) , "dump must be False, None or a string"
assert pull is False or pull is None or isinstance ( pull , basestring ) , "pull must be False, None or a string"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
# get name and path
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
savePath = os . path . join ( self . __path , relativePath )
fPath , fName = os . path . split ( savePath )
# get locker
LF = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( fPath , self . __fileLock % fName ) )
acquired , code = LF . acquire_lock ( )
if not acquired :
    error = "Code %s. Unable to aquire the lock to update '%s'" % ( code , relativePath )
    assert not raiseError , error
    return False , error
# update file
for _trial in range ( ntrials ) :
    message = [ ]
    updated = False
    try : # check file in repository
        isRepoFile , fileOnDisk , infoOnDisk , classOnDisk = self . is_repository_file ( relativePath )
        assert isRepoFile , "file '%s' is not registered in repository, no update can be performed." % ( relativePath , )
        # get file info
        if not fileOnDisk :
            assert description is not False , "file '%s' is found on disk, description must be provided" % ( relativePath , )
            assert dump is not False , "file '%s' is found on disk, dump must be provided" % ( relativePath , )
            assert pull is not False , "file '%s' is found on disk, pull must be provided" % ( relativePath , )
            info = { }
            info [ 'repository_unique_name' ] = self . __repo [ 'repository_unique_name' ]
            info [ 'create_utctime' ] = info [ 'last_update_utctime' ] = time . time ( )
        else :
            with open ( os . path . join ( fPath , self . __fileInfo % fName ) , 'rb' ) as fd :
                info = pickle . load ( fd )
                info [ 'last_update_utctime' ] = time . time ( )
        if not fileOnDisk :
            message . append ( "file %s is registered in repository but it was found on disk prior to updating" % relativePath )
        if not infoOnDisk :
            message . append ( "%s is not found on disk prior to updating" % self . __fileInfo % fName )
        if not classOnDisk :
            message . append ( "%s is not found on disk prior to updating" % self . __fileClass % fName )
        # get dump and pull
        if ( description is False ) or ( dump is False ) or ( pull is False ) :
            if description is False :
                description = info [ 'description' ]
            elif description is None :
                description = ''
            if dump is False :
                dump = info [ 'dump' ]
            elif dump is None :
                dump = get_dump_method ( dump , protocol = self . _DEFAULT_PICKLE_PROTOCOL )
            if pull is False :
                pull = info [ 'pull' ]
            elif pull is None :
                pull = get_pull_method ( pull )
        # update dump , pull and description
        info [ 'dump' ] = dump
        info [ 'pull' ] = pull
        info [ 'description' ] = description
        # dump file
        my_exec ( dump . replace ( "$FILE_PATH" , str ( savePath ) ) , locals = locals ( ) , globals = globals ( ) , description = 'update' )
        # exec ( dump . replace ( " $ FILE _ PATH " , str ( savePath ) ) )
        # remove file if exists
        _path = os . path . join ( fPath , self . __fileInfo % fName )
        # update info
        with open ( _path , 'wb' ) as fd :
            pickle . dump ( info , fd , protocol = self . _DEFAULT_PICKLE_PROTOCOL )
            fd . flush ( )
            os . fsync ( fd . fileno ( ) )
        # update class file
        fileClassPath = os . path . join ( self . __path , os . path . dirname ( relativePath ) , self . __fileClass % fName )
        with open ( fileClassPath , 'wb' ) as fd :
            if value is None :
                klass = None
            else :
                klass = value . __class__
            pickle . dump ( klass , fd , protocol = self . _DEFAULT_PICKLE_PROTOCOL )
            fd . flush ( )
            os . fsync ( fd . fileno ( ) )
    except Exception as err :
        message . append ( str ( err ) )
        updated = False
        try :
            if 'pickle.dump(' in dump :
                mi = get_pickling_errors ( value )
                if mi is not None :
                    message . append ( 'more info: %s' % str ( mi ) )
        except :
            pass
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , '\n' . join ( message ) ) )
    else :
        updated = True
        break
# release lock
LF . release_lock ( )
# check and return
assert updated or not raiseError , "Unable to update file '%s' (%s)" % ( relativePath , '\n' . join ( message ) , )
return updated , '\n' . join ( message )


signature:
def pull_file ( self , relativePath , pull = None , update = True , ntrials = 3 ) :
    """Pull a file ' s data from the Repository .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path from
 where to pull the file .
 # . pull ( None , string ) : The pulling method .
 If None , the pull method saved in the file info will be used .
 If a string is given , the string should include all the necessary
 imports , a ' $ FILE _ PATH ' that replaces the absolute file path when
 the dumping will be performed and finally a PULLED _ DATA variable .
 e . g " import numpy as np ; PULLED _ DATA = np . loadtxt ( fname = ' $ FILE _ PATH ' ) "
 # . update ( boolean ) : If pull is not None , Whether to update the pull
 method stored in the file info by the given pull method .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . data ( object ) : The pulled data from the file ."""

implementation:
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
# check name and path
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
realPath = os . path . join ( self . __path , relativePath )
fPath , fName = os . path . split ( realPath )
# check whether it ' s a repository file
isRepoFile , fileOnDisk , infoOnDisk , classOnDisk = self . is_repository_file ( relativePath )
if not isRepoFile :
    fileOnDisk = [ "" , ". File itself is found on disk" ] [ fileOnDisk ]
    infoOnDisk = [ "" , ". %s is found on disk" % self . __fileInfo % fName ] [ infoOnDisk ]
    classOnDisk = [ "" , ". %s is found on disk" % self . __fileClass % fName ] [ classOnDisk ]
    assert False , "File '%s' is not a repository file%s%s%s" % ( relativePath , fileOnDisk , infoOnDisk , classOnDisk )
assert fileOnDisk , "File '%s' is registered in repository but the file itself was not found on disk" % ( relativePath , )
if not infoOnDisk :
    if pull is not None :
        warnings . warn ( "'%s' was not found on disk but pull method is given" % ( self . __fileInfo % fName ) )
    else :
        raise Exception ( "File '%s' is registered in repository but the '%s' was not found on disk and pull method is not specified" % ( relativePath , ( self . __fileInfo % fName ) ) )
# lock repository
LF = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( fPath , self . __fileLock % fName ) )
acquired , code = LF . acquire_lock ( )
if not acquired :
    error = "Code %s. Unable to aquire the lock when adding '%s'" % ( code , relativePath )
    return False , error
# pull file
for _trial in range ( ntrials ) :
    error = None
    try : # get pull method
        if pull is not None :
            pull = get_pull_method ( pull )
        else :
            with open ( os . path . join ( fPath , self . __fileInfo % fName ) , 'rb' ) as fd :
                info = pickle . load ( fd )
            pull = info [ 'pull' ]
        # try to pull file
        # namespace = { }
        # namespace . update ( globals ( ) )
        # exec ( pull . replace ( " $ FILE _ PATH " , str ( realPath ) ) , namespace )
        my_exec ( pull . replace ( "$FILE_PATH" , str ( realPath ) ) , locals = locals ( ) , globals = globals ( ) , description = 'pull' )
    except Exception as err :
        LF . release_lock ( )
        m = str ( pull ) . replace ( "$FILE_PATH" , str ( realPath ) )
        error = "Unable to pull data using '%s' from file (%s)" % ( m , err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        break
LF . release_lock ( )
assert error is None , "After %i trials, %s" % ( ntrials , error )
# return data
return locals ( ) [ 'PULLED_DATA' ]


signature:
def rename_file ( self , relativePath , newRelativePath , force = False , raiseError = True , ntrials = 3 ) :
    """Rename a file in the repository . It insures renaming the file in the system .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of
 the file that needst to be renamed .
 # . newRelativePath ( string ) : The new relative to the repository path
 of where to move and rename the file .
 # . force ( boolean ) : Whether to force renaming even when another
 repository file exists . In this case old repository file
 will be removed from the repository and the system as well .
 # . raiseError ( boolean ) : Whether to raise encountered error instead
 of returning failure .
 # . ntrials ( int ) : After aquiring all locks , ntrials is the maximum
 number of trials allowed before failing .
 In rare cases , when multiple processes
 are accessing the same repository components , different processes
 can alter repository components between successive lock releases
 of some other process . Bigger number of trials lowers the
 likelyhood of failure due to multiple processes same time
 alteration .
 : Returns :
 # . success ( boolean ) : Whether renaming the file was successful .
 # . message ( None , string ) : Some explanatory message or error reason
 why directory was not updated ."""

implementation:
assert isinstance ( raiseError , bool ) , "raiseError must be boolean"
assert isinstance ( force , bool ) , "force must be boolean"
assert isinstance ( ntrials , int ) , "ntrials must be integer"
assert ntrials > 0 , "ntrials must be >0"
# check old name and path
relativePath = self . to_repo_relative_path ( path = relativePath , split = False )
realPath = os . path . join ( self . __path , relativePath )
fPath , fName = os . path . split ( realPath )
# check new name and path
newRelativePath = self . to_repo_relative_path ( path = newRelativePath , split = False )
newRealPath = os . path . join ( self . __path , newRelativePath )
nfPath , nfName = os . path . split ( newRealPath )
# lock old file
LO = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( fPath , self . __fileLock % fName ) )
acquired , code = LO . acquire_lock ( )
if not acquired :
    error = "Code %s. Unable to aquire the lock for old file '%s'" % ( code , relativePath )
    assert not raiseError , error
    return False , error
# add directory
try :
    success , reason = self . add_directory ( nfPath , raiseError = False , ntrials = ntrials )
except Exception as err :
    reason = "Unable to add directory (%s)" % ( str ( err ) )
    success = False
if not success :
    LO . release_lock ( )
    assert not raiseError , reason
    return False , reason
# create new file lock
LN = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( nfPath , self . __fileLock % nfName ) )
acquired , code = LN . acquire_lock ( )
if not acquired :
    LO . release_lock ( )
    error = "Code %s. Unable to aquire the lock for new file path '%s'" % ( code , newRelativePath )
    assert not raiseError , error
    return False , error
# rename file
for _trial in range ( ntrials ) :
    renamed = False
    error = None
    try : # check whether it ' s a repository file
        isRepoFile , fileOnDisk , infoOnDisk , classOnDisk = self . is_repository_file ( relativePath )
        assert isRepoFile , "file '%s' is not a repository file" % ( relativePath , )
        assert fileOnDisk , "file '%s' is found on disk" % ( relativePath , )
        assert infoOnDisk , "%s is found on disk" % self . __fileInfo % fName
        assert classOnDisk , "%s is found on disk" % self . __fileClass % fName
        # get new file path
        nisRepoFile , nfileOnDisk , ninfoOnDisk , nclassOnDisk = self . is_repository_file ( newRelativePath )
        assert not nisRepoFile or force , "New file path is a registered repository file, set force to True to proceed regardless"
        # get parent directories list
        oDirList = self . __get_repository_directory ( fPath )
        nDirList = self . __get_repository_directory ( nfPath )
        # remove new file and all repository files from disk
        if os . path . isfile ( newRealPath ) :
            os . remove ( newRealPath )
        if os . path . isfile ( os . path . join ( nfPath , self . __fileInfo % nfName ) ) :
            os . remove ( os . path . join ( nfPath , self . __fileInfo % nfName ) )
        if os . path . isfile ( os . path . join ( nfPath , self . __fileClass % nfName ) ) :
            os . remove ( os . path . join ( nfPath , self . __fileClass % nfName ) )
        # move old file to new path
        os . rename ( realPath , newRealPath )
        os . rename ( os . path . join ( fPath , self . __fileInfo % fName ) , os . path . join ( nfPath , self . __fileInfo % nfName ) )
        os . rename ( os . path . join ( fPath , self . __fileClass % fName ) , os . path . join ( nfPath , self . __fileClass % nfName ) )
        # update list
        findex = oDirList . index ( fName )
        oDirList . pop ( findex )
        # update new list
        if nfName not in nDirList :
            nDirList . append ( nfName )
    except Exception as err :
        renamed = False
        error = str ( err )
        if self . DEBUG_PRINT_FAILED_TRIALS :
            print ( "Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute" % ( _trial , inspect . stack ( ) [ 1 ] [ 3 ] , str ( error ) ) )
    else :
        renamed = True
        break
# release locks
LO . release_lock ( )
LN . release_lock ( )
# always clean old file lock
try :
    if os . path . isfile ( os . path . join ( fPath , self . __fileLock % fName ) ) :
        os . remove ( os . path . join ( fPath , self . __fileLock % fName ) )
except :
    pass
# return
assert renamed or not raiseError , "Unable to rename file '%s' to '%s' after %i trials (%s)" % ( relativePath , newRelativePath , ntrials , error , )
# assert renamed or not raiseError , ' \ n ' . join ( message )
return renamed , error