signature:
def _issue_cert ( self , domain ) :
    """Issue a certificate for the given domain ."""

implementation:
def errback ( failure ) : # Don ' t fail on some of the errors we could get from the ACME
# server , rather just log an error so that we can continue with
# other domains .
    failure . trap ( txacme_ServerError )
    acme_error = failure . value . message
    if acme_error . code in [ 'rateLimited' , 'serverInternal' , 'connection' , 'unknownHost' ] : # TODO : Fire off an error to Sentry or something ?
        self . log . error ( 'Error ({code}) issuing certificate for "{domain}": ' '{detail}' , code = acme_error . code , domain = domain , detail = acme_error . detail )
    else : # There are more error codes but if they happen then something
    # serious has gone wrong - - carry on error - ing .
        return failure
d = self . txacme_service . issue_cert ( domain )
return d . addErrback ( errback )


signature:
def warn_if_detached ( func ) :
    """Warn if self / cls is detached ."""

implementation:
@ wraps ( func )
def wrapped ( this , * args , ** kwargs ) : # Check for _ detached in _ _ dict _ _ instead of using hasattr
# to avoid infinite loop in _ _ getattr _ _
    if '_detached' in this . __dict__ and this . _detached :
        warnings . warn ( 'here' )
    return func ( this , * args , ** kwargs )
return wrapped


signature:
def has_storage ( func ) :
    """Ensure that self / cls contains a Storage backend ."""

implementation:
@ wraps ( func )
def wrapped ( * args , ** kwargs ) :
    me = args [ 0 ]
    if not hasattr ( me , '_storage' ) or not me . _storage :
        raise exceptions . ImproperConfigurationError ( 'No storage backend attached to schema <{0}>.' . format ( me . _name . upper ( ) ) )
    return func ( * args , ** kwargs )
return wrapped


signature:
def rm_fwd_refs ( obj ) :
    """When removing an object , other objects with references to the current
 object should remove those references . This function identifies objects
 with forward references to the current object , then removes those
 references .
 : param obj : Object to which forward references should be removed"""

implementation:
for stack , key in obj . _backrefs_flat : # Unpack stack
    backref_key , parent_schema_name , parent_field_name = stack
    # Get parent info
    parent_schema = obj . _collections [ parent_schema_name ]
    parent_key_store = parent_schema . _pk_to_storage ( key )
    parent_object = parent_schema . load ( parent_key_store )
    if parent_object is None :
        continue
    # Remove forward references
    if parent_object . _fields [ parent_field_name ] . _list :
        getattr ( parent_object , parent_field_name ) . remove ( obj )
    else :
        parent_field_object = parent_object . _fields [ parent_field_name ]
        setattr ( parent_object , parent_field_name , parent_field_object . _gen_default ( ) )
    # Save
    parent_object . save ( )


signature:
def rm_back_refs ( obj ) :
    """When removing an object with foreign fields , back - references from
 other objects to the current object should be deleted . This function
 identifies foreign fields of the specified object whose values are not
 None and which specify back - reference keys , then removes back - references
 from linked objects to the specified object .
 : param obj : Object for which back - references should be removed"""

implementation:
for ref in _collect_refs ( obj ) :
    ref [ 'value' ] . _remove_backref ( ref [ 'field_instance' ] . _backref_field_name , obj , ref [ 'field_name' ] , strict = False )


signature:
def ensure_backrefs ( obj , fields = None ) :
    """Ensure that all forward references on the provided object have the
 appropriate backreferences .
 : param StoredObject obj : Database record
 : param list fields : Optional list of field names to check"""

implementation:
for ref in _collect_refs ( obj , fields ) :
    updated = ref [ 'value' ] . _update_backref ( ref [ 'field_instance' ] . _backref_field_name , obj , ref [ 'field_name' ] , )
    if updated :
        logging . debug ( 'Updated reference {}:{}:{}:{}:{}' . format ( obj . _name , obj . _primary_key , ref [ 'field_name' ] , ref [ 'value' ] . _name , ref [ 'value' ] . _primary_key , ) )


signature:
def _remove_by_pk ( self , key , flush = True ) :
    """Retrieve value from store .
 : param key : Key"""

implementation:
try :
    del self . store [ key ]
except Exception as error :
    pass
if flush :
    self . flush ( )


signature:
def eventhandler ( * args , ** kwargs ) :
    """Decorator . Marks a function as a receiver for the specified slack event ( s ) .
 * events - String or list of events to handle"""

implementation:
def wrapper ( func ) :
    if isinstance ( kwargs [ 'events' ] , basestring ) :
        kwargs [ 'events' ] = [ kwargs [ 'events' ] ]
    func . is_eventhandler = True
    func . events = kwargs [ 'events' ]
    return func
return wrapper


signature:
def start ( self ) :
    """Initializes the bot , plugins , and everything ."""

implementation:
self . bot_start_time = datetime . now ( )
self . webserver = Webserver ( self . config [ 'webserver' ] [ 'host' ] , self . config [ 'webserver' ] [ 'port' ] )
self . plugins . load ( )
self . plugins . load_state ( )
self . _find_event_handlers ( )
self . sc = ThreadedSlackClient ( self . config [ 'slack_token' ] )
self . always_send_dm = [ '_unauthorized_' ]
if 'always_send_dm' in self . config :
    self . always_send_dm . extend ( map ( lambda x : '!' + x , self . config [ 'always_send_dm' ] ) )
# Rocket is very noisy at debug
logging . getLogger ( 'Rocket.Errors.ThreadPool' ) . setLevel ( logging . INFO )
self . is_setup = True
if self . test_mode :
    self . metrics [ 'startup_time' ] = ( datetime . now ( ) - self . bot_start_time ) . total_seconds ( ) * 1000.0


signature:
def run ( self , start = True ) :
    """Connects to slack and enters the main loop .
 * start - If True , rtm . start API is used . Else rtm . connect API is used
 For more info , refer to
 https : / / python - slackclient . readthedocs . io / en / latest / real _ time _ messaging . html # rtm - start - vs - rtm - connect"""

implementation:
# Fail out if setup wasn ' t run
if not self . is_setup :
    raise NotSetupError
# Start the web server
self . webserver . start ( )
first_connect = True
try :
    while self . runnable :
        if self . reconnect_needed :
            if not self . sc . rtm_connect ( with_team_state = start ) :
                return False
            self . reconnect_needed = False
            if first_connect :
                first_connect = False
                self . plugins . connect ( )
        # Get all waiting events - this always returns a list
        try :
            events = self . sc . rtm_read ( )
        except AttributeError :
            self . log . exception ( 'Something has failed in the slack rtm library.  This is fatal.' )
            self . runnable = False
            events = [ ]
        except :
            self . log . exception ( 'Unhandled exception in rtm_read()' )
            self . reconnect_needed = True
            events = [ ]
        for e in events :
            try :
                self . _handle_event ( e )
            except KeyboardInterrupt : # Gracefully shutdown
                self . runnable = False
            except :
                self . log . exception ( 'Unhandled exception in event handler' )
        sleep ( 0.1 )
except KeyboardInterrupt : # On ctrl - c , just exit
    pass
except :
    self . log . exception ( 'Unhandled exception' )


signature:
def stop ( self ) :
    """Does cleanup of bot and plugins ."""

implementation:
if self . webserver is not None :
    self . webserver . stop ( )
if not self . test_mode :
    self . plugins . save_state ( )


signature:
def send_message ( self , channel , text , thread = None , reply_broadcast = None ) :
    """Sends a message to the specified channel
 * channel - The channel to send to . This can be a SlackChannel object , a channel id , or a channel name
 ( without the # )
 * text - String to send
 * thread - reply to the thread . See https : / / api . slack . com / docs / message - threading # threads _ party
 * reply _ broadcast - Set to true to indicate your reply is germane to all members of a channel"""

implementation:
# This doesn ' t want the # in the channel name
if isinstance ( channel , SlackRoomIMBase ) :
    channel = channel . id
self . log . debug ( "Trying to send to %s: %s" , channel , text )
self . sc . rtm_send_message ( channel , text , thread = thread , reply_broadcast = reply_broadcast )


signature:
def send_im ( self , user , text ) :
    """Sends a message to a user as an IM
 * user - The user to send to . This can be a SlackUser object , a user id , or the username ( without the @ )
 * text - String to send"""

implementation:
if isinstance ( user , SlackUser ) :
    user = user . id
    channelid = self . _find_im_channel ( user )
else :
    channelid = user . id
self . send_message ( channelid , text )


signature:
def token_is_correct ( self , token ) :
    """Подходит ли токен , для генерации текста .
 Допускаются русские слова , знаки препинания и символы начала и конца ."""

implementation:
if self . is_rus_word ( token ) :
    return True
elif self . ONLY_MARKS . search ( token ) :
    return True
elif self . END_TOKENS . search ( token ) :
    return True
elif token in "$^" :
    return True
return False


signature:
def get_optimal_variant ( self , variants , start_words , ** kwargs ) :
    """Возвращает оптимальный вариант , из выборки ."""

implementation:
if not start_words :
    return ( choice ( variants ) , { } )
_variants = [ ]
_weights = [ ]
for tok in frozenset ( variants ) :
    if not self . token_is_correct ( tok ) :
        continue
    weight = variants . count ( tok )
    for word in start_words :
        for token in self . ONLY_WORDS . finditer ( word . strip ( ) . lower ( ) ) :
            if token . group ( ) == tok :
                weight <<= 1
    _variants . append ( tok )
    _weights . append ( weight )
if not _variants :
    return ( choice ( variants ) , { } )
return ( choices ( _variants , weights = _weights , k = 1 ) [ 0 ] , { } )


signature:
def start_generation ( self , * start_words , ** kwargs ) :
    """Генерирует предложение .
 : start _ words : Попытаться начать предложение с этих слов ."""

implementation:
out_text = ""
_need_capialize = True
for token in self . _get_generate_tokens ( * start_words , ** kwargs ) :
    if token in "$^" :
        _need_capialize = True
        continue
    if self . ONLY_WORDS . search ( token ) :
        out_text += " "
    if _need_capialize :
        _need_capialize = False
        token = token . title ( )
    out_text += token
return out_text . strip ( )


signature:
def get_start_array ( self , * start_words , ** kwargs ) :
    """Генерирует начало предложения .
 : start _ words : Попытаться начать предложение с этих слов ."""

implementation:
if not self . start_arrays :
    raise MarkovTextExcept ( "Не с чего начинать генерацию." )
if not start_words :
    return choice ( self . start_arrays )
_variants = [ ]
_weights = [ ]
for tokens in self . start_arrays :
    weight = 0b1
    for word in start_words :
        word = word . strip ( ) . lower ( )
        for token in self . ONLY_WORDS . finditer ( word ) :
            if token . group ( ) in tokens :
                weight <<= 1
    if weight > 0b1 :
        _variants . append ( tokens )
        _weights . append ( weight )
if not _variants :
    return choice ( self . start_arrays )
return choices ( _variants , weights = _weights , k = 1 ) [ 0 ]


signature:
def create_base ( self ) :
    """Метод создаёт базовый словарь , на основе массива токенов .
 Вызывается из метода обновления ."""

implementation:
self . base_dict = { }
_start_arrays = [ ]
for tokens , word in self . chain_generator ( ) :
    self . base_dict . setdefault ( tokens , [ ] ) . append ( word )
    if tokens [ 0 ] == "^" : # Первые ключи , для начала генерации .
        _start_arrays . append ( tokens )
self . start_arrays = tuple ( frozenset ( self . get_corrected_arrays ( _start_arrays ) ) )


signature:
def chain_generator ( self ) :
    """Возвращает генератор , формата :
 ( ( " токен " , . . . ) , " вариант " )
 Где количество токенов определяет переменная объекта chain _ order ."""

implementation:
n_chain = self . chain_order
if n_chain < 1 :
    raise MarkovTextExcept ( "Цепь не может быть {0}-порядка." . format ( n_chain ) )
n_chain += 1
# Последнее значение - результат возврата .
changing_array = deque ( maxlen = n_chain )
for token in self . tokens_array :
    changing_array . append ( token )
    if len ( changing_array ) < n_chain :
        continue
        # Массив ещё неполон .
    yield ( tuple ( changing_array ) [ : - 1 ] , changing_array [ - 1 ] )


signature:
def set_vocabulary ( self , peer_id , from_dialogue = None , update = False ) :
    """Получает вокабулар из функции get _ vocabulary и делает его активным ."""

implementation:
self . tokens_array = self . get_vocabulary ( peer_id , from_dialogue , update )
self . create_base ( )


signature:
def create_dump ( self , name = None ) :
    """Сохраняет текущую базу на жёсткий диск .
 : name : Имя файла , без расширения ."""

implementation:
name = name or "vocabularDump"
backup_dump_file = os_join ( self . temp_folder , "{0}.backup" . format ( name ) )
dump_file = os_join ( self . temp_folder , "{0}.json" . format ( name ) )
with open ( backup_dump_file , "w" , encoding = "utf-8" ) as js_file :
    json . dump ( self . tokens_array , js_file , ensure_ascii = False )
copy2 ( backup_dump_file , dump_file )
remove ( backup_dump_file )


signature:
def load_dump ( self , name = None ) :
    """Загружает базу с жёсткого диска .
 Текущая база заменяется .
 : name : Имя файла , без расширения ."""

implementation:
name = name or "vocabularDump"
dump_file = os_join ( self . temp_folder , "{0}.json" . format ( name ) )
if not isfile ( dump_file ) :
    raise MarkovTextExcept ( "Файл {0!r} не найден." . format ( dump_file ) )
with open ( dump_file , "rb" ) as js_file :
    self . tokens_array = tuple ( json . load ( js_file ) )
self . create_base ( )


signature:
def get_vocabulary ( self , target , user = None , update = False ) :
    """Возвращает запас слов , на основе переписок ВК .
 Для имитации речи конкретного человека .
 Работает только с импортом объекта " Владя - бота " .
 : target :
 Объект собседника . Источник переписки .
 : user :
 Объект юзера , чью речь имитируем .
 Если None , то вся переписка .
 : update :
 Не использовать бэкап . Обновить форсированно ."""

implementation:
if not self . vk_object :
    raise MarkovTextExcept ( "Объект бота не задан." )
json_name = "{0}{1}_{2}" . format ( target . __class__ . __name__ , target . id , user . id )
json_file = os_join ( self . temp_folder , "{0}.json" . format ( json_name ) )
if not update :
    result = self . vocabulars . get ( json_name , None )
    if result :
        return result
    elif isfile ( json_file ) :
        with open ( json_file , "rb" ) as js_file :
            result = self . vocabulars [ json_name ] = tuple ( json . load ( js_file ) )
        return result
_tokens_array = tuple ( self . __parse_from_vk_dialogue ( target , user ) )
backup_file = "{0}.backup" . format ( splitext ( json_file ) [ 0 ] )
with open ( backup_file , "w" , encoding = "utf-8" ) as js_file :
    json . dump ( _tokens_array , js_file , ensure_ascii = False )
copy2 ( backup_file , json_file )
remove ( backup_file )
self . vocabulars [ json_name ] = _tokens_array
return _tokens_array


signature:
def update ( self , data , fromfile = True ) :
    """Принимает текст , или путь к файлу и обновляет существующую базу ."""

implementation:
func = ( self . _parse_from_file if fromfile else self . _parse_from_text )
new_data = tuple ( func ( data ) )
if new_data :
    self . tokens_array += new_data
    self . create_base ( )


signature:
def _parse_from_text ( self , text ) :
    """Возвращает генератор токенов , из текста ."""

implementation:
if not isinstance ( text , str ) :
    raise MarkovTextExcept ( "Передан не текст." )
text = text . strip ( ) . lower ( )
need_start_token = True
token = "$"
# На случай , если переданная строка пуста .
for token in self . WORD_OR_MARKS . finditer ( text ) :
    token = token . group ( )
    if need_start_token :
        need_start_token = False
        yield "^"
    yield token
    if self . END_TOKENS . search ( token ) :
        need_start_token = True
        yield "$"
if token != "$" :
    yield "$"


signature:
def _parse_from_file ( self , file_path ) :
    """см . описание _ parse _ from _ text .
 Только на вход подаётся не текст , а путь к файлу ."""

implementation:
file_path = abspath ( file_path )
if not isfile ( file_path ) :
    raise MarkovTextExcept ( "Передан не файл." )
with open ( file_path , "rb" ) as txt_file :
    for line in txt_file :
        text = line . decode ( "utf-8" , "ignore" ) . strip ( )
        if not text :
            continue
        yield from self . _parse_from_text ( text )


signature:
def push ( self , message ) :
    """Takes a SlackEvent , parses it for a command , and runs against registered plugin"""

implementation:
if self . _ignore_event ( message ) :
    return None , None
args = self . _parse_message ( message )
self . log . debug ( "Searching for command using chunks: %s" , args )
cmd , msg_args = self . _find_longest_prefix_command ( args )
if cmd is not None :
    if message . user is None :
        self . log . debug ( "Discarded message with no originating user: %s" , message )
        return None , None
    sender = message . user . username
    if message . channel is not None :
        sender = "#%s/%s" % ( message . channel . name , sender )
    self . log . info ( "Received from %s: %s, args %s" , sender , cmd , msg_args )
    f = self . _get_command ( cmd , message . user )
    if f :
        if self . _is_channel_ignored ( f , message . channel ) :
            self . log . info ( "Channel %s is ignored, discarding command %s" , message . channel , cmd )
            return '_ignored_' , ""
        return cmd , f . execute ( message , msg_args )
    return '_unauthorized_' , "Sorry, you are not authorized to run %s" % cmd
return None , None


signature:
def _ignore_event ( self , message ) :
    """message _ replied event is not truly a message event and does not have a message . text
 don ' t process such events
 commands may not be idempotent , so ignore message _ changed events ."""

implementation:
if hasattr ( message , 'subtype' ) and message . subtype in self . ignored_events :
    return True
return False


signature:
def register_plugin ( self , plugin ) :
    """Registers a plugin and commands with the dispatcher for push ( )"""

implementation:
self . log . info ( "Registering plugin %s" , type ( plugin ) . __name__ )
self . _register_commands ( plugin )
plugin . on_load ( )


signature:
def acl_show ( self , msg , args ) :
    """Show current allow and deny blocks for the given acl ."""

implementation:
name = args [ 0 ] if len ( args ) > 0 else None
if name is None :
    return "%s: The following ACLs are defined: %s" % ( msg . user , ', ' . join ( self . _acl . keys ( ) ) )
if name not in self . _acl :
    return "Sorry, couldn't find an acl named '%s'" % name
return '\n' . join ( [ "%s: ACL '%s' is defined as follows:" % ( msg . user , name ) , "allow: %s" % ', ' . join ( self . _acl [ name ] [ 'allow' ] ) , "deny: %s" % ', ' . join ( self . _acl [ name ] [ 'deny' ] ) ] )


signature:
def add_user_to_allow ( self , name , user ) :
    """Add a user to the given acl allow block ."""

implementation:
# Clear user from both allow and deny before adding
if not self . remove_user_from_acl ( name , user ) :
    return False
if name not in self . _acl :
    return False
self . _acl [ name ] [ 'allow' ] . append ( user )
return True


signature:
def remove_user_from_acl ( self , name , user ) :
    """Remove a user from the given acl ( both allow and deny ) ."""

implementation:
if name not in self . _acl :
    return False
if user in self . _acl [ name ] [ 'allow' ] :
    self . _acl [ name ] [ 'allow' ] . remove ( user )
if user in self . _acl [ name ] [ 'deny' ] :
    self . _acl [ name ] [ 'deny' ] . remove ( user )
return True


signature:
def create_acl ( self , name ) :
    """Create a new acl ."""

implementation:
if name in self . _acl :
    return False
self . _acl [ name ] = { 'allow' : [ ] , 'deny' : [ ] }
return True


signature:
def delete_acl ( self , name ) :
    """Delete an acl ."""

implementation:
if name not in self . _acl :
    return False
del self . _acl [ name ]
return True


signature:
def mongo ( daemon = False , port = 20771 ) :
    '''Run the mongod process .'''

implementation:
cmd = "mongod --port {0}" . format ( port )
if daemon :
    cmd += " --fork"
run ( cmd )


signature:
def proxy_factory ( BaseSchema , label , ProxiedClass , get_key ) :
    """Create a proxy to a class instance stored in ` ` proxies ` ` .
 : param class BaseSchema : Base schema ( e . g . ` ` StoredObject ` ` )
 : param str label : Name of class variable to set
 : param class ProxiedClass : Class to get or create
 : param function get _ key : Extension - specific key function ; may return e . g .
 the current Flask request"""

implementation:
def local ( ) :
    key = get_key ( )
    try :
        return proxies [ BaseSchema ] [ label ] [ key ]
    except KeyError :
        proxies [ BaseSchema ] [ label ] [ key ] = ProxiedClass ( )
        return proxies [ BaseSchema ] [ label ] [ key ]
return LocalProxy ( local )


signature:
def with_proxies ( proxy_map , get_key ) :
    """Class decorator factory ; adds proxy class variables to target class .
 : param dict proxy _ map : Mapping between class variable labels and proxied
 classes
 : param function get _ key : Extension - specific key function ; may return e . g .
 the current Flask request"""

implementation:
def wrapper ( cls ) :
    for label , ProxiedClass in six . iteritems ( proxy_map ) :
        proxy = proxy_factory ( cls , label , ProxiedClass , get_key )
        setattr ( cls , label , proxy )
    return cls
return wrapper


signature:
def _to_primary_key ( self , value ) :
    """Return primary key ; if value is StoredObject , verify
 that it is loaded ."""

implementation:
if value is None :
    return None
if isinstance ( value , self . base_class ) :
    if not value . _is_loaded :
        raise exceptions . DatabaseError ( 'Record must be loaded.' )
    return value . _primary_key
return self . base_class . _to_primary_key ( value )


signature:
def set_nested ( data , value , * keys ) :
    """Assign to a nested dictionary .
 : param dict data : Dictionary to mutate
 : param value : Value to set
 : param list * keys : List of nested keys
 > > > data = { }
 > > > set _ nested ( data , ' hi ' , ' k0 ' , ' k1 ' , ' k2 ' )
 > > > data
 { ' k0 ' : { ' k1 ' : { ' k2 ' : ' hi ' } } }"""

implementation:
if len ( keys ) == 1 :
    data [ keys [ 0 ] ] = value
else :
    if keys [ 0 ] not in data :
        data [ keys [ 0 ] ] = { }
    set_nested ( data [ keys [ 0 ] ] , value , * keys [ 1 : ] )


signature:
def get_by_username ( self , username ) :
    """Retrieve user by username"""

implementation:
res = filter ( lambda x : x . username == username , self . users . values ( ) )
if len ( res ) > 0 :
    return res [ 0 ]
return None


signature:
def set ( self , user ) :
    """Adds a user object to the user manager
 user - a SlackUser object"""

implementation:
self . log . info ( "Loading user information for %s/%s" , user . id , user . username )
self . load_user_info ( user )
self . log . info ( "Loading user rights for %s/%s" , user . id , user . username )
self . load_user_rights ( user )
self . log . info ( "Added user: %s/%s" , user . id , user . username )
self . _add_user_to_cache ( user )
return user


signature:
def load_user_rights ( self , user ) :
    """Sets permissions on user object"""

implementation:
if user . username in self . admins :
    user . is_admin = True
elif not hasattr ( user , 'is_admin' ) :
    user . is_admin = False


signature:
def send_message ( self , channel , text ) :
    """Used to send a message to the specified channel .
 * channel - can be a channel or user
 * text - message to send"""

implementation:
if isinstance ( channel , SlackIM ) or isinstance ( channel , SlackUser ) :
    self . _bot . send_im ( channel , text )
elif isinstance ( channel , SlackRoom ) :
    self . _bot . send_message ( channel , text )
elif isinstance ( channel , basestring ) :
    if channel [ 0 ] == '@' :
        self . _bot . send_im ( channel [ 1 : ] , text )
    elif channel [ 0 ] == '#' :
        self . _bot . send_message ( channel [ 1 : ] , text )
    else :
        self . _bot . send_message ( channel , text )
else :
    self . _bot . send_message ( channel , text )


signature:
def start_timer ( self , duration , func , * args ) :
    """Schedules a function to be called after some period of time .
 * duration - time in seconds to wait before firing
 * func - function to be called
 * args - arguments to pass to the function"""

implementation:
t = threading . Timer ( duration , self . _timer_callback , ( func , args ) )
self . _timer_callbacks [ func ] = t
t . start ( )
self . log . info ( "Scheduled call to %s in %ds" , func . __name__ , duration )


signature:
def stop_timer ( self , func ) :
    """Stops a timer if it hasn ' t fired yet
 * func - the function passed in start _ timer"""

implementation:
if func in self . _timer_callbacks :
    t = self . _timer_callbacks [ func ]
    t . cancel ( )
    del self . _timer_callbacks [ func ]


signature:
def get_user ( self , username ) :
    """Utility function to query slack for a particular user
 : param username : The username of the user to lookup
 : return : SlackUser object or None"""

implementation:
if hasattr ( self . _bot , 'user_manager' ) :
    user = self . _bot . user_manager . get_by_username ( username )
    if user :
        return user
    user = SlackUser . get_user ( self . _bot . sc , username )
    self . _bot . user_manager . set ( user )
    return user
return SlackUser . get_user ( self . _bot . sc , username )


signature:
def print_res ( data ) :
    """Print translate result in a better format
 Args :
 data ( str ) : result"""


signature:
def cmd ( admin_only = False , acl = '*' , aliases = None , while_ignored = False , * args , ** kwargs ) :
    """Decorator to mark plugin functions as commands in the form of ! < cmd _ name >
 * admin _ only - indicates only users in bot _ admin are allowed to execute ( only used if AuthManager is loaded )
 * acl - indicates which ACL to perform permission checks against ( only used if AuthManager is loaded )
 * aliases - register function with additional commands ( i . e . ! alias1 , ! alias2 , etc )
 * while _ ignored - allows a command to be run , even if channel has been ! sleep"""

implementation:
def wrapper ( func ) :
    func . is_cmd = True
    func . is_subcmd = len ( func . __name__ . split ( '_' ) ) > 1
    func . cmd_name = func . __name__ . replace ( '_' , ' ' )
    func . admin_only = admin_only
    func . acl = acl
    func . aliases = aliases
    func . while_ignored = while_ignored
    return func
return wrapper


signature:
def webhook ( * args , ** kwargs ) :
    """Decorator to mark plugin functions as entry points for web calls
 * route - web route to register , uses Flask syntax
 * method - GET / POST , defaults to POST"""

implementation:
def wrapper ( func ) :
    func . is_webhook = True
    func . route = args [ 0 ]
    func . form_params = kwargs . get ( 'form_params' , [ ] )
    func . method = kwargs . get ( 'method' , 'POST' )
    return func
return wrapper


signature:
def _get_underlying_data ( self , instance ) :
    """Return data from raw data store , rather than overridden
 _ _ get _ _ methods . Should NOT be overwritten ."""

implementation:
self . _touch ( instance )
return self . data . get ( instance , None )


signature:
def freeze ( value ) :
    """Cast value to its frozen counterpart ."""

implementation:
if isinstance ( value , list ) :
    return FrozenList ( * value )
if isinstance ( value , dict ) :
    return FrozenDict ( ** value )
return value


signature:
def help ( self , msg , args ) :
    """Displays help for each command"""

implementation:
output = [ ]
if len ( args ) == 0 :
    commands = sorted ( self . _bot . dispatcher . commands . items ( ) , key = itemgetter ( 0 ) )
    commands = filter ( lambda x : x [ 1 ] . is_subcmd is False , commands )
    # Filter commands if auth is enabled , hide _ admin _ commands is enabled , and user is not admin
    if self . _should_filter_help_commands ( msg . user ) :
        commands = filter ( lambda x : x [ 1 ] . admin_only is False , commands )
    for name , cmd in commands :
        output . append ( self . _get_short_help_for_command ( name ) )
else :
    name = '!' + args [ 0 ]
    output = [ self . _get_help_for_command ( name ) ]
return '\n' . join ( output )


signature:
def save ( self , msg , args ) :
    """Causes the bot to write its current state to backend ."""

implementation:
self . send_message ( msg . channel , "Saving current state..." )
self . _bot . plugins . save_state ( )
self . send_message ( msg . channel , "Done." )


signature:
def shutdown ( self , msg , args ) :
    """Causes the bot to gracefully shutdown ."""

implementation:
self . log . info ( "Received shutdown from %s" , msg . user . username )
self . _bot . runnable = False
return "Shutting down..."


signature:
def whoami ( self , msg , args ) :
    """Prints information about the user and bot version ."""

implementation:
output = [ "Hello %s" % msg . user ]
if hasattr ( self . _bot . dispatcher , 'auth_manager' ) and msg . user . is_admin is True :
    output . append ( "You are a *bot admin*." )
output . append ( "Bot version: %s-%s" % ( self . _bot . version , self . _bot . commit ) )
return '\n' . join ( output )


signature:
def sleep ( self , channel ) :
    """Causes the bot to ignore all messages from the channel .
 Usage :
 ! sleep [ channel name ] - ignore the specified channel ( or current if none specified )"""

implementation:
self . log . info ( 'Sleeping in %s' , channel )
self . _bot . dispatcher . ignore ( channel )
self . send_message ( channel , 'Good night' )


signature:
def wake ( self , channel ) :
    """Causes the bot to resume operation in the channel .
 Usage :
 ! wake [ channel name ] - unignore the specified channel ( or current if none specified )"""

implementation:
self . log . info ( 'Waking up in %s' , channel )
self . _bot . dispatcher . unignore ( channel )
self . send_message ( channel , 'Hello, how may I be of service?' )


signature:
def _arg_name ( self , name , types , prefix = "--" ) :
    if 'type:a10_nullable' in types :
        return self . _arg_name ( name , types [ 'type:a10_nullable' ] , prefix )
    if 'type:a10_list' in types :
        return self . _arg_name ( name , types [ 'type:a10_list' ] , prefix )
    if 'type:a10_reference' in types :
        if name . endswith ( '_id' ) :
            name = name [ : - 3 ]
    """- - shish - kabob it"""

implementation: return prefix + name . replace ( '_' , '-' )


signature:
def _sort_by ( key ) :
    """High order function for sort methods ."""

implementation:
@ staticmethod
def sort_by ( p_list , reverse = False ) :
    return sorted ( p_list , key = lambda p : getattr ( p , key ) , reverse = reverse , )
return sort_by


signature:
def select ( self , filters = all_true , recursive = True ) :
    """Select path by criterion .
 : param filters : a lambda function that take a ` pathlib . Path ` as input ,
 boolean as a output .
 : param recursive : include files in subfolder or not .
 * * 中文文档 * *
 根据filters中定义的条件选择路径 。"""

implementation:
self . assert_is_dir_and_exists ( )
if recursive :
    for p in self . glob ( "**/*" ) :
        if filters ( p ) :
            yield p
else :
    for p in self . iterdir ( ) :
        if filters ( p ) :
            yield p


signature:
def select_file ( self , filters = all_true , recursive = True ) :
    """Select file path by criterion .
 * * 中文文档 * *
 根据filters中定义的条件选择文件 。"""

implementation:
for p in self . select ( filters , recursive ) :
    if p . is_file ( ) :
        yield p


signature:
def select_dir ( self , filters = all_true , recursive = True ) :
    """Select dir path by criterion .
 * * 中文文档 * *
 根据filters中定义的条件选择文件夹 。"""

implementation:
for p in self . select ( filters , recursive ) :
    if p . is_dir ( ) :
        yield p


signature:
def n_file ( self ) :
    """Count how many files in this directory . Including file in sub folder ."""

implementation:
self . assert_is_dir_and_exists ( )
n = 0
for _ in self . select_file ( recursive = True ) :
    n += 1
return n


signature:
def n_dir ( self ) :
    """Count how many folders in this directory . Including folder in sub folder ."""

implementation:
self . assert_is_dir_and_exists ( )
n = 0
for _ in self . select_dir ( recursive = True ) :
    n += 1
return n


signature:
def n_subfile ( self ) :
    """Count how many files in this directory ( doesn ' t include files in
 sub folders ) ."""

implementation:
self . assert_is_dir_and_exists ( )
n = 0
for _ in self . select_file ( recursive = False ) :
    n += 1
return n


signature:
def n_subdir ( self ) :
    """Count how many folders in this directory ( doesn ' t include folder in
 sub folders ) ."""

implementation:
self . assert_is_dir_and_exists ( )
n = 0
for _ in self . select_dir ( recursive = False ) :
    n += 1
return n


signature:
def select_by_ext ( self , ext , recursive = True ) :
    """Select file path by extension .
 : param ext :
 * * 中文文档 * *
 选择与预定义的若干个扩展名匹配的文件 。"""

implementation:
ext = [ ext . strip ( ) . lower ( ) for ext in ensure_list ( ext ) ]
def filters ( p ) :
    return p . suffix . lower ( ) in ext
return self . select_file ( filters , recursive )


signature:
def select_by_pattern_in_fname ( self , pattern , recursive = True , case_sensitive = False ) :
    """Select file path by text pattern in file name .
 * * 中文文档 * *
 选择文件名中包含指定子字符串的文件 。"""

implementation:
if case_sensitive :
    def filters ( p ) :
        return pattern in p . fname
else :
    pattern = pattern . lower ( )
    def filters ( p ) :
        return pattern in p . fname . lower ( )
return self . select_file ( filters , recursive )


signature:
def select_by_pattern_in_abspath ( self , pattern , recursive = True , case_sensitive = False ) :
    """Select file path by text pattern in absolute path .
 * * 中文文档 * *
 选择绝对路径中包含指定子字符串的文件 。"""

implementation:
if case_sensitive :
    def filters ( p ) :
        return pattern in p . abspath
else :
    pattern = pattern . lower ( )
    def filters ( p ) :
        return pattern in p . abspath . lower ( )
return self . select_file ( filters , recursive )


signature:
def select_by_size ( self , min_size = 0 , max_size = 1 << 40 , recursive = True ) :
    """Select file path by size .
 * * 中文文档 * *
 选择所有文件大小在一定范围内的文件 。"""

implementation:
def filters ( p ) :
    return min_size <= p . size <= max_size
return self . select_file ( filters , recursive )


signature:
def select_by_mtime ( self , min_time = 0 , max_time = ts_2100 , recursive = True ) :
    """Select file path by modify time .
 : param min _ time : lower bound timestamp
 : param max _ time : upper bound timestamp
 * * 中文文档 * *
 选择所有 : attr : ` pathlib _ mate . pathlib2 . Path . mtime ` 在一定范围内的文件 。"""

implementation:
def filters ( p ) :
    return min_time <= p . mtime <= max_time
return self . select_file ( filters , recursive )


signature:
def select_by_atime ( self , min_time = 0 , max_time = ts_2100 , recursive = True ) :
    """Select file path by access time .
 : param min _ time : lower bound timestamp
 : param max _ time : upper bound timestamp
 * * 中文文档 * *
 选择所有 : attr : ` pathlib _ mate . pathlib2 . Path . atime ` 在一定范围内的文件 。"""

implementation:
def filters ( p ) :
    return min_time <= p . atime <= max_time
return self . select_file ( filters , recursive )


signature:
def select_by_ctime ( self , min_time = 0 , max_time = ts_2100 , recursive = True ) :
    """Select file path by create time .
 : param min _ time : lower bound timestamp
 : param max _ time : upper bound timestamp
 * * 中文文档 * *
 选择所有 : attr : ` pathlib _ mate . pathlib2 . Path . ctime ` 在一定范围内的文件 。"""

implementation:
def filters ( p ) :
    return min_time <= p . ctime <= max_time
return self . select_file ( filters , recursive )


signature:
def dirsize ( self ) :
    """Return total file size ( include sub folder ) . Symlink doesn ' t count ."""

implementation:
total = 0
for p in self . select_file ( recursive = True ) :
    try :
        total += p . size
    except : # pragma : no cover
        print ( "Unable to get file size of: %s" % p )
return total


signature:
def make_zip_archive ( self , dst = None , filters = all_true , compress = True , overwrite = False , makedirs = False , verbose = False ) : # pragma : no cover
    """Make a zip archive .
 : param dst : output file path . if not given , will be automatically assigned .
 : param filters : custom path filter . By default it allows any file .
 : param compress : compress or not .
 : param overwrite : overwrite exists or not .
 : param verbose : display log or not .
 : return :"""

implementation:
self . assert_exists ( )
if dst is None :
    dst = self . _auto_zip_archive_dst ( )
else :
    dst = self . change ( new_abspath = dst )
if not dst . basename . lower ( ) . endswith ( ".zip" ) :
    raise ValueError ( "zip archive name has to be endswith '.zip'!" )
if dst . exists ( ) :
    if not overwrite :
        raise IOError ( "'%s' already exists!" % dst )
if compress :
    compression = ZIP_DEFLATED
else :
    compression = ZIP_STORED
if not dst . parent . exists ( ) :
    if makedirs :
        os . makedirs ( dst . parent . abspath )
if verbose :
    msg = "Making zip archive for '%s' ..." % self
    print ( msg )
current_dir = os . getcwd ( )
if self . is_dir ( ) :
    total_size = 0
    selected = list ( )
    for p in self . glob ( "**/*" ) :
        if filters ( p ) :
            selected . append ( p )
            total_size += p . size
    if verbose :
        msg = "Got {} files, total size is {}, compressing ..." . format ( len ( selected ) , repr_data_size ( total_size ) , )
        print ( msg )
    with ZipFile ( dst . abspath , "w" , compression ) as f :
        os . chdir ( self . abspath )
        for p in selected :
            relpath = p . relative_to ( self ) . __str__ ( )
            f . write ( relpath )
elif self . is_file ( ) :
    with ZipFile ( dst . abspath , "w" , compression ) as f :
        os . chdir ( self . parent . abspath )
        f . write ( self . basename )
os . chdir ( current_dir )
if verbose :
    msg = "Complete! Archive size is {}." . format ( dst . size_in_text )
    print ( msg )


signature:
def backup ( self , dst = None , ignore = None , ignore_ext = None , ignore_pattern = None , ignore_size_smaller_than = None , ignore_size_larger_than = None , case_sensitive = False ) : # pragma : no cover
    """Create a compressed zip archive backup for a directory .
 : param dst : the output file path .
 : param ignore : file or directory defined in this list will be ignored .
 : param ignore _ ext : file with extensions defined in this list will be ignored .
 : param ignore _ pattern : any file or directory that contains this pattern
 will be ignored .
 : param ignore _ size _ smaller _ than : any file size smaller than this
 will be ignored .
 : param ignore _ size _ larger _ than : any file size larger than this
 will be ignored .
 * * 中文文档 * *
 为一个目录创建一个备份压缩包 。 可以通过过滤器选择你要备份的文件 。"""

implementation:
def preprocess_arg ( arg ) : # pragma : no cover
    if arg is None :
        return [ ]
    if isinstance ( arg , ( tuple , list ) ) :
        return list ( arg )
    else :
        return [ arg , ]
self . assert_is_dir_and_exists ( )
ignore = preprocess_arg ( ignore )
for i in ignore :
    if i . startswith ( "/" ) or i . startswith ( "\\" ) :
        raise ValueError
ignore_ext = preprocess_arg ( ignore_ext )
for ext in ignore_ext :
    if not ext . startswith ( "." ) :
        raise ValueError
ignore_pattern = preprocess_arg ( ignore_pattern )
if case_sensitive :
    pass
else :
    ignore = [ i . lower ( ) for i in ignore ]
    ignore_ext = [ i . lower ( ) for i in ignore_ext ]
    ignore_pattern = [ i . lower ( ) for i in ignore_pattern ]
def filters ( p ) :
    relpath = p . relative_to ( self ) . abspath
    if not case_sensitive :
        relpath = relpath . lower ( )
    # ignore
    for i in ignore :
        if relpath . startswith ( i ) :
            return False
    # ignore _ ext
    if case_sensitive :
        ext = p . ext
    else :
        ext = p . ext . lower ( )
    if ext in ignore_ext :
        return False
    # ignore _ pattern
    for pattern in ignore_pattern :
        if pattern in relpath :
            return False
    # ignore _ size _ smaller _ than
    if ignore_size_smaller_than :
        if p . size < ignore_size_smaller_than :
            return False
    # ignore _ size _ larger _ than
    if ignore_size_larger_than :
        if p . size > ignore_size_larger_than :
            return False
    return True
self . make_zip_archive ( dst = dst , filters = filters , compress = True , overwrite = False , verbose = True , )


signature:
def acquire_lock ( func ) :
    """Decorate methods when locking repository is required ."""

implementation:
@ wraps ( func )
def wrapper ( self , * args , ** kwargs ) :
    with self . locker as r : # get the result
        acquired , code , _ = r
        if acquired :
            try :
                r = func ( self , * args , ** kwargs )
            except Exception as err :
                e = str ( err )
            else :
                e = None
        else :
            warnings . warn ( "code %s. Unable to aquire the lock when calling '%s'. You may try again!" % ( code , func . __name__ ) )
            e = None
            r = None
    # raise error after exiting with statement and releasing the lock !
    if e is not None :
        traceback . print_stack ( )
        raise Exception ( e )
    return r
return wrapper


signature:
def sync_required ( func ) :
    """Decorate methods when synchronizing repository is required ."""

implementation:
@ wraps ( func )
def wrapper ( self , * args , ** kwargs ) :
    if not self . _keepSynchronized :
        r = func ( self , * args , ** kwargs )
    else :
        state = self . _load_state ( )
        # print ( " - - - - - > " , state , self . state )
        if state is None :
            r = func ( self , * args , ** kwargs )
        elif state == self . state :
            r = func ( self , * args , ** kwargs )
        else :
            warnings . warn ( "Repository at '%s' is out of date. Need to load it again to avoid conflict." % self . path )
            r = None
    return r
return wrapper


signature:
def get_pickling_errors ( obj , seen = None ) :
    """Investigate pickling errors ."""

implementation:
if seen == None :
    seen = [ ]
if hasattr ( obj , "__getstate__" ) :
    state = obj . __getstate__ ( )
# elif hasattr ( obj , " _ _ dict _ _ " ) :
# state = obj . _ _ dict _ _
else :
    return None
# try :
# state = obj . _ _ getstate _ _ ( )
# except AttributeError as e :
# # state = obj . _ _ dict _ _
# return str ( e )
if state == None :
    return 'object state is None'
if isinstance ( state , tuple ) :
    if not isinstance ( state [ 0 ] , dict ) :
        state = state [ 1 ]
    else :
        state = state [ 0 ] . update ( state [ 1 ] )
result = { }
for i in state :
    try :
        pickle . dumps ( state [ i ] , protocol = 2 )
    except pickle . PicklingError as e :
        if not state [ i ] in seen :
            seen . append ( state [ i ] )
            result [ i ] = get_pickling_errors ( state [ i ] , seen )
return result


signature:
def get_list_representation ( self ) :
    """Gets a representation of the Repository content in a list of directories ( files ) format .
 : Returns :
 # . repr ( list ) : The list representation of the Repository content ."""

implementation:
if self . __path is None :
    return [ ]
repr = [ self . __path + ":[" + ',' . join ( list ( dict . __getitem__ ( self , 'files' ) ) ) + ']' ]
# walk directories
for directory in sorted ( list ( self . walk_directories_relative_path ( ) ) ) :
    directoryRepr = os . path . normpath ( directory )
    # get directory info
    dirInfoDict , errorMessage = self . get_directory_info ( directory )
    assert dirInfoDict is not None , errorMessage
    directoryRepr += ":[" + ',' . join ( list ( dict . __getitem__ ( dirInfoDict , 'files' ) ) ) + ']'
    repr . append ( directoryRepr )
return repr


signature:
def walk_files_relative_path ( self , relativePath = "" ) :
    """Walk the repository and yield all found files relative path joined with file name .
 : parameters :
 # . relativePath ( str ) : The relative path from which start the walk ."""

implementation:
def walk_files ( directory , relativePath ) :
    directories = dict . __getitem__ ( directory , 'directories' )
    files = dict . __getitem__ ( directory , 'files' )
    for f in sorted ( files ) :
        yield os . path . join ( relativePath , f )
    for k in sorted ( dict . keys ( directories ) ) :
        path = os . path . join ( relativePath , k )
        dir = directories . __getitem__ ( k )
        for e in walk_files ( dir , path ) :
            yield e
dir , errorMessage = self . get_directory_info ( relativePath )
assert dir is not None , errorMessage
return walk_files ( dir , relativePath = '' )


signature:
def walk_files_info ( self , relativePath = "" ) :
    """Walk the repository and yield tuples as the following : \n
 ( relative path to relativePath joined with file name , file info dict ) .
 : parameters :
 # . relativePath ( str ) : The relative path from which start the walk ."""

implementation:
def walk_files ( directory , relativePath ) :
    directories = dict . __getitem__ ( directory , 'directories' )
    files = dict . __getitem__ ( directory , 'files' )
    for fname in sorted ( files ) :
        info = dict . __getitem__ ( files , fname )
        yield os . path . join ( relativePath , fname ) , info
    for k in sorted ( dict . keys ( directories ) ) :
        path = os . path . join ( relativePath , k )
        dir = dict . __getitem__ ( directories , k )
        for e in walk_files ( dir , path ) :
            yield e
dir , errorMessage = self . get_directory_info ( relativePath )
assert dir is not None , errorMessage
return walk_files ( dir , relativePath = '' )


signature:
def walk_directories_relative_path ( self , relativePath = "" ) :
    """Walk repository and yield all found directories relative path
 : parameters :
 # . relativePath ( str ) : The relative path from which start the walk ."""

implementation:
def walk_directories ( directory , relativePath ) :
    directories = dict . __getitem__ ( directory , 'directories' )
    dirNames = dict . keys ( directories )
    for d in sorted ( dirNames ) :
        yield os . path . join ( relativePath , d )
    for k in sorted ( dict . keys ( directories ) ) :
        path = os . path . join ( relativePath , k )
        dir = dict . __getitem__ ( directories , k )
        for e in walk_directories ( dir , path ) :
            yield e
dir , errorMessage = self . get_directory_info ( relativePath )
assert dir is not None , errorMessage
return walk_directories ( dir , relativePath = '' )


signature:
def walk_directories_info ( self , relativePath = "" ) :
    """Walk repository and yield all found directories relative path .
 : parameters :
 # . relativePath ( str ) : The relative path from which start the walk ."""

implementation:
def walk_directories ( directory , relativePath ) :
    directories = dict . __getitem__ ( directory , 'directories' )
    for fname in sorted ( directories ) :
        info = dict . __getitem__ ( directories , fname )
        yield os . path . join ( relativePath , fname ) , info
    for k in sorted ( dict . keys ( directories ) ) :
        path = os . path . join ( relativePath , k )
        dir = dict . __getitem__ ( directories , k )
        for e in walk_directories ( dir , path ) :
            yield e
dir , errorMessage = self . get_directory_info ( relativePath )
assert dir is not None , errorMessage
return walk_directories ( dir , relativePath = '' )


signature:
def walk_directory_directories_relative_path ( self , relativePath = "" ) :
    """Walk a certain directory in repository and yield all found directories relative path .
 : parameters :
 # . relativePath ( str ) : The relative path of the directory ."""

implementation:
# get directory info dict
errorMessage = ""
relativePath = os . path . normpath ( relativePath )
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
assert dirInfoDict is not None , errorMessage
for dname in dict . __getitem__ ( dirInfoDict , "directories" ) :
    yield os . path . join ( relativePath , dname )


signature:
def walk_directory_directories_info ( self , relativePath = "" ) :
    """Walk a certain directory in repository and yield tuples as the following : \n
 ( relative path joined with directory name , file info dict ) .
 : parameters :
 # . relativePath ( str ) : The relative path of the directory ."""

implementation:
# get directory info dict
relativePath = os . path . normpath ( relativePath )
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
assert dirInfoDict is not None , errorMessage
for fname in dict . __getitem__ ( dirInfoDict , "directories" ) :
    yield os . path . join ( relativePath , fname ) , dict . __getitem__ ( dirInfoDict , "directories" ) [ fname ]


signature:
def synchronize ( self , verbose = False ) :
    """Synchronizes the Repository information with the directory .
 All registered but missing files and directories in the directory ,
 will be automatically removed from the Repository .
 : parameters :
 # . verbose ( boolean ) : Whether to be warn and inform about any abnormalities ."""

implementation:
if self . __path is None :
    return
# walk directories
for dirPath in sorted ( list ( self . walk_directories_relative_path ( ) ) ) :
    realPath = os . path . join ( self . __path , dirPath )
    # if directory exist
    if os . path . isdir ( realPath ) :
        continue
    if verbose :
        warnings . warn ( "%s directory is missing" % realPath )
    # loop to get dirInfoDict
    keys = dirPath . split ( os . sep )
    dirInfoDict = self
    for idx in range ( len ( keys ) - 1 ) :
        dirs = dict . get ( dirInfoDict , 'directories' , None )
        if dirs is None :
            break
        dirInfoDict = dict . get ( dirs , keys [ idx ] , None )
        if dirInfoDict is None :
            break
    # remove dirInfoDict directory if existing
    if dirInfoDict is not None :
        dirs = dict . get ( dirInfoDict , 'directories' , None )
        if dirs is not None :
            dict . pop ( dirs , keys [ - 1 ] , None )
# walk files
for filePath in sorted ( list ( self . walk_files_relative_path ( ) ) ) :
    realPath = os . path . join ( self . __path , filePath )
    # if file exists
    if os . path . isfile ( realPath ) :
        continue
    if verbose :
        warnings . warn ( "%s file is missing" % realPath )
    # loop to get dirInfoDict
    keys = filePath . split ( os . sep )
    dirInfoDict = self
    for idx in range ( len ( keys ) - 1 ) :
        dirs = dict . get ( dirInfoDict , 'directories' , None )
        if dirs is None :
            break
        dirInfoDict = dict . get ( dirs , keys [ idx ] , None )
        if dirInfoDict is None :
            break
    # remove dirInfoDict file if existing
    if dirInfoDict is not None :
        files = dict . get ( dirInfoDict , 'files' , None )
        if files is not None :
            dict . pop ( files , keys [ - 1 ] , None )


signature:
def load_repository ( self , path ) :
    """Load repository from a directory path and update the current instance .
 : Parameters :
 # . path ( string ) : The path of the directory from where to load the repository .
 If ' . ' or an empty string is passed , the current working directory will be used .
 : Returns :
 # . repository ( pyrep . Repository ) : returns self repository with loaded data ."""

implementation:
# try to open
if path . strip ( ) in ( '' , '.' ) :
    path = os . getcwd ( )
repoPath = os . path . realpath ( os . path . expanduser ( path ) )
if not self . is_repository ( repoPath ) :
    raise Exception ( "no repository found in '%s'" % str ( repoPath ) )
# get pyrepinfo path
repoInfoPath = os . path . join ( repoPath , ".pyrepinfo" )
try :
    fd = open ( repoInfoPath , 'rb' )
except Exception as e :
    raise Exception ( "unable to open repository file(%s)" % e )
# before doing anything try to lock repository
# can ' t decorate with @ acquire _ lock because this will point to old repository
# path or to current working directory which might not be the path anyways
L = Locker ( filePath = None , lockPass = str ( uuid . uuid1 ( ) ) , lockPath = os . path . join ( repoPath , ".pyreplock" ) )
acquired , code = L . acquire_lock ( )
# check if acquired .
if not acquired :
    warnings . warn ( "code %s. Unable to aquire the lock when calling 'load_repository'. You may try again!" % ( code , ) )
    return
try : # unpickle file
    try :
        repo = pickle . load ( fd )
    except Exception as e :
        fd . close ( )
        raise Exception ( "unable to pickle load repository (%s)" % e )
    finally :
        fd . close ( )
    # check if it ' s a PyrepInfo instance
    if not isinstance ( repo , Repository ) :
        raise Exception ( ".pyrepinfo in '%s' is not a repository instance." % s )
    else : # update info path
        self . __reset_repository ( )
        self . __update_repository ( repo )
        self . __path = repoPath
    # set timestamp
    self . __state = self . _get_or_create_state ( )
except Exception as e :
    L . release_lock ( )
    raise Exception ( e )
finally :
    L . release_lock ( )
# set loaded repo locker path to L because repository have been moved to another directory
self . __locker = L
# return
return self


signature:
def create_repository ( self , path , info = None , verbose = True ) :
    """create a repository in a directory .
 This method insures the creation of the directory in the system if it is missing . \n
 * * N . B . This method erases existing pyrep repository in the path but not the repository files . * *
 : Parameters :
 # . path ( string ) : The real absolute path where to create the Repository .
 If ' . ' or an empty string is passed , the current working directory will be used .
 # . info ( None , object ) : Any information that can identify the repository .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
try :
    info = copy . deepcopy ( info )
except :
    raise Exception ( "Repository info must be a copyable python object." )
# get real path
if path . strip ( ) in ( '' , '.' ) :
    path = os . getcwd ( )
realPath = os . path . realpath ( os . path . expanduser ( path ) )
# create directory if not existing
if not os . path . isdir ( realPath ) :
    os . makedirs ( realPath )
self . __path = realPath
self . __info = info
# reset if replace is set to True
if self . is_repository ( realPath ) :
    if verbose :
        warnings . warn ( "A pyrep Repository already exists in the given path '%s' and therefore it has been erased and replaced by a fresh repository." % path )
# reset repository
self . __reset_repository ( )
# update locker because normally this is done in _ _ update _ repository method
lp = '.pyreplock'
if self . __path is not None :
    lp = os . path . join ( self . __path , lp )
self . __locker . set_lock_path ( lp )
self . __locker . set_lock_pass ( str ( uuid . uuid1 ( ) ) )
# save repository
self . save ( )


signature:
def get_repository ( self , path , info = None , verbose = True ) :
    """Create a repository at given real path or load any existing one .
 This method insures the creation of the directory in the system if it is missing . \n
 Unlike create _ repository , this method doesn ' t erase any existing repository
 in the path but loads it instead .
 * * N . B . On some systems and some paths , creating a directory may requires root permissions . * *
 : Parameters :
 # . path ( string ) : The real absolute path where to create the Repository .
 If ' . ' or an empty string is passed , the current working directory will be used .
 # . info ( None , object ) : Any information that can identify the repository .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
# get real path
if path . strip ( ) in ( '' , '.' ) :
    path = os . getcwd ( )
realPath = os . path . realpath ( os . path . expanduser ( path ) )
# create directory if not existing
if not os . path . isdir ( realPath ) :
    os . makedirs ( realPath )
# create Repository
if not self . is_repository ( realPath ) :
    self . create_repository ( realPath , info = info , verbose = verbose )
else :
    self . load_repository ( realPath )


signature:
def remove_repository ( self , path = None , relatedFiles = False , relatedFolders = False , verbose = True ) :
    """Remove . pyrepinfo file from path if exists and related files and directories
 when respective flags are set to True .
 : Parameters :
 # . path ( None , string ) : The path of the directory where to remove an existing repository .
 If None , current repository is removed if initialized .
 # . relatedFiles ( boolean ) : Whether to also remove all related files from system as well .
 # . relatedFolders ( boolean ) : Whether to also remove all related directories from system as well .
 Directories will be removed only if they are left empty after removing the files .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
if path is not None :
    realPath = os . path . realpath ( os . path . expanduser ( path ) )
else :
    realPath = self . __path
if realPath is None :
    if verbose :
        warnings . warn ( 'path is None and current Repository is not initialized!' )
    return
if not self . is_repository ( realPath ) :
    if verbose :
        warnings . warn ( "No repository found in '%s'!" % realPath )
    return
# check for security
if realPath == os . path . realpath ( '/..' ) :
    if verbose :
        warnings . warn ( 'You are about to wipe out your system !!! action aboarded' )
    return
# get repo
if path is not None :
    repo = Repository ( )
    repo . load_repository ( realPath )
else :
    repo = self
# delete files
if relatedFiles :
    for relativePath in repo . walk_files_relative_path ( ) :
        realPath = os . path . join ( repo . path , relativePath )
        if not os . path . isfile ( realPath ) :
            continue
        if not os . path . exists ( realPath ) :
            continue
        os . remove ( realPath )
# delete directories
if relatedFolders :
    for relativePath in reversed ( list ( repo . walk_directories_relative_path ( ) ) ) :
        realPath = os . path . join ( repo . path , relativePath )
        # protect from wiping out the system
        if not os . path . isdir ( realPath ) :
            continue
        if not os . path . exists ( realPath ) :
            continue
        if not len ( os . listdir ( realPath ) ) :
            os . rmdir ( realPath )
# delete repository
os . remove ( os . path . join ( repo . path , ".pyrepinfo" ) )
for fname in ( ".pyrepstate" , ".pyreplock" ) :
    p = os . path . join ( repo . path , fname )
    if os . path . exists ( p ) :
        os . remove ( p )
# remove main directory if empty
if os . path . isdir ( repo . path ) :
    if not len ( os . listdir ( repo . path ) ) :
        os . rmdir ( repo . path )
# reset repository
repo . __reset_repository ( )


signature:
def save ( self ) :
    """Save repository . pyrepinfo to disk ."""

implementation:
# open file
repoInfoPath = os . path . join ( self . __path , ".pyrepinfo" )
try :
    fdinfo = open ( repoInfoPath , 'wb' )
except Exception as e :
    raise Exception ( "unable to open repository info for saving (%s)" % e )
# save repository
try :
    pickle . dump ( self , fdinfo , protocol = 2 )
except Exception as e :
    fdinfo . flush ( )
    os . fsync ( fdinfo . fileno ( ) )
    fdinfo . close ( )
    raise Exception ( "Unable to save repository info (%s)" % e )
finally :
    fdinfo . flush ( )
    os . fsync ( fdinfo . fileno ( ) )
    fdinfo . close ( )
# save timestamp
repoTimePath = os . path . join ( self . __path , ".pyrepstate" )
try :
    self . __state = ( "%.6f" % time . time ( ) ) . encode ( )
    with open ( repoTimePath , 'wb' ) as fdtime :
        fdtime . write ( self . __state )
        fdtime . flush ( )
        os . fsync ( fdtime . fileno ( ) )
except Exception as e :
    raise Exception ( "unable to open repository time stamp for saving (%s)" % e )


signature:
def create_package ( self , path = None , name = None , mode = None ) :
    """Create a tar file package of all the repository files and directories .
 Only files and directories that are stored in the repository info
 are stored in the package tar file .
 * * N . B . On some systems packaging requires root permissions . * *
 : Parameters :
 # . path ( None , string ) : The real absolute path where to create the package .
 If None , it will be created in the same directory as the repository
 If ' . ' or an empty string is passed , the current working directory will be used .
 # . name ( None , string ) : The name to give to the package file
 If None , the package directory name will be used with the appropriate extension added .
 # . mode ( None , string ) : The writing mode of the tarfile .
 If None , automatically the best compression mode will be chose .
 Available modes are ( ' w ' , ' w : ' , ' w : gz ' , ' w : bz2 ' )"""

implementation:
# check mode
assert mode in ( None , 'w' , 'w:' , 'w:gz' , 'w:bz2' ) , 'unkown archive mode %s' % str ( mode )
if mode is None :
    mode = 'w:bz2'
    mode = 'w:'
# get root
if path is None :
    root = os . path . split ( self . __path ) [ 0 ]
elif path . strip ( ) in ( '' , '.' ) :
    root = os . getcwd ( )
else :
    root = os . path . realpath ( os . path . expanduser ( path ) )
assert os . path . isdir ( root ) , 'absolute path %s is not a valid directory' % path
# get name
if name is None :
    ext = mode . split ( ":" )
    if len ( ext ) == 2 :
        if len ( ext [ 1 ] ) :
            ext = "." + ext [ 1 ]
        else :
            ext = '.tar'
    else :
        ext = '.tar'
    name = os . path . split ( self . __path ) [ 1 ] + ext
# save repository
self . save ( )
# create tar file
tarfilePath = os . path . join ( root , name )
try :
    tarHandler = tarfile . TarFile . open ( tarfilePath , mode = mode )
except Exception as e :
    raise Exception ( "Unable to create package (%s)" % e )
# walk directory and create empty directories
for directory in sorted ( list ( self . walk_directories_relative_path ( ) ) ) :
    t = tarfile . TarInfo ( directory )
    t . type = tarfile . DIRTYPE
    tarHandler . addfile ( t )
# walk files and add to tar
for file in self . walk_files_relative_path ( ) :
    tarHandler . add ( os . path . join ( self . __path , file ) , arcname = file )
# save repository . pyrepinfo
tarHandler . add ( os . path . join ( self . __path , ".pyrepinfo" ) , arcname = ".pyrepinfo" )
# close tar file
tarHandler . close ( )


signature:
def is_repository ( self , path ) :
    """Check if there is a Repository in path .
 : Parameters :
 # . path ( string ) : The real path of the directory where to check if there is a repository .
 : Returns :
 # . result ( boolean ) : Whether its a repository or not ."""

implementation:
realPath = os . path . realpath ( os . path . expanduser ( path ) )
if not os . path . isdir ( realPath ) :
    return False
if ".pyrepinfo" not in os . listdir ( realPath ) :
    return False
return True


signature:
def get_directory_info ( self , relativePath ) :
    """get directory info from the Repository .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory .
 : Returns :
 # . info ( None , dictionary ) : The directory information dictionary .
 If None , it means an error has occurred .
 # . error ( string ) : The error message if any error occurred ."""

implementation:
relativePath = os . path . normpath ( relativePath )
# if root directory
if relativePath in ( '' , '.' ) :
    return self , ""
currentDir = self . __path
dirInfoDict = self
for dir in relativePath . split ( os . sep ) :
    dirInfoDict = dict . __getitem__ ( dirInfoDict , "directories" )
    currentDir = os . path . join ( currentDir , dir )
    # check if path exists
    if not os . path . exists ( currentDir ) :
        return None , "directory '%s' is not found" % currentDir
    val = dirInfoDict . get ( dir , None )
    # check if directory is registered in repository
    if val is None :
        return None , "directory '%s' is not registered in PyrepInfo" % currentDir
    dirInfoDict = val
return dirInfoDict , ""


signature:
def get_parent_directory_info ( self , relativePath ) :
    """get parent directory info of a file or directory from the Repository .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the file or directory of which the parent directory info is requested .
 : Returns :
 # . info ( None , dictionary ) : The directory information dictionary .
 If None , it means an error has occurred .
 # . error ( string ) : The error message if any error occurred ."""

implementation:
relativePath = os . path . normpath ( relativePath )
# if root directory
if relativePath in ( '' , '.' ) :
    return self , "relativePath is empty pointing to the repostitory itself."
# split path
parentDirPath , _ = os . path . split ( relativePath )
# get parent directory info
return self . get_directory_info ( parentDirPath )


signature:
def get_file_info ( self , relativePath , name = None ) :
    """get file information dict from the repository given its relative path and name .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory where the file is .
 # . name ( string ) : The file name .
 If None is given , name will be split from relativePath .
 : Returns :
 # . info ( None , dictionary ) : The file information dictionary .
 If None , it means an error has occurred .
 # . errorMessage ( string ) : The error message if any error occurred ."""

implementation:
# normalize relative path and name
relativePath = os . path . normpath ( relativePath )
if relativePath == '.' :
    relativePath = ''
    assert name != '.pyrepinfo' , "'.pyrepinfo' can't be a file name."
if name is None :
    assert len ( relativePath ) , "name must be given when relative path is given as empty string or as a simple dot '.'"
    relativePath , name = os . path . split ( relativePath )
# initialize message
errorMessage = ""
# get directory info
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
if dirInfoDict is None :
    return None , errorMessage
# get file info
fileInfo = dict . __getitem__ ( dirInfoDict , "files" ) . get ( name , None )
if fileInfo is None :
    errorMessage = "file %s does not exist in relative path '%s'" % ( name , relativePath )
return fileInfo , errorMessage


signature:
def get_file_info_by_id ( self , id ) :
    """Given an id , get the corresponding file info as the following : \n
 ( relative path joined with file name , file info dict )
 Parameters :
 # . id ( string ) : The file unique id string .
 : Returns :
 # . relativePath ( string ) : The file relative path joined with file name .
 If None , it means file was not found .
 # . info ( None , dictionary ) : The file information dictionary .
 If None , it means file was not found ."""

implementation:
for path , info in self . walk_files_info ( ) :
    if info [ 'id' ] == id :
        return path , info
# none was found
return None , None


signature:
def get_file_relative_path_by_id ( self , id ) :
    """Given an id , get the corresponding file info relative path joined with file name .
 Parameters :
 # . id ( string ) : The file unique id string .
 : Returns :
 # . relativePath ( string ) : The file relative path joined with file name .
 If None , it means file was not found ."""

implementation:
for path , info in self . walk_files_info ( ) :
    if info [ 'id' ] == id :
        return path
# none was found
return None


signature:
def get_file_relative_path_by_name ( self , name , skip = 0 ) :
    """Get file relative path given the file name . If file name is redundant in different
 directories in the repository , this method ensures to return all or some of the
 files according to skip value .
 Parameters :
 # . name ( string ) : The file name .
 # . skip ( None , integer ) : As file names can be identical , skip determines
 the number of satisfying files name to skip before returning . \n
 If None is given , a list of all files relative path will be returned .
 : Returns :
 # . relativePath ( string , list ) : The file relative path .
 If None , it means file was not found . \n
 If skip is None a list of all found files relative paths will be returned ."""

implementation:
if skip is None :
    paths = [ ]
else :
    paths = None
for path , info in self . walk_files_info ( ) :
    _ , n = os . path . split ( path )
    if n == name :
        if skip is None :
            paths . append ( path )
        elif skip > 0 :
            skip -= 1
        else :
            paths = path
            break
return paths


signature:
def add_directory ( self , relativePath , info = None ) :
    """Adds a directory in the repository and creates its
 attribute in the Repository with utc timestamp .
 It insures adding all the missing directories in the path .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory to add in the repository .
 # . info ( None , string , pickable object ) : Any random info about the folder .
 : Returns :
 # . info ( dict ) : The directory info dict ."""

implementation:
path = os . path . normpath ( relativePath )
# create directories
currentDir = self . path
currentDict = self
if path in ( "" , "." ) :
    return currentDict
save = False
for dir in path . split ( os . sep ) :
    dirPath = os . path . join ( currentDir , dir )
    # create directory
    if not os . path . exists ( dirPath ) :
        os . mkdir ( dirPath )
    # create dictionary key
    currentDict = dict . __getitem__ ( currentDict , "directories" )
    if currentDict . get ( dir , None ) is None :
        save = True
        currentDict [ dir ] = { "directories" : { } , "files" : { } , "timestamp" : datetime . utcnow ( ) , "id" : str ( uuid . uuid1 ( ) ) , "info" : info }
    # INFO MUST BE SET ONLY FOR THE LAST DIRECTORY
    currentDict = currentDict [ dir ]
    currentDir = dirPath
# save repository
if save :
    self . save ( )
# return currentDict
return currentDict


signature:
def remove_directory ( self , relativePath , removeFromSystem = False ) :
    """Remove directory from repository .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory to remove from the repository .
 # . removeFromSystem ( boolean ) : Whether to also remove directory and all files from the system . \n
 Only files saved in the repository will be removed and empty left directories ."""

implementation:
# get parent directory info
relativePath = os . path . normpath ( relativePath )
parentDirInfoDict , errorMessage = self . get_parent_directory_info ( relativePath )
assert parentDirInfoDict is not None , errorMessage
# split path
path , name = os . path . split ( relativePath )
if dict . __getitem__ ( parentDirInfoDict , 'directories' ) . get ( name , None ) is None :
    raise Exception ( "'%s' is not a registered directory in repository relative path '%s'" % ( name , path ) )
# remove from system
if removeFromSystem : # remove files
    for rp in self . walk_files_relative_path ( relativePath = relativePath ) :
        ap = os . path . join ( self . __path , relativePath , rp )
        if not os . path . isfile ( ap ) :
            continue
        if not os . path . exists ( ap ) :
            continue
        if os . path . isfile ( ap ) :
            os . remove ( ap )
    # remove directories
    for rp in self . walk_directories_relative_path ( relativePath = relativePath ) :
        ap = os . path . join ( self . __path , relativePath , rp )
        if not os . path . isdir ( ap ) :
            continue
        if not os . path . exists ( ap ) :
            continue
        if not len ( os . listdir ( ap ) ) :
            os . rmdir ( ap )
# pop directory from repo
dict . __getitem__ ( parentDirInfoDict , 'directories' ) . pop ( name , None )
ap = os . path . join ( self . __path , relativePath )
if not os . path . isdir ( ap ) :
    if not len ( os . listdir ( ap ) ) :
        os . rmdir ( ap )
# save repository
self . save ( )


signature:
def move_directory ( self , relativePath , relativeDestination , replace = False , verbose = True ) :
    """Move a directory in the repository from one place to another . It insures moving all the
 files and subdirectories in the system .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory to be moved .
 # . relativeDestination ( string ) : The new relative to the repository path of the directory .
 # . replace ( boolean ) : Whether to replace existing files with the same name in the new created directory .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
# normalize path
relativePath = os . path . normpath ( relativePath )
relativeDestination = os . path . normpath ( relativeDestination )
# get files and directories
filesInfo = list ( self . walk_files_info ( relativePath = relativePath ) )
dirsPath = list ( self . walk_directories_relative_path ( relativePath = relativePath ) )
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
assert dirInfoDict is not None , errorMessage
# remove directory info only
self . remove_directory ( relativePath = relativePath , removeFromSystem = False )
# create new relative path
self . add_directory ( relativeDestination )
# move files
for RP , info in filesInfo :
    source = os . path . join ( self . __path , relativePath , RP )
    destination = os . path . join ( self . __path , relativeDestination , RP )
    # add directory
    newDirRP , fileName = os . path . split ( os . path . join ( relativeDestination , RP ) )
    dirInfoDict = self . add_directory ( newDirRP )
    # move file
    if os . path . isfile ( destination ) :
        if replace :
            os . remove ( destination )
            if verbose :
                warnings . warn ( "file '%s' is copied replacing existing one in destination '%s'." % ( fileName , newDirRP ) )
        else :
            if verbose :
                warnings . warn ( "file '%s' is not copied because the same file exists in destination '%s'." % ( fileName , destination ) )
            continue
    os . rename ( source , destination )
    # set file information
    dict . __getitem__ ( dirInfoDict , "files" ) [ fileName ] = info
# save repository
self . save ( )


signature:
def rename_directory ( self , relativePath , newName , replace = False , verbose = True ) :
    """Rename a directory in the repository . It insures renaming the directory in the system .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory to be renamed .
 # . newName ( string ) : The new directory name .
 # . replace ( boolean ) : Whether to force renaming when new name exists in the system .
 It fails when new folder name is registered in repository .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
# normalize path
relativePath = os . path . normpath ( relativePath )
parentDirInfoDict , errorMessage = self . get_parent_directory_info ( relativePath )
assert parentDirInfoDict is not None , errorMessage
# split path
parentDirPath , dirName = os . path . split ( relativePath )
# get real path
realPath = os . path . join ( self . __path , relativePath )
assert os . path . isdir ( realPath ) , "directory '%s' is not found in system" % realPath
# check directory in repository
assert dirName in dict . __getitem__ ( parentDirInfoDict , "directories" ) , "directory '%s' is not found in repository relative path '%s'" % ( dirName , parentDirPath )
# assert directory new name doesn ' t exist in repository
assert newName not in dict . __getitem__ ( parentDirInfoDict , "directories" ) , "directory '%s' already exists in repository, relative path '%s'" % ( newName , parentDirPath )
# check new directory in system
newRealPath = os . path . join ( self . __path , parentDirPath , newName )
if os . path . isdir ( newRealPath ) :
    if replace :
        shutil . rmtree ( newRealPath )
        if verbose :
            warnings . warn ( "directory '%s' already exists found in system, it is therefore deleted." % newRealPath )
    else :
        raise Exception ( "directory '%s' already exists in system" % newRealPath )
# rename directory
os . rename ( realPath , newRealPath )
dict . __setitem__ ( dict . __getitem__ ( parentDirInfoDict , "directories" ) , newName , dict . __getitem__ ( parentDirInfoDict , "directories" ) . pop ( dirName ) )
# save repository
self . save ( )


signature:
def rename_file ( self , relativePath , name , newName , replace = False , verbose = True ) :
    """Rename a directory in the repository . It insures renaming the file in the system .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory where the file is located .
 # . name ( string ) : The file name .
 # . newName ( string ) : The file new name .
 # . replace ( boolean ) : Whether to force renaming when new folder name exists in the system .
 It fails when new folder name is registered in repository .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
# normalize path
relativePath = os . path . normpath ( relativePath )
if relativePath == '.' :
    relativePath = ''
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
assert dirInfoDict is not None , errorMessage
# check directory in repository
assert name in dict . __getitem__ ( dirInfoDict , "files" ) , "file '%s' is not found in repository relative path '%s'" % ( name , relativePath )
# get real path
realPath = os . path . join ( self . __path , relativePath , name )
assert os . path . isfile ( realPath ) , "file '%s' is not found in system" % realPath
# assert directory new name doesn ' t exist in repository
assert newName not in dict . __getitem__ ( dirInfoDict , "files" ) , "file '%s' already exists in repository relative path '%s'" % ( newName , relativePath )
# check new directory in system
newRealPath = os . path . join ( self . __path , relativePath , newName )
if os . path . isfile ( newRealPath ) :
    if replace :
        os . remove ( newRealPath )
        if verbose :
            warnings . warn ( "file '%s' already exists found in system, it is now replaced by '%s' because 'replace' flag is True." % ( newRealPath , realPath ) )
    else :
        raise Exception ( "file '%s' already exists in system but not registered in repository." % newRealPath )
# rename file
os . rename ( realPath , newRealPath )
dict . __setitem__ ( dict . __getitem__ ( dirInfoDict , "files" ) , newName , dict . __getitem__ ( dirInfoDict , "files" ) . pop ( name ) )
# save repository
self . save ( )


signature:
def remove_file ( self , relativePath , name = None , removeFromSystem = False ) :
    """Remove file from repository .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory where the file should be dumped .
 If relativePath does not exist , it will be created automatically .
 # . name ( string ) : The file name .
 If None is given , name will be split from relativePath .
 # . removeFromSystem ( boolean ) : Whether to also remove directory and all files from the system . \n
 Only files saved in the repository will be removed and empty left directories ."""

implementation:
# get relative path normalized
relativePath = os . path . normpath ( relativePath )
if relativePath == '.' :
    relativePath = ''
    assert name != '.pyrepinfo' , "'.pyrepinfo' is not allowed as file name in main repository directory"
    assert name != '.pyrepstate' , "'.pyrepstate' is not allowed as file name in main repository directory"
    assert name != '.pyreplock' , "'.pyreplock' is not allowed as file name in main repository directory"
if name is None :
    assert len ( relativePath ) , "name must be given when relative path is given as empty string or as a simple dot '.'"
    relativePath , name = os . path . split ( relativePath )
# get file info dict
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
assert dirInfoDict is not None , errorMessage
# check directory in repository
assert name in dict . __getitem__ ( dirInfoDict , "files" ) , "file '%s' is not found in repository relative path '%s'" % ( name , relativePath )
# remove file from repo
dict . __getitem__ ( dirInfoDict , "files" ) . pop ( name )
# remove file from system
if removeFromSystem :
    ap = os . path . join ( self . __path , relativePath , name )
    if os . path . isfile ( ap ) :
        os . remove ( ap )
# save repository
self . save ( )


signature:
def dump_copy ( self , path , relativePath , name = None , description = None , replace = False , verbose = False ) :
    """Copy an exisitng system file to the repository .
 attribute in the Repository with utc timestamp .
 : Parameters :
 # . path ( str ) : The full path of the file to copy into the repository .
 # . relativePath ( str ) : The relative to the repository path of the directory where the file should be dumped .
 If relativePath does not exist , it will be created automatically .
 # . name ( string ) : The file name .
 If None is given , name will be split from path .
 # . description ( None , string , pickable object ) : Any random description about the file .
 # . replace ( boolean ) : Whether to replace any existing file with the same name if existing .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
relativePath = os . path . normpath ( relativePath )
if relativePath == '.' :
    relativePath = ''
if name is None :
    _ , name = os . path . split ( path )
# ensure directory added
self . add_directory ( relativePath )
# ger real path
realPath = os . path . join ( self . __path , relativePath )
# get directory info dict
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
assert dirInfoDict is not None , errorMessage
if name in dict . __getitem__ ( dirInfoDict , "files" ) :
    if not replace :
        if verbose :
            warnings . warn ( "a file with the name '%s' is already defined in repository dictionary info. Set replace flag to True if you want to replace the existing file" % ( name ) )
        return
# convert dump and pull methods to strings
dump = "raise Exception(\"dump is ambiguous for copied file '$FILE_PATH' \")"
pull = "raise Exception(\"pull is ambiguous for copied file '$FILE_PATH' \")"
# dump file
try :
    shutil . copyfile ( path , os . path . join ( realPath , name ) )
except Exception as e :
    if verbose :
        warnings . warn ( e )
    return
# set info
klass = None
# save the new file to the repository
dict . __getitem__ ( dirInfoDict , "files" ) [ name ] = { "dump" : dump , "pull" : pull , "timestamp" : datetime . utcnow ( ) , "id" : str ( uuid . uuid1 ( ) ) , "class" : klass , "description" : description }
# save repository
self . save ( )


signature:
def dump_file ( self , value , relativePath , name = None , description = None , klass = None , dump = None , pull = None , replace = False , ACID = None , verbose = False ) :
    """Dump a file using its value to the system and creates its
 attribute in the Repository with utc timestamp .
 : Parameters :
 # . value ( object ) : The value of a file to dump and add to the repository . It is any python object or file .
 # . relativePath ( str ) : The relative to the repository path of the directory where the file should be dumped .
 If relativePath does not exist , it will be created automatically .
 # . name ( string ) : The file name .
 If None is given , name will be split from relativePath .
 # . description ( None , string , pickable object ) : Any random description about the file .
 # . klass ( None , class ) : The dumped object class . If None is given
 klass will be automatically set to the following value . _ _ class _ _
 # . dump ( None , string ) : The dumping method .
 If None it will be set automatically to pickle and therefore the object must be pickleable .
 If a string is given , the string should include all the necessary imports
 and a ' $ FILE _ PATH ' that replaces the absolute file path when the dumping will be performed . \n
 e . g . " import numpy as np ; np . savetxt ( fname = ' $ FILE _ PATH ' , X = value , fmt = ' % . 6e ' ) "
 # . pull ( None , string ) : The pulling method .
 If None it will be set automatically to pickle and therefore the object must be pickleable .
 If a string is given , the string should include all the necessary imports ,
 a ' $ FILE _ PATH ' that replaces the absolute file path when the dumping will be performed
 and finally a PULLED _ DATA variable . \n
 e . g " import numpy as np ; PULLED _ DATA = np . loadtxt ( fname = ' $ FILE _ PATH ' ) "
 # . replace ( boolean ) : Whether to replace any existing file with the same name if existing .
 # . ACID ( None , boolean ) : Whether to ensure the ACID ( Atomicity , Consistency , Isolation , Durability )
 properties of the repository upon dumping a file . This is ensured by dumping the file in
 a temporary path first and then moving it to the desired path .
 If None is given , repository ACID property will be used .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
# check ACID
if ACID is None :
    ACID = self . __ACID
assert isinstance ( ACID , bool ) , "ACID must be boolean"
# check name and path
relativePath = os . path . normpath ( relativePath )
if relativePath == '.' :
    relativePath = ''
    assert name != '.pyrepinfo' , "'.pyrepinfo' is not allowed as file name in main repository directory"
    assert name != '.pyrepstate' , "'.pyrepstate' is not allowed as file name in main repository directory"
    assert name != '.pyreplock' , "'.pyreplock' is not allowed as file name in main repository directory"
if name is None :
    assert len ( relativePath ) , "name must be given when relative path is given as empty string or as a simple dot '.'"
    relativePath , name = os . path . split ( relativePath )
# ensure directory added
self . add_directory ( relativePath )
# get real path
realPath = os . path . join ( self . __path , relativePath )
# get directory info dict
dirInfoDict , errorMessage = self . get_directory_info ( relativePath )
assert dirInfoDict is not None , errorMessage
if name in dict . __getitem__ ( dirInfoDict , "files" ) :
    if not replace :
        if verbose :
            warnings . warn ( "a file with the name '%s' is already defined in repository dictionary info. Set replace flag to True if you want to replace the existing file" % ( name ) )
        return
# convert dump and pull methods to strings
if dump is None :
    dump = DEFAULT_DUMP
if pull is None :
    pull = DEFAULT_PULL
# get savePath
if ACID :
    savePath = os . path . join ( tempfile . gettempdir ( ) , str ( uuid . uuid1 ( ) ) )
else :
    savePath = os . path . join ( realPath , name )
# dump file
try :
    exec ( dump . replace ( "$FILE_PATH" , str ( savePath ) ) )
except Exception as e :
    message = "unable to dump the file (%s)" % e
    if 'pickle.dump(' in dump :
        message += '\nmore info: %s' % str ( get_pickling_errors ( value ) )
    raise Exception ( message )
# copy if ACID
if ACID :
    try :
        shutil . copyfile ( savePath , os . path . join ( realPath , name ) )
    except Exception as e :
        os . remove ( savePath )
        if verbose :
            warnings . warn ( e )
        return
    os . remove ( savePath )
# set info
if klass is None and value is not None :
    klass = value . __class__
if klass is not None :
    assert inspect . isclass ( klass ) , "klass must be a class definition"
# MUST TRY PICLKING KLASS TEMPORARILY FIRST
# save the new file to the repository
dict . __getitem__ ( dirInfoDict , "files" ) [ name ] = { "dump" : dump , "pull" : pull , "timestamp" : datetime . utcnow ( ) , "id" : str ( uuid . uuid1 ( ) ) , "class" : klass , "description" : description }
# save repository
self . save ( )


signature:
def update_file ( self , value , relativePath , name = None , description = False , klass = False , dump = False , pull = False , ACID = None , verbose = False ) :
    """Update the value and the utc timestamp of a file that is already in the Repository . \n
 If file is not registered in repository , and error will be thrown . \n
 If file is missing in the system , it will be regenerated as dump method is called .
 : Parameters :
 # . value ( object ) : The value of the file to update . It is any python object or a file .
 # . relativePath ( str ) : The relative to the repository path of the directory where the file should be dumped .
 # . name ( None , string ) : The file name .
 If None is given , name will be split from relativePath .
 # . description ( False , string , pickable object ) : Any random description about the file .
 If False is given , the description info won ' t be updated ,
 otherwise it will be update to what description argument value is .
 # . klass ( False , class ) : The dumped object class . If False is given ,
 the class info won ' t be updated , otherwise it will be update to what klass argument value is .
 # . dump ( False , string ) : The new dump method . If False is given , the old one will be used .
 # . pull ( False , string ) : The new pull method . If False is given , the old one will be used .
 # . ACID ( boolean ) : Whether to ensure the ACID ( Atomicity , Consistency , Isolation , Durability )
 properties of the repository upon dumping a file . This is ensured by dumping the file in
 a temporary path first and then moving it to the desired path .
 If None is given , repository ACID property will be used .
 # . verbose ( boolean ) : Whether to be warn and informed about any abnormalities ."""

implementation:
# check ACID
if ACID is None :
    ACID = self . __ACID
assert isinstance ( ACID , bool ) , "ACID must be boolean"
# get relative path normalized
relativePath = os . path . normpath ( relativePath )
if relativePath == '.' :
    relativePath = ''
    assert name != '.pyrepinfo' , "'.pyrepinfo' is not allowed as file name in main repository directory"
    assert name != '.pyrepstate' , "'.pyrepstate' is not allowed as file name in main repository directory"
    assert name != '.pyreplock' , "'.pyreplock' is not allowed as file name in main repository directory"
if name is None :
    assert len ( relativePath ) , "name must be given when relative path is given as empty string or as a simple dot '.'"
    relativePath , name = os . path . split ( relativePath )
# get file info dict
fileInfoDict , errorMessage = self . get_file_info ( relativePath , name )
assert fileInfoDict is not None , errorMessage
# get real path
realPath = os . path . join ( self . __path , relativePath )
# check if file exists
if verbose :
    if not os . path . isfile ( os . path . join ( realPath , name ) ) :
        warnings . warn ( "file '%s' is in repository but does not exist in the system. It is therefore being recreated." % os . path . join ( realPath , name ) )
# convert dump and pull methods to strings
if not dump :
    dump = fileInfoDict [ "dump" ]
if not pull :
    pull = fileInfoDict [ "pull" ]
# get savePath
if ACID :
    savePath = os . path . join ( tempfile . gettempdir ( ) , name )
else :
    savePath = os . path . join ( realPath , name )
# dump file
try :
    exec ( dump . replace ( "$FILE_PATH" , str ( savePath ) ) )
except Exception as e :
    message = "unable to dump the file (%s)" % e
    if 'pickle.dump(' in dump :
        message += '\nmore info: %s' % str ( get_pickling_errors ( value ) )
    raise Exception ( message )
# copy if ACID
if ACID :
    try :
        shutil . copyfile ( savePath , os . path . join ( realPath , name ) )
    except Exception as e :
        os . remove ( savePath )
        if verbose :
            warnings . warn ( e )
        return
    os . remove ( savePath )
# update timestamp
fileInfoDict [ "timestamp" ] = datetime . utcnow ( )
if description is not False :
    fileInfoDict [ "description" ] = description
if klass is not False :
    assert inspect . isclass ( klass ) , "klass must be a class definition"
    fileInfoDict [ "class" ] = klass
# save repository
self . save ( )


signature:
def pull_file ( self , relativePath , name = None , pull = None , update = True ) :
    """Pull a file ' s data from the Repository .
 : Parameters :
 # . relativePath ( string ) : The relative to the repository path of the directory where the file should be pulled .
 # . name ( string ) : The file name .
 If None is given , name will be split from relativePath .
 # . pull ( None , string ) : The pulling method .
 If None , the pull method saved in the file info will be used .
 If a string is given , the string should include all the necessary imports ,
 a ' $ FILE _ PATH ' that replaces the absolute file path when the dumping will be performed
 and finally a PULLED _ DATA variable .
 e . g " import numpy as np ; PULLED _ DATA = np . loadtxt ( fname = ' $ FILE _ PATH ' ) "
 # . update ( boolean ) : If pull is not None , Whether to update the pull method stored in the file info by the given pull method .
 : Returns :
 # . data ( object ) : The pulled data from the file ."""

implementation:
# get relative path normalized
relativePath = os . path . normpath ( relativePath )
if relativePath == '.' :
    relativePath = ''
    assert name != '.pyrepinfo' , "pulling '.pyrepinfo' from main repository directory is not allowed."
    assert name != '.pyrepstate' , "pulling '.pyrepstate' from main repository directory is not allowed."
    assert name != '.pyreplock' , "pulling '.pyreplock' from main repository directory is not allowed."
if name is None :
    assert len ( relativePath ) , "name must be given when relative path is given as empty string or as a simple dot '.'"
    relativePath , name = os . path . split ( relativePath )
# get file info
fileInfo , errorMessage = self . get_file_info ( relativePath , name )
assert fileInfo is not None , errorMessage
# get absolute path
realPath = os . path . join ( self . __path , relativePath )
assert os . path . exists ( realPath ) , "relative path '%s'within repository '%s' does not exist" % ( relativePath , self . __path )
# file path
fileAbsPath = os . path . join ( realPath , name )
assert os . path . isfile ( fileAbsPath ) , "file '%s' does not exist in absolute path '%s'" % ( name , realPath )
if pull is None :
    pull = fileInfo [ "pull" ]
# try to pull file
try :
    namespace = { }
    namespace . update ( globals ( ) )
    exec ( pull . replace ( "$FILE_PATH" , str ( os . path . join ( realPath , name ) ) ) , namespace )
except Exception as e :
    m = pull . replace ( "$FILE_PATH" , str ( os . path . join ( realPath , name ) ) )
    raise Exception ( "unable to pull data using '%s' from file (%s)" % ( m , e ) )
# update
if update :
    fileInfo [ "pull" ] = pull
# return data
return namespace [ 'PULLED_DATA' ]


signature:
def ensure_str ( value ) :
    """Ensure value is string ."""

implementation:
if isinstance ( value , six . string_types ) :
    return value
else :
    return six . text_type ( value )


signature:
def ensure_list ( path_or_path_list ) :
    """Pre - process input argument , whether if it is :
 1 . abspath
 2 . Path instance
 3 . string
 4 . list or set of any of them
 It returns list of path .
 : return path _ or _ path _ list : always return list of path in string
 * * 中文文档 * *
 预处理输入参数 。"""

implementation:
if isinstance ( path_or_path_list , ( tuple , list , set ) ) :
    return [ ensure_str ( path ) for path in path_or_path_list ]
else :
    return [ ensure_str ( path_or_path_list ) , ]


signature:
def stream ( self , report ) :
    """Stream reports to application logs"""

implementation:
with self . ClientSession ( ) as session :
    lines = [ ]
    for job in report [ 'traces' ] :
        key = '%s:%s' % ( self . name , job )
        for minute in report [ 'traces' ] [ job ] :
            for k , v in report [ 'traces' ] [ job ] [ minute ] . items ( ) :
                lines . append ( '# TYPE %s_%s gauge' % ( key , k ) )
                lines . append ( '%s_%s %0.2f' % ( key , k , v ) )
    # Empty is required at the end of the payload
    lines . append ( "" )
    data = "\n" . join ( lines )
    logger . info ( data )
    yield from session . post ( self . url , data = bytes ( data . encode ( 'utf-8' ) ) )


implementation:
payload = { "agent" : { "host" : report [ 'instance' ] [ 'hostname' ] , "version" : "1.0.0" } , "components" : [ { "name" : self . name , "guid" : "com.darwinmonroy.aiometrics" , "duration" : 60 , "metrics" : { 'Component/{}' . format ( key ) : { "total" : metric [ 'count' ] * metric [ 'avg' ] , "count" : metric [ 'count' ] , "min" : metric [ 'min' ] , "max" : metric [ 'max' ] , "sum_of_squares" : metric [ 'min' ] ** 2 + metric [ 'max' ] ** 2 , } for key , metric in report [ 'traces' ] . items ( ) } } ] }
with self . ClientSession ( ) as session :
    try :
        r = yield from session . post ( 'https://platform-api.newrelic.com/platform/v1/metrics' , data = json . dumps ( payload ) , headers = ( ( 'X-License-Key' , self . license_key ) , ( 'Content-Type' , 'application/json' ) , ( 'Accept' , 'application/json' ) , ) )
        r . close ( )
    except Exception as e : # Any exception should affect the execution of the main
    # program , so we must explicitly silence any error caused by
    # by the streaming of metrics
    # TODO : consider the implementation of a retry logic
        logger . exception ( e )


signature:
def stats ( cls , traces ) :
    """Build per minute stats for each key"""

implementation:
data = { }
stats = { }
# Group traces by key and minute
for trace in traces :
    key = trace [ 'key' ]
    if key not in data :
        data [ key ] = [ ]
        stats [ key ] = { }
    data [ key ] . append ( trace [ 'total_time' ] )
    cls . _traces . pop ( trace [ 'id' ] )
for key in data :
    times = data [ key ]
    stats [ key ] = dict ( count = len ( times ) , max = max ( times ) , min = min ( times ) , avg = sum ( times ) / len ( times ) )
return stats