signature:
def create_new_version ( self , name , subject , text = '' , template_id = None , html = None , locale = None , timeout = None ) :
    """API call to create a new version of a template"""

implementation:
if ( html ) :
    payload = { 'name' : name , 'subject' : subject , 'html' : html , 'text' : text }
else :
    payload = { 'name' : name , 'subject' : subject , 'text' : text }
if locale :
    url = self . TEMPLATES_SPECIFIC_LOCALE_VERSIONS_ENDPOINT % ( template_id , locale )
else :
    url = self . TEMPLATES_NEW_VERSION_ENDPOINT % template_id
return self . _api_request ( url , self . HTTP_POST , payload = payload , timeout = timeout )


signature:
def update_template_version ( self , name , subject , template_id , version_id , text = '' , html = None , timeout = None ) :
    """API call to update a template version"""

implementation:
if ( html ) :
    payload = { 'name' : name , 'subject' : subject , 'html' : html , 'text' : text }
else :
    payload = { 'name' : name , 'subject' : subject , 'text' : text }
return self . _api_request ( self . TEMPLATES_VERSION_ENDPOINT % ( template_id , version_id ) , self . HTTP_PUT , payload = payload , timeout = timeout )


signature:
def snippets ( self , timeout = None ) :
    """API call to get list of snippets"""

implementation: return self . _api_request ( self . SNIPPETS_ENDPOINT , self . HTTP_GET , timeout = timeout )


signature:
def get_snippet ( self , snippet_id , timeout = None ) :
    """API call to get a specific Snippet"""

implementation: return self . _api_request ( self . SNIPPET_ENDPOINT % ( snippet_id ) , self . HTTP_GET , timeout = timeout )


signature:
def create_snippet ( self , name , body , timeout = None ) :
    """API call to create a Snippet"""

implementation:
payload = { 'name' : name , 'body' : body }
return self . _api_request ( self . SNIPPETS_ENDPOINT , self . HTTP_POST , payload = payload , timeout = timeout )


signature:
def _make_file_dict ( self , f ) :
    """Make a dictionary with filename and base64 file data"""

implementation:
if isinstance ( f , dict ) :
    file_obj = f [ 'file' ]
    if 'filename' in f :
        file_name = f [ 'filename' ]
    else :
        file_name = file_obj . name
else :
    file_obj = f
    file_name = f . name
b64_data = base64 . b64encode ( file_obj . read ( ) )
return { 'id' : file_name , 'data' : b64_data . decode ( ) if six . PY3 else b64_data , }


signature:
def send ( self , email_id , recipient , email_data = None , sender = None , cc = None , bcc = None , tags = [ ] , headers = { } , esp_account = None , locale = None , email_version_name = None , inline = None , files = [ ] , timeout = None ) :
    """API call to send an email"""

implementation:
if not email_data :
    email_data = { }
# for backwards compatibility , will be removed
if isinstance ( recipient , string_types ) :
    warnings . warn ( "Passing email directly for recipient is deprecated" , DeprecationWarning )
    recipient = { 'address' : recipient }
payload = { 'email_id' : email_id , 'recipient' : recipient , 'email_data' : email_data }
if sender :
    payload [ 'sender' ] = sender
if cc :
    if not type ( cc ) == list :
        logger . error ( 'kwarg cc must be type(list), got %s' % type ( cc ) )
    payload [ 'cc' ] = cc
if bcc :
    if not type ( bcc ) == list :
        logger . error ( 'kwarg bcc must be type(list), got %s' % type ( bcc ) )
    payload [ 'bcc' ] = bcc
if tags :
    if not type ( tags ) == list :
        logger . error ( 'kwarg tags must be type(list), got %s' % ( type ( tags ) ) )
    payload [ 'tags' ] = tags
if headers :
    if not type ( headers ) == dict :
        logger . error ( 'kwarg headers must be type(dict), got %s' % ( type ( headers ) ) )
    payload [ 'headers' ] = headers
if esp_account :
    if not isinstance ( esp_account , string_types ) :
        logger . error ( 'kwarg esp_account must be a string, got %s' % ( type ( esp_account ) ) )
    payload [ 'esp_account' ] = esp_account
if locale :
    if not isinstance ( locale , string_types ) :
        logger . error ( 'kwarg locale must be a string, got %s' % ( type ( locale ) ) )
    payload [ 'locale' ] = locale
if email_version_name :
    if not isinstance ( email_version_name , string_types ) :
        logger . error ( 'kwarg email_version_name must be a string, got %s' % ( type ( email_version_name ) ) )
    payload [ 'version_name' ] = email_version_name
if inline :
    payload [ 'inline' ] = self . _make_file_dict ( inline )
if files :
    payload [ 'files' ] = [ self . _make_file_dict ( f ) for f in files ]
return self . _api_request ( self . SEND_ENDPOINT , self . HTTP_POST , payload = payload , timeout = timeout )


signature:
def _api_request ( self , endpoint , http_method , * args , ** kwargs ) :
    """Private method for api requests"""

implementation:
logger . debug ( ' > Queing batch api request for endpoint: %s' % endpoint )
path = self . _build_request_path ( endpoint , absolute = False )
logger . debug ( '\tpath: %s' % path )
data = None
if 'payload' in kwargs :
    data = kwargs [ 'payload' ]
logger . debug ( '\tdata: %s' % data )
command = { "path" : path , "method" : http_method }
if data :
    command [ 'body' ] = data
self . _commands . append ( command )


signature:
def execute ( self , timeout = None ) :
    """Execute all currently queued batch commands"""

implementation:
logger . debug ( ' > Batch API request (length %s)' % len ( self . _commands ) )
auth = self . _build_http_auth ( )
headers = self . _build_request_headers ( )
logger . debug ( '\tbatch headers: %s' % headers )
logger . debug ( '\tbatch command length: %s' % len ( self . _commands ) )
path = self . _build_request_path ( self . BATCH_ENDPOINT )
data = json . dumps ( self . _commands , cls = self . _json_encoder )
r = requests . post ( path , auth = auth , headers = headers , data = data , timeout = ( self . DEFAULT_TIMEOUT if timeout is None else timeout ) )
self . _commands = [ ]
logger . debug ( '\tresponse code:%s' % r . status_code )
try :
    logger . debug ( '\tresponse: %s' % r . json ( ) )
except :
    logger . debug ( '\tresponse: %s' % r . content )
return r


signature:
def get_group_tabs ( self ) :
    """Return instances of all other tabs that are members of the tab ' s
 tab group ."""

implementation:
if self . tab_group is None :
    raise ImproperlyConfigured ( "%s requires a definition of 'tab_group'" % self . __class__ . __name__ )
group_members = [ t for t in self . _registry if t . tab_group == self . tab_group ]
return [ t ( ) for t in group_members ]


signature:
def _process_tabs ( self , tabs , current_tab , group_current_tab ) :
    """Process and prepare tabs .
 This includes steps like updating references to the current tab ,
 filtering out hidden tabs , sorting tabs etc . . .
 Args :
 tabs :
 The list of tabs to process .
 current _ tab :
 The reference to the currently loaded tab .
 group _ current _ tab :
 The reference to the active tab in the current tab group . For
 parent tabs , this is different than for the current tab group .
 Returns :
 Processed list of tabs . Note that the method may have side effects ."""

implementation:
# Update references to the current tab
for t in tabs :
    t . current_tab = current_tab
    t . group_current_tab = group_current_tab
# Filter out hidden tabs
tabs = list ( filter ( lambda t : t . tab_visible , tabs ) )
# Sort remaining tabs in - place
tabs . sort ( key = lambda t : t . weight )
return tabs


signature:
def get_context_data ( self , ** kwargs ) :
    """Adds tab information to context .
 To retrieve a list of all group tab instances , use
 ` ` { { tabs } } ` ` in your template .
 The id of the current tab is added as ` ` current _ tab _ id ` ` to the
 template context .
 If the current tab has a parent tab the parent ' s id is added to
 the template context as ` ` parent _ tab _ id ` ` . Instances of all tabs
 of the parent level are added as ` ` parent _ tabs ` ` to the context .
 If the current tab has children they are added to the template
 context as ` ` child _ tabs ` ` ."""

implementation:
context = super ( TabView , self ) . get_context_data ( ** kwargs )
# Update the context with kwargs , TemplateView doesn ' t do this .
context . update ( kwargs )
# Add tabs and " current " references to context
process_tabs_kwargs = { 'tabs' : self . get_group_tabs ( ) , 'current_tab' : self , 'group_current_tab' : self , }
context [ 'tabs' ] = self . _process_tabs ( ** process_tabs_kwargs )
context [ 'current_tab_id' ] = self . tab_id
# Handle parent tabs
if self . tab_parent is not None : # Verify that tab parent is valid
    if self . tab_parent not in self . _registry :
        msg = '%s has no attribute _is_tab' % self . tab_parent . __class__ . __name__
        raise ImproperlyConfigured ( msg )
    # Get parent tab instance
    parent = self . tab_parent ( )
    # Add parent tabs to context
    process_parents_kwargs = { 'tabs' : parent . get_group_tabs ( ) , 'current_tab' : self , 'group_current_tab' : parent , }
    context [ 'parent_tabs' ] = self . _process_tabs ( ** process_parents_kwargs )
    context [ 'parent_tab_id' ] = parent . tab_id
# Handle child tabs
if self . tab_id in self . _children :
    process_children_kwargs = { 'tabs' : [ t ( ) for t in self . _children [ self . tab_id ] ] , 'current_tab' : self , 'group_current_tab' : None , }
    context [ 'child_tabs' ] = self . _process_tabs ( ** process_children_kwargs )
return context


signature:
def normalize_name ( s ) :
    """Convert a string into a valid python attribute name .
 This function is called to convert ASCII strings to something that can pass as
 python attribute name , to be used with namedtuples .
 > > > str ( normalize _ name ( ' class ' ) )
 ' class _ '
 > > > str ( normalize _ name ( ' a - name ' ) )
 ' a _ name '
 > > > str ( normalize _ name ( ' a n \u00e4 me ' ) )
 ' a _ name '
 > > > str ( normalize _ name ( ' Name ' ) )
 ' Name '
 > > > str ( normalize _ name ( ' ' ) )
 > > > str ( normalize _ name ( ' 1 ' ) )
 ' _ 1'"""

implementation:
s = s . replace ( '-' , '_' ) . replace ( '.' , '_' ) . replace ( ' ' , '_' )
if s in keyword . kwlist :
    return s + '_'
s = '_' . join ( slug ( ss , lowercase = False ) for ss in s . split ( '_' ) )
if not s :
    s = '_'
if s [ 0 ] not in string . ascii_letters + '_' :
    s = '_' + s
return s


signature:
def schema ( tg ) :
    """Convert the table and column descriptions of a ` TableGroup ` into specifications for the
 DB schema .
 : param ds :
 : return : A pair ( tables , reference _ tables ) ."""

implementation:
tables = { }
for tname , table in tg . tabledict . items ( ) :
    t = TableSpec . from_table_metadata ( table )
    tables [ t . name ] = t
    for at in t . many_to_many . values ( ) :
        tables [ at . name ] = at
# We must determine the order in which tables must be created !
ordered = OrderedDict ( )
i = 0
# We loop through the tables repeatedly , and whenever we find one , which has all
# referenced tables already in ordered , we move it from tables to ordered .
while tables and i < 100 :
    i += 1
    for table in list ( tables . keys ( ) ) :
        if all ( ( ref [ 1 ] in ordered ) or ref [ 1 ] == table for ref in tables [ table ] . foreign_keys ) : # All referenced tables are already created ( or self - referential ) .
            ordered [ table ] = tables . pop ( table )
            break
if tables : # pragma : no cover
    raise ValueError ( 'there seem to be cyclic dependencies between the tables' )
return list ( ordered . values ( ) )


signature:
def write ( self , _force = False , _exists_ok = False , ** items ) :
    """Creates a db file with the core schema .
 : param force : If ` True ` an existing db file will be overwritten ."""

implementation:
if self . fname and self . fname . exists ( ) :
    raise ValueError ( 'db file already exists, use force=True to overwrite' )
with self . connection ( ) as db :
    for table in self . tables :
        db . execute ( table . sql ( translate = self . translate ) )
    db . execute ( 'PRAGMA foreign_keys = ON;' )
    db . commit ( )
    refs = defaultdict ( list )
    # collects rows in association tables .
    for t in self . tables :
        if t . name not in items :
            continue
        rows , keys = [ ] , [ ]
        cols = { c . name : c for c in t . columns }
        for i , row in enumerate ( items [ t . name ] ) :
            pk = row [ t . primary_key [ 0 ] ] if t . primary_key and len ( t . primary_key ) == 1 else None
            values = [ ]
            for k , v in row . items ( ) :
                if k in t . many_to_many :
                    assert pk
                    at = t . many_to_many [ k ]
                    atkey = tuple ( [ at . name ] + [ c . name for c in at . columns ] )
                    for vv in v :
                        fkey , context = self . association_table_context ( t , k , vv )
                        refs [ atkey ] . append ( ( pk , fkey , context ) )
                else :
                    col = cols [ k ]
                    if isinstance ( v , list ) : # Note : This assumes list - valued columns are of datatype string !
                        v = ( col . separator or ';' ) . join ( col . convert ( vv ) for vv in v )
                    else :
                        v = col . convert ( v ) if v is not None else None
                    if i == 0 :
                        keys . append ( col . name )
                    values . append ( v )
            rows . append ( tuple ( values ) )
        insert ( db , self . translate , t . name , keys , * rows )
    for atkey , rows in refs . items ( ) :
        insert ( db , self . translate , atkey [ 0 ] , atkey [ 1 : ] , * rows )
    db . commit ( )


signature:
def iterrows ( lines_or_file , namedtuples = False , dicts = False , encoding = 'utf-8' , ** kw ) :
    """Convenience factory function for csv reader .
 : param lines _ or _ file : Content to be read . Either a file handle , a file path or a list of strings .
 : param namedtuples : Yield namedtuples .
 : param dicts : Yield dicts .
 : param encoding : Encoding of the content .
 : param kw : Keyword parameters are passed through to csv . reader .
 : return : A generator over the rows ."""

implementation:
if namedtuples and dicts :
    raise ValueError ( 'either namedtuples or dicts can be chosen as output format' )
elif namedtuples :
    _reader = NamedTupleReader
elif dicts :
    _reader = UnicodeDictReader
else :
    _reader = UnicodeReader
with _reader ( lines_or_file , encoding = encoding , ** fix_kw ( kw ) ) as r :
    for item in r :
        yield item


signature:
def rewrite ( fname , visitor , ** kw ) :
    """Utility function to rewrite rows in tsv files .
 : param fname : Path of the dsv file to operate on .
 : param visitor : A callable that takes a line - number and a row as input and returns a ( modified ) row or None to filter out the row .
 : param kw : Keyword parameters are passed through to csv . reader / csv . writer ."""

implementation:
if not isinstance ( fname , pathlib . Path ) :
    assert isinstance ( fname , string_types )
    fname = pathlib . Path ( fname )
assert fname . is_file ( )
with tempfile . NamedTemporaryFile ( delete = False ) as fp :
    tmp = pathlib . Path ( fp . name )
with UnicodeReader ( fname , ** kw ) as reader_ :
    with UnicodeWriter ( tmp , ** kw ) as writer :
        for i , row in enumerate ( reader_ ) :
            row = visitor ( i , row )
            if row is not None :
                writer . writerow ( row )
shutil . move ( str ( tmp ) , str ( fname ) )


signature:
def filter_rows_as_dict ( fname , filter_ , ** kw ) :
    """Rewrite a dsv file , filtering the rows .
 : param fname : Path to dsv file
 : param filter _ : callable which accepts a ` dict ` with a row ' s data as single argument returning a ` Boolean ` indicating whether to keep the row ( ` True ` ) or to discard it ` False ` .
 : param kw : Keyword arguments to be passed ` UnicodeReader ` and ` UnicodeWriter ` .
 : return : The number of rows that have been removed ."""

implementation:
filter_ = DictFilter ( filter_ )
rewrite ( fname , filter_ , ** kw )
return filter_ . removed


signature:
def dump_grid ( grid ) :
    """Dump a single grid to its ZINC representation ."""

implementation:
header = 'ver:%s' % dump_str ( str ( grid . _version ) , version = grid . _version )
if bool ( grid . metadata ) :
    header += ' ' + dump_meta ( grid . metadata , version = grid . _version )
columns = dump_columns ( grid . column , version = grid . _version )
rows = dump_rows ( grid )
return '\n' . join ( [ header , columns ] + rows + [ '' ] )


signature:
def parse ( grid_str , mode = MODE_ZINC , charset = 'utf-8' ) :
    '''Parse the given Zinc text and return the equivalent data .'''

implementation:
# Decode incoming text ( or python3 will whine ! )
if isinstance ( grid_str , six . binary_type ) :
    grid_str = grid_str . decode ( encoding = charset )
# Split the separate grids up , the grammar definition has trouble splitting
# them up normally . This will truncate the newline off the end of the last
# row .
_parse = functools . partial ( parse_grid , mode = mode , charset = charset )
if mode == MODE_JSON :
    if isinstance ( grid_str , six . string_types ) :
        grid_data = json . loads ( grid_str )
    else :
        grid_data = grid_str
    if isinstance ( grid_data , dict ) :
        return _parse ( grid_data )
    else :
        return list ( map ( _parse , grid_data ) )
else :
    return list ( map ( _parse , GRID_SEP . split ( grid_str . rstrip ( ) ) ) )


signature:
def append ( self , key , value = MARKER , replace = True ) :
    '''Append the item to the metadata .'''

implementation: return self . add_item ( key , value , replace = replace )


signature:
def extend ( self , items , replace = True ) :
    '''Append the items to the metadata .'''

implementation:
if isinstance ( items , dict ) or isinstance ( items , SortableDict ) :
    items = list ( items . items ( ) )
for ( key , value ) in items :
    self . append ( key , value , replace = replace )


signature:
def regular_polygon ( cls , center , radius , n_vertices , start_angle = 0 , ** kwargs ) :
    """Construct a regular polygon .
 Parameters
 center : array - like
 radius : float
 n _ vertices : int
 start _ angle : float , optional
 Where to put the first point , relative to ` center ` ,
 in radians counter - clockwise starting from the horizontal axis .
 kwargs
 Other keyword arguments are passed to the | Shape | constructor ."""

implementation:
angles = ( np . arange ( n_vertices ) * 2 * np . pi / n_vertices ) + start_angle
return cls ( center + radius * np . array ( [ np . cos ( angles ) , np . sin ( angles ) ] ) . T , ** kwargs )


signature:
def circle ( cls , center , radius , n_vertices = 50 , ** kwargs ) :
    """Construct a circle .
 Parameters
 center : array - like
 radius : float
 n _ vertices : int , optional
 Number of points to draw .
 Decrease for performance , increase for appearance .
 kwargs
 Other keyword arguments are passed to the | Shape | constructor ."""

implementation: return cls . regular_polygon ( center , radius , n_vertices , ** kwargs )


signature:
def rectangle ( cls , vertices , ** kwargs ) :
    """Shortcut for creating a rectangle aligned with the screen axes from only two corners .
 Parameters
 vertices : array - like
 An array containing the ` ` [ x , y ] ` ` positions of two corners .
 kwargs
 Other keyword arguments are passed to the | Shape | constructor ."""

implementation:
bottom_left , top_right = vertices
top_left = [ bottom_left [ 0 ] , top_right [ 1 ] ]
bottom_right = [ top_right [ 0 ] , bottom_left [ 1 ] ]
return cls ( [ bottom_left , bottom_right , top_right , top_left ] , ** kwargs )


signature:
def from_dict ( cls , spec ) :
    """Create a | Shape | from a dictionary specification .
 Parameters
 spec : dict
 A dictionary with either the fields ` ` ' center ' ` ` and ` ` ' radius ' ` ` ( for a circle ) ,
 ` ` ' center ' ` ` , ` ` ' radius ' ` ` , and ` ` ' n _ vertices ' ` ` ( for a regular polygon ) ,
 or ` ` ' vertices ' ` ` .
 If only two vertices are given , they are assumed to be lower left and top right corners of a rectangle .
 Other fields are interpreted as keyword arguments ."""

implementation:
spec = spec . copy ( )
center = spec . pop ( 'center' , None )
radius = spec . pop ( 'radius' , None )
if center and radius :
    return cls . circle ( center , radius , ** spec )
vertices = spec . pop ( 'vertices' )
if len ( vertices ) == 2 :
    return cls . rectangle ( vertices , ** spec )
return cls ( vertices , ** spec )


signature:
def _kwargs ( self ) :
    """Keyword arguments for recreating the Shape from the vertices ."""

implementation: return dict ( color = self . color , velocity = self . velocity , colors = self . colors )


signature:
def scale ( self , factor , center = None ) :
    """Resize the shape by a proportion ( e . g . , 1 is unchanged ) , in - place .
 Parameters
 factor : float or array - like
 If a scalar , the same factor will be applied in the x and y dimensions .
 center : array - like , optional
 Point around which to perform the scaling .
 If not passed , the center of the shape is used ."""

implementation:
factor = np . asarray ( factor )
if len ( factor . shape ) :
    args = list ( factor )
else :
    args = [ factor , factor ]
if center is not None :
    args . extend ( center )
self . poly . scale ( * args )
return self


signature:
def rotate ( self , angle , center = None ) :
    """Rotate the shape , in - place .
 Parameters
 angle : float
 Angle to rotate , in radians counter - clockwise .
 center : array - like , optional
 Point about which to rotate .
 If not passed , the center of the shape will be used ."""

implementation:
args = [ angle ]
if center is not None :
    args . extend ( center )
self . poly . rotate ( * args )
return self


signature:
def flip_x ( self , center = None ) :
    """Flip the shape in the x direction , in - place .
 Parameters
 center : array - like , optional
 Point about which to flip .
 If not passed , the center of the shape will be used ."""

implementation:
if center is None :
    self . poly . flip ( )
else :
    self . poly . flip ( center [ 0 ] )


signature:
def flip_y ( self , center = None ) :
    """Flip the shape in the y direction , in - place .
 Parameters
 center : array - like , optional
 Point about which to flip .
 If not passed , the center of the shape will be used ."""

implementation:
if center is None :
    self . poly . flop ( )
else :
    self . poly . flop ( center [ 1 ] )
return self


signature:
def flip ( self , angle , center = None ) :
    """Flip the shape in an arbitrary direction .
 Parameters
 angle : array - like
 The angle , in radians counter - clockwise from the horizontal axis ,
 defining the angle about which to flip the shape ( of a line through ` center ` ) .
 center : array - like , optional
 The point about which to flip .
 If not passed , the center of the shape will be used ."""

implementation: return self . rotate ( - angle , center = center ) . flip_y ( center = center ) . rotate ( angle , center = center )


signature:
def draw ( self ) :
    """Draw the shape in the current OpenGL context ."""

implementation:
if self . enabled :
    self . _vertex_list . colors = self . _gl_colors
    self . _vertex_list . vertices = self . _gl_vertices
    self . _vertex_list . draw ( pyglet . gl . GL_TRIANGLES )


signature:
def update ( self , dt ) :
    """Update the shape ' s position by moving it forward according to its velocity .
 Parameters
 dt : float"""

implementation:
self . translate ( dt * self . velocity )
self . rotate ( dt * self . angular_velocity )


signature:
def _map_timezones ( ) :
    """Map the official Haystack timezone list to those recognised by pytz ."""

implementation:
tz_map = { }
todo = HAYSTACK_TIMEZONES_SET . copy ( )
for full_tz in pytz . all_timezones : # Finished case :
    if not bool ( todo ) : # pragma : no cover
    # This is nearly impossible for us to cover , and an unlikely case .
        break
    # Case 1 : exact match
    if full_tz in todo :
        tz_map [ full_tz ] = full_tz
        # Exact match
        todo . discard ( full_tz )
        continue
    # Case 2 : suffix match after ' / '
    if '/' not in full_tz :
        continue
    ( prefix , suffix ) = full_tz . split ( '/' , 1 )
    # Case 2 exception : full timezone contains more than one ' / ' - > ignore
    if '/' in suffix :
        continue
    if suffix in todo :
        tz_map [ suffix ] = full_tz
        todo . discard ( suffix )
        continue
return tz_map


signature:
def timezone ( haystack_tz , version = LATEST_VER ) :
    """Retrieve the Haystack timezone"""

implementation:
tz_map = get_tz_map ( version = version )
try :
    tz_name = tz_map [ haystack_tz ]
except KeyError :
    raise ValueError ( '%s is not a recognised timezone on this host' % haystack_tz )
return pytz . timezone ( tz_name )


signature:
def timezone_name ( dt , version = LATEST_VER ) :
    """Determine an appropriate timezone for the given date / time object"""

implementation:
tz_rmap = get_tz_rmap ( version = version )
if dt . tzinfo is None :
    raise ValueError ( '%r has no timezone' % dt )
# Easy case : pytz timezone .
try :
    tz_name = dt . tzinfo . zone
    return tz_rmap [ tz_name ]
except KeyError : # Not in timezone map
    pass
except AttributeError : # Not a pytz - compatible tzinfo
    pass
# Hard case , try to find one that ' s equivalent . Hopefully we don ' t get
# many of these . Start by getting the current timezone offset , and a
# timezone - naïve copy of the timestamp .
offset = dt . utcoffset ( )
dt_notz = dt . replace ( tzinfo = None )
if offset == datetime . timedelta ( 0 ) : # UTC ?
    return 'UTC'
for olson_name , haystack_name in list ( tz_rmap . items ( ) ) :
    if pytz . timezone ( olson_name ) . utcoffset ( dt_notz ) == offset :
        return haystack_name
raise ValueError ( 'Unable to get timezone of %r' % dt )


signature:
def _unescape ( s , uri = False ) :
    """Iterative parser for string escapes ."""

implementation:
out = ''
while len ( s ) > 0 :
    c = s [ 0 ]
    if c == '\\' : # Backslash escape
        esc_c = s [ 1 ]
        if esc_c in ( 'u' , 'U' ) : # Unicode escape
            out += six . unichr ( int ( s [ 2 : 6 ] , base = 16 ) )
            s = s [ 6 : ]
            continue
        else :
            if esc_c == 'b' :
                out += '\b'
            elif esc_c == 'f' :
                out += '\f'
            elif esc_c == 'n' :
                out += '\n'
            elif esc_c == 'r' :
                out += '\r'
            elif esc_c == 't' :
                out += '\t'
            else :
                if uri and ( esc_c == '#' ) : # \ # is passed through with backslash .
                    out += '\\'
                # Pass through
                out += esc_c
            s = s [ 2 : ]
            continue
    else :
        out += c
        s = s [ 1 : ]
return out


signature:
def parse_grid ( grid_data ) :
    """Parse the incoming grid ."""

implementation:
try : # Split the grid up .
    grid_parts = NEWLINE_RE . split ( grid_data )
    if len ( grid_parts ) < 2 :
        raise ZincParseException ( 'Malformed grid received' , grid_data , 1 , 1 )
    # Grid and column metadata are the first two lines .
    grid_meta_str = grid_parts . pop ( 0 )
    col_meta_str = grid_parts . pop ( 0 )
    # First element is the grid metadata
    ver_match = VERSION_RE . match ( grid_meta_str )
    if ver_match is None :
        raise ZincParseException ( 'Could not determine version from %r' % grid_meta_str , grid_data , 1 , 1 )
    version = Version ( ver_match . group ( 1 ) )
    # Now parse the rest of the grid accordingly
    try :
        grid_meta = hs_gridMeta [ version ] . parseString ( grid_meta_str , parseAll = True ) [ 0 ]
    except pp . ParseException as pe : # Raise a new exception with the appropriate line number .
        raise ZincParseException ( 'Failed to parse grid metadata: %s' % pe , grid_data , 1 , pe . col )
    except : # pragma : no cover
    # Report an error to the log if we fail to parse something .
        LOG . debug ( 'Failed to parse grid meta: %r' , grid_meta_str )
        raise
    try :
        col_meta = hs_cols [ version ] . parseString ( col_meta_str , parseAll = True ) [ 0 ]
    except pp . ParseException as pe : # Raise a new exception with the appropriate line number .
        raise ZincParseException ( 'Failed to parse column metadata: %s' % reformat_exception ( pe , 2 ) , grid_data , 2 , pe . col )
    except : # pragma : no cover
    # Report an error to the log if we fail to parse something .
        LOG . debug ( 'Failed to parse column meta: %r' , col_meta_str )
        raise
    row_grammar = hs_row [ version ]
    def _parse_row ( row_num_and_data ) :
        ( row_num , row ) = row_num_and_data
        line_num = row_num + 3
        try :
            return dict ( zip ( col_meta . keys ( ) , row_grammar . parseString ( row , parseAll = True ) [ 0 ] . asList ( ) ) )
        except pp . ParseException as pe : # Raise a new exception with the appropriate line number .
            raise ZincParseException ( 'Failed to parse row: %s' % reformat_exception ( pe , line_num ) , grid_data , line_num , pe . col )
        except : # pragma : no cover
        # Report an error to the log if we fail to parse something .
            LOG . debug ( 'Failed to parse row: %r' , row )
            raise
    g = Grid ( version = grid_meta . pop ( 'ver' ) , metadata = grid_meta , columns = list ( col_meta . items ( ) ) )
    g . extend ( map ( _parse_row , filter ( lambda gp : bool ( gp [ 1 ] ) , enumerate ( grid_parts ) ) ) )
    return g
except :
    LOG . debug ( 'Failing grid: %r' , grid_data )
    raise


signature:
def parse_scalar ( scalar_data , version ) :
    """Parse a Project Haystack scalar in ZINC format ."""

implementation:
try :
    return hs_scalar [ version ] . parseString ( scalar_data , parseAll = True ) [ 0 ]
except pp . ParseException as pe : # Raise a new exception with the appropriate line number .
    raise ZincParseException ( 'Failed to parse scalar: %s' % reformat_exception ( pe ) , scalar_data , 1 , pe . col )
except :
    LOG . debug ( 'Failing scalar data: %r (version %r)' , scalar_data , version )


signature:
def add_item ( self , key , value , after = False , index = None , pos_key = None , replace = True ) :
    """Add an item at a specific location , possibly replacing the
 existing item .
 If after is True , we insert * after * the given index , otherwise we
 insert before .
 The position is specified using either index or pos _ key , the former
 specifies the position from the start of the array ( base 0 ) . pos _ key
 specifies the name of another key , and positions the new key relative
 to that key .
 When replacing , the position will be left un - changed unless a location
 is specified explicitly ."""

implementation:
if self . _validate_fn :
    self . _validate_fn ( value )
if ( index is not None ) and ( pos_key is not None ) :
    raise ValueError ( 'Either specify index or pos_key, not both.' )
elif pos_key is not None :
    try :
        index = self . index ( pos_key )
    except ValueError :
        raise KeyError ( '%r not found' % pos_key )
if after and ( index is not None ) : # insert inserts * before * index , so increment by one .
    index += 1
if key in self . _values :
    if not replace :
        raise KeyError ( '%r is duplicate' % key )
    if index is not None : # We are re - locating .
        del self [ key ]
    else : # We are updating
        self . _values [ key ] = value
        return
if index is not None : # Place at given position
    self . _order . insert ( index , key )
else : # Place at end
    self . _order . append ( key )
self . _values [ key ] = value


signature:
def dump ( grids , mode = MODE_ZINC ) :
    """Dump the given grids in the specified over - the - wire format ."""

implementation:
if isinstance ( grids , Grid ) :
    return dump_grid ( grids , mode = mode )
_dump = functools . partial ( dump_grid , mode = mode )
if mode == MODE_ZINC :
    return '\n' . join ( map ( _dump , grids ) )
elif mode == MODE_JSON :
    return '[%s]' % ',' . join ( map ( _dump , grids ) )
else : # pragma : no cover
    raise NotImplementedError ( 'Format not implemented: %s' % mode )


signature:
def to_haystack ( unit ) :
    """Some parsing tweaks to fit pint units / handling of edge cases ."""

implementation:
unit = str ( unit )
global HAYSTACK_CONVERSION
global PINT_CONVERSION
if unit == 'per_minute' or unit == '/min' or unit == 'per_second' or unit == '/s' or unit == 'per_hour' or unit == '/h' or unit == None :
    return ''
    # Those units are not units . . . they are impossible to fit anywhere in Pint
for pint_value , haystack_value in PINT_CONVERSION :
    unit = unit . replace ( pint_value , haystack_value )
for haystack_value , pint_value in HAYSTACK_CONVERSION :
    if pint_value == '' :
        continue
    unit = unit . replace ( pint_value , haystack_value )
return unit


signature:
def to_pint ( unit ) :
    """Some parsing tweaks to fit pint units / handling of edge cases ."""

implementation:
global HAYSTACK_CONVERSION
if unit == 'per_minute' or unit == '/min' or unit == 'per_second' or unit == '/s' or unit == 'per_hour' or unit == '/h' or unit == None :
    return ''
    # Those units are not units . . . they are impossible to fit anywhere in Pint
for haystack_value , pint_value in HAYSTACK_CONVERSION :
    unit = unit . replace ( haystack_value , pint_value )
return unit


signature:
def define_haystack_units ( ) :
    """Missing units found in project - haystack
 Added to the registry"""

implementation:
ureg = UnitRegistry ( )
ureg . define ( '% = [] = percent' )
ureg . define ( 'pixel = [] = px = dot = picture_element = pel' )
ureg . define ( 'decibel = [] = dB' )
ureg . define ( 'ppu = [] = parts_per_unit' )
ureg . define ( 'ppm = [] = parts_per_million' )
ureg . define ( 'ppb = [] = parts_per_billion' )
ureg . define ( '%RH = [] = percent_relative_humidity = percentRH' )
ureg . define ( 'cubic_feet = ft ** 3 = cu_ft' )
ureg . define ( 'cfm = cu_ft * minute = liter_per_second / 0.4719475' )
ureg . define ( 'cfh = cu_ft * hour' )
ureg . define ( 'cfs = cu_ft * second' )
ureg . define ( 'VAR = volt * ampere' )
ureg . define ( 'kVAR = 1000 * volt * ampere' )
ureg . define ( 'MVAR = 1000000 * volt * ampere' )
ureg . define ( 'inH2O = in_H2O' )
ureg . define ( 'dry_air = []' )
ureg . define ( 'gas = []' )
ureg . define ( 'energy_efficiency_ratio = [] = EER' )
ureg . define ( 'coefficient_of_performance = [] = COP' )
ureg . define ( 'data_center_infrastructure_efficiency = [] = DCIE' )
ureg . define ( 'power_usage_effectiveness = [] = PUE' )
ureg . define ( 'formazin_nephelometric_unit = [] = fnu' )
ureg . define ( 'nephelometric_turbidity_units = [] = ntu' )
ureg . define ( 'power_factor = [] = PF' )
ureg . define ( 'degree_day_celsius = [] = degdaysC' )
ureg . define ( 'degree_day_farenheit = degree_day_celsius * 9 / 5 = degdaysF' )
ureg . define ( 'footcandle = lumen / sq_ft = ftcd' )
ureg . define ( 'Nm = newton * meter' )
ureg . define ( '%obsc = [] = percent_obscuration = percentobsc' )
ureg . define ( 'cycle = []' )
ureg . define ( 'cph = cycle / hour' )
ureg . define ( 'cpm = cycle / minute' )
ureg . define ( 'cps = cycle / second' )
ureg . define ( 'hecto_cubic_foot = 100 * cubic_foot' )
ureg . define ( 'tenths_second = second / 10' )
ureg . define ( 'hundredths_second = second / 100' )
# ureg . define ( ' irradiance = W / sq _ meter = irr ' )
# In the definition of project haystack , there ' s a redundancy as irr = W / m ^ 2
# no need to use : watts _ per _ square _ meter _ irradiance
# CURRENCY
# I know . . . we won ' T be able to convert right now !
ureg . define ( 'australian_dollar = [] = AUD' )
ureg . define ( 'british_pound = [] = GBP = £' )
ureg . define ( 'canadian_dollar = [] = CAD' )
ureg . define ( 'chinese_yuan = [] = CNY = 元' )
ureg . define ( 'emerati_dirham = [] = AED' )
ureg . define ( 'euro = [] = EUR = €' )
ureg . define ( 'indian_rupee = [] = INR = ₹' )
ureg . define ( 'japanese_yen = [] = JPY = ¥' )
ureg . define ( 'russian_ruble = [] = RUB = руб' )
ureg . define ( 'south_korean_won = [] = KRW = ₩' )
ureg . define ( 'swedish_krona = [] = SEK = kr' )
ureg . define ( 'swiss_franc = [] = CHF = Fr' )
ureg . define ( 'taiwan_dollar = [] = TWD' )
ureg . define ( 'us_dollar = [] = USD = $' )
ureg . define ( 'new_israeli_shekel = [] = NIS' )
return ureg


signature:
def _detect_or_validate ( self , val ) :
    '''Detect the version used from the row content , or validate against
 the version if given .'''

implementation:
if isinstance ( val , list ) or isinstance ( val , dict ) or isinstance ( val , SortableDict ) or isinstance ( val , Grid ) : # Project Haystack 3.0 type .
    self . _assert_version ( VER_3_0 )


signature:
def _assert_version ( self , version ) :
    '''Assert that the grid version is equal to or above the given value .
 If no version is set , set the version .'''

implementation:
if self . nearest_version < version :
    if self . _version_given :
        raise ValueError ( 'Data type requires version %s' % version )
    else :
        self . _version = version


signature:
def _cmp ( self , other ) :
    """Compare two Project Haystack version strings , then return
 -1 if self < other ,
 0 if self = = other
 or 1 if self > other ."""

implementation:
if not isinstance ( other , Version ) :
    other = Version ( other )
num1 = self . version_nums
num2 = other . version_nums
# Pad both to be the same length
ver_len = max ( len ( num1 ) , len ( num2 ) )
num1 += tuple ( [ 0 for n in range ( len ( num1 ) , ver_len ) ] )
num2 += tuple ( [ 0 for n in range ( len ( num2 ) , ver_len ) ] )
# Compare the versions
for ( p1 , p2 ) in zip ( num1 , num2 ) :
    if p1 < p2 :
        return - 1
    elif p1 > p2 :
        return 1
# All the same , compare the extra strings .
# If a version misses the extra part ; we consider that as coming * before * .
if self . version_extra is None :
    if other . version_extra is None :
        return 0
    else :
        return - 1
elif other . version_extra is None :
    return 1
elif self . version_extra == other . version_extra :
    return 0
elif self . version_extra < other . version_extra :
    return - 1
else :
    return 1


signature:
def nearest ( self , ver ) :
    """Retrieve the official version nearest the one given ."""

implementation:
if not isinstance ( ver , Version ) :
    ver = Version ( ver )
if ver in OFFICIAL_VERSIONS :
    return ver
# We might not have an exact match for that .
# See if we have one that ' s newer than the grid we ' re looking at .
versions = list ( OFFICIAL_VERSIONS )
versions . sort ( reverse = True )
best = None
for candidate in versions : # Due to ambiguities , we might have an exact match and not know it .
# '2.0 ' will not hash to the same value as ' 2.0.0 ' , but both are
# equivalent .
    if candidate == ver : # We can ' t beat this , make a note of the match for later
        return candidate
    # If we have not seen a better candidate , and this is older
    # then we may have to settle for that .
    if ( best is None ) and ( candidate < ver ) :
        warnings . warn ( 'This version of hszinc does not yet ' 'support version %s, please seek a newer version ' 'or file a bug.  Closest (older) version supported is %s.' % ( ver , candidate ) )
        return candidate
    # Probably the best so far , but see if we can go closer
    if candidate > ver :
        best = candidate
# Unhappy path , no best option ? This should not happen .
assert best is not None
warnings . warn ( 'This version of hszinc does not yet ' 'support version %s, please seek a newer version ' 'or file a bug.  Closest (newer) version supported is %s.' % ( ver , best ) )
return best


signature:
def encrypt_files ( selected_host , only_link , file_name ) :
    """Encrypts file with gpg and random generated password"""

implementation:
if ENCRYPTION_DISABLED :
    print ( 'For encryption please install gpg' )
    exit ( )
passphrase = '%030x' % random . randrange ( 16 ** 30 )
source_filename = file_name
cmd = 'gpg --batch --symmetric --cipher-algo AES256 --passphrase-fd 0 ' '--output - {}' . format ( source_filename )
encrypted_output = Popen ( shlex . split ( cmd ) , stdout = PIPE , stdin = PIPE , stderr = PIPE )
encrypted_data = encrypted_output . communicate ( passphrase . encode ( ) ) [ 0 ]
return upload_files ( encrypted_data , selected_host , only_link , file_name ) + '#' + passphrase


signature:
def check_max_filesize ( chosen_file , max_size ) :
    """Checks file sizes for host"""

implementation:
if os . path . getsize ( chosen_file ) > max_size :
    return False
else :
    return True


signature:
def parse_arguments ( args , clone_list ) :
    """Makes parsing arguments a function ."""

implementation:
returned_string = ""
host_number = args . host
if args . show_list :
    print ( generate_host_string ( clone_list , "Available hosts: " ) )
    exit ( )
if args . decrypt :
    for i in args . files :
        print ( decrypt_files ( i ) )
        exit ( )
if args . files :
    for i in args . files :
        if args . limit_size :
            if args . host == host_number and host_number is not None :
                if not check_max_filesize ( i , clone_list [ host_number ] [ 3 ] ) :
                    host_number = None
            for n , host in enumerate ( clone_list ) :
                if not check_max_filesize ( i , host [ 3 ] ) :
                    clone_list [ n ] = None
            if not clone_list :
                print ( 'None of the clones is able to support so big file.' )
        if args . no_cloudflare :
            if args . host == host_number and host_number is not None and not clone_list [ host_number ] [ 4 ] :
                print ( "This host uses Cloudflare, please choose different host." )
                exit ( 1 )
            else :
                for n , host in enumerate ( clone_list ) :
                    if not host [ 4 ] :
                        clone_list [ n ] = None
        clone_list = list ( filter ( None , clone_list ) )
        if host_number is None or args . host != host_number :
            host_number = random . randrange ( 0 , len ( clone_list ) )
        while True :
            try :
                if args . encrypt :
                    returned_string = encrypt_files ( clone_list [ host_number ] , args . only_link , i )
                else :
                    returned_string = upload_files ( open ( i , 'rb' ) , clone_list [ host_number ] , args . only_link , i )
                if args . only_link :
                    print ( returned_string [ 0 ] )
                else :
                    print ( returned_string )
            except IndexError : # print ( ' Selected server ( ' + clone _ list [ host _ number ] [ 0 ] + ' ) is offline . ' )
            # print ( ' Trying other host . ' )
                host_number = random . randrange ( 0 , len ( clone_list ) )
                continue
            except IsADirectoryError :
                print ( 'limf does not support directory upload, if you want to upload ' 'every file in directory use limf {}/*.' . format ( i . replace ( '/' , '' ) ) )
            if args . log :
                with open ( os . path . expanduser ( args . logfile ) , "a+" ) as logfile :
                    if args . only_link :
                        logfile . write ( returned_string [ 1 ] )
                    else :
                        logfile . write ( returned_string )
                    logfile . write ( "\n" )
            break
else :
    print ( "limf: try 'limf -h' for more information" )


signature:
def upload_files ( selected_file , selected_host , only_link , file_name ) :
    """Uploads selected file to the host , thanks to the fact that
 every pomf . se based site has pretty much the same architecture ."""

implementation:
try :
    answer = requests . post ( url = selected_host [ 0 ] + "upload.php" , files = { 'files[]' : selected_file } )
    file_name_1 = re . findall ( r'"url": *"((h.+\/){0,1}(.+?))"[,\}]' , answer . text . replace ( "\\" , "" ) ) [ 0 ] [ 2 ]
    if only_link :
        return [ selected_host [ 1 ] + file_name_1 , "{}: {}{}" . format ( file_name , selected_host [ 1 ] , file_name_1 ) ]
    else :
        return "{}: {}{}" . format ( file_name , selected_host [ 1 ] , file_name_1 )
except requests . exceptions . ConnectionError :
    print ( file_name + ' couldn\'t be uploaded to ' + selected_host [ 0 ] )


signature:
def swagger_ui_template_view ( request ) :
    """Serves Swagger UI page , default Swagger UI config is used but you can
 override the callable that generates the ` < script > ` tag by setting
 ` cornice _ swagger . swagger _ ui _ script _ generator ` in pyramid config , it defaults
 to ' cornice _ swagger . views : swagger _ ui _ script _ template '
 : param request :
 : return :"""

implementation:
script_generator = request . registry . settings . get ( 'cornice_swagger.swagger_ui_script_generator' , 'cornice_swagger.views:swagger_ui_script_template' )
package , callable = script_generator . split ( ':' )
imported_package = importlib . import_module ( package )
script_callable = getattr ( imported_package , callable )
template = pkg_resources . resource_string ( 'cornice_swagger' , 'templates/index.html' ) . decode ( 'utf8' )
html = Template ( template ) . safe_substitute ( ui_css_url = ui_css_url , ui_js_bundle_url = ui_js_bundle_url , ui_js_standalone_url = ui_js_standalone_url , swagger_ui_script = script_callable ( request ) , )
return Response ( html )


signature:
def open_api_json_view ( request ) :
    """: param request :
 : return :
 Generates JSON representation of Swagger spec"""

implementation:
doc = cornice_swagger . CorniceSwagger ( cornice . service . get_services ( ) , pyramid_registry = request . registry )
kwargs = request . registry . settings [ 'cornice_swagger.spec_kwargs' ]
my_spec = doc . generate ( ** kwargs )
return my_spec


signature:
def swagger_ui_script_template ( request , ** kwargs ) :
    """: param request :
 : return :
 Generates the < script > code that bootstraps Swagger UI , it will be injected
 into index template"""

implementation:
swagger_spec_url = request . route_url ( 'cornice_swagger.open_api_path' )
template = pkg_resources . resource_string ( 'cornice_swagger' , 'templates/index_script_template.html' ) . decode ( 'utf8' )
return Template ( template ) . safe_substitute ( swagger_spec_url = swagger_spec_url , )


signature:
def decrypt_files ( file_link ) :
    """Decrypts file from entered links"""

implementation:
if ENCRYPTION_DISABLED :
    print ( 'For decryption please install gpg' )
    exit ( )
try :
    parsed_link = re . findall ( r'(.*/(.*))#(.{30})' , file_link ) [ 0 ]
    req = urllib . request . Request ( parsed_link [ 0 ] , data = None , headers = { 'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) ' ' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36' } )
    # downloads the file using fake useragent
    file_response = urllib . request . urlopen ( req )
    file_to_decrypt = file_response . read ( )
    # decrypts the data using piping to ggp
    decrypt_r , decrypt_w = os . pipe ( )
    cmd = 'gpg --batch --decrypt --passphrase-fd {}' . format ( decrypt_r )
    decrypt_output = Popen ( shlex . split ( cmd ) , stdout = PIPE , stdin = PIPE , stderr = PIPE , pass_fds = ( decrypt_r , ) )
    os . close ( decrypt_r )
    open ( decrypt_w , 'w' ) . write ( parsed_link [ 2 ] )
    decrypted_data , stderr = decrypt_output . communicate ( file_to_decrypt )
    with open ( parsed_link [ 1 ] , 'wb' ) as decrypted_file :
        decrypted_file . write ( decrypted_data )
    return parsed_link [ 1 ] + ' is decrypted and saved.'
except IndexError :
    return 'Please enter valid link.'


signature:
def set_value ( request ) :
    """Set the value and returns * True * or * False * ."""

implementation:
key = request . matchdict [ 'key' ]
_VALUES [ key ] = request . json_body
return _VALUES . get ( key )


signature:
def from_schema ( self , schema_node , base_name = None ) :
    """Creates a Swagger definition from a colander schema .
 : param schema _ node :
 Colander schema to be transformed into a Swagger definition .
 : param base _ name :
 Schema alternative title .
 : rtype : dict
 : returns : Swagger schema ."""

implementation: return self . _ref_recursive ( self . type_converter ( schema_node ) , self . ref , base_name )


signature:
def _ref_recursive ( self , schema , depth , base_name = None ) :
    """Dismantle nested swagger schemas into several definitions using JSON pointers .
 Note : This can be dangerous since definition titles must be unique .
 : param schema :
 Base swagger schema .
 : param depth :
 How many levels of the swagger object schemas should be split into
 swaggger definitions with JSON pointers . Default ( 0 ) is no split .
 You may use negative values to split everything .
 : param base _ name :
 If schema doesn ' t have a name , the caller may provide it to be
 used as reference .
 : rtype : dict
 : returns :
 JSON pointer to the root definition schema ,
 or the original definition if depth is zero ."""

implementation:
if depth == 0 :
    return schema
if schema [ 'type' ] != 'object' :
    return schema
name = base_name or schema [ 'title' ]
pointer = self . json_pointer + name
for child_name , child in schema . get ( 'properties' , { } ) . items ( ) :
    schema [ 'properties' ] [ child_name ] = self . _ref_recursive ( child , depth - 1 )
self . definition_registry [ name ] = schema
return { '$ref' : pointer }


signature:
def from_schema ( self , schema_node ) :
    """Creates a list of Swagger params from a colander request schema .
 : param schema _ node :
 Request schema to be transformed into Swagger .
 : param validators :
 Validators used in colander with the schema .
 : rtype : list
 : returns : List of Swagger parameters ."""

implementation:
params = [ ]
for param_schema in schema_node . children :
    location = param_schema . name
    if location is 'body' :
        name = param_schema . __class__ . __name__
        if name == 'body' :
            name = schema_node . __class__ . __name__ + 'Body'
        param = self . parameter_converter ( location , param_schema )
        param [ 'name' ] = name
        if self . ref :
            param = self . _ref ( param )
        params . append ( param )
    elif location in ( ( 'path' , 'header' , 'headers' , 'querystring' , 'GET' ) ) :
        for node_schema in param_schema . children :
            param = self . parameter_converter ( location , node_schema )
            if self . ref :
                param = self . _ref ( param )
            params . append ( param )
return params


signature:
def from_path ( self , path ) :
    """Create a list of Swagger path params from a cornice service path .
 : type path : string
 : rtype : list"""

implementation:
path_components = path . split ( '/' )
param_names = [ comp [ 1 : - 1 ] for comp in path_components if comp . startswith ( '{' ) and comp . endswith ( '}' ) ]
params = [ ]
for name in param_names :
    param_schema = colander . SchemaNode ( colander . String ( ) , name = name )
    param = self . parameter_converter ( 'path' , param_schema )
    if self . ref :
        param = self . _ref ( param )
    params . append ( param )
return params


signature:
def _ref ( self , param , base_name = None ) :
    """Store a parameter schema and return a reference to it .
 : param schema :
 Swagger parameter definition .
 : param base _ name :
 Name that should be used for the reference .
 : rtype : dict
 : returns : JSON pointer to the original parameter definition ."""

implementation:
name = base_name or param . get ( 'title' , '' ) or param . get ( 'name' , '' )
pointer = self . json_pointer + name
self . parameter_registry [ name ] = param
return { '$ref' : pointer }


signature:
def from_schema_mapping ( self , schema_mapping ) :
    """Creates a Swagger response object from a dict of response schemas .
 : param schema _ mapping :
 Dict with entries matching ` ` { status _ code : response _ schema } ` ` .
 : rtype : dict
 : returns : Response schema ."""

implementation:
responses = { }
for status , response_schema in schema_mapping . items ( ) :
    response = { }
    if response_schema . description :
        response [ 'description' ] = response_schema . description
    else :
        raise CorniceSwaggerException ( 'Responses must have a description.' )
    for field_schema in response_schema . children :
        location = field_schema . name
        if location == 'body' :
            title = field_schema . __class__ . __name__
            if title == 'body' :
                title = response_schema . __class__ . __name__ + 'Body'
            field_schema . title = title
            response [ 'schema' ] = self . definitions . from_schema ( field_schema )
        elif location in ( 'header' , 'headers' ) :
            header_schema = self . type_converter ( field_schema )
            headers = header_schema . get ( 'properties' )
            if headers : # Response headers doesn ' t accept titles
                for header in headers . values ( ) :
                    header . pop ( 'title' )
                response [ 'headers' ] = headers
    pointer = response_schema . __class__ . __name__
    if self . ref :
        response = self . _ref ( response , pointer )
    responses [ status ] = response
return responses


signature:
def _ref ( self , resp , base_name = None ) :
    """Store a response schema and return a reference to it .
 : param schema :
 Swagger response definition .
 : param base _ name :
 Name that should be used for the reference .
 : rtype : dict
 : returns : JSON pointer to the original response definition ."""

implementation:
name = base_name or resp . get ( 'title' , '' ) or resp . get ( 'name' , '' )
pointer = self . json_pointer + name
self . response_registry [ name ] = resp
return { '$ref' : pointer }


signature:
def generate ( self , title = None , version = None , base_path = None , info = None , swagger = None , ** kwargs ) :
    """Generate a Swagger 2.0 documentation . Keyword arguments may be used
 to provide additional information to build methods as such ignores .
 : param title :
 The name presented on the swagger document .
 : param version :
 The version of the API presented on the swagger document .
 : param base _ path :
 The path that all requests to the API must refer to .
 : param info :
 Swagger info field .
 : param swagger :
 Extra fields that should be provided on the swagger documentation .
 : rtype : dict
 : returns : Full OpenAPI / Swagger compliant specification for the application ."""

implementation:
title = title or self . api_title
version = version or self . api_version
info = info or self . swagger . get ( 'info' , { } )
swagger = swagger or self . swagger
base_path = base_path or self . base_path
swagger = swagger . copy ( )
info . update ( title = title , version = version )
swagger . update ( swagger = '2.0' , info = info , basePath = base_path )
paths , tags = self . _build_paths ( )
# Update the provided tags with the extracted ones preserving order
if tags :
    swagger . setdefault ( 'tags' , [ ] )
    tag_names = { t [ 'name' ] for t in swagger [ 'tags' ] }
    for tag in tags :
        if tag [ 'name' ] not in tag_names :
            swagger [ 'tags' ] . append ( tag )
# Create / Update swagger sections with extracted values where not provided
if paths :
    swagger . setdefault ( 'paths' , { } )
    merge_dicts ( swagger [ 'paths' ] , paths )
definitions = self . definitions . definition_registry
if definitions :
    swagger . setdefault ( 'definitions' , { } )
    merge_dicts ( swagger [ 'definitions' ] , definitions )
parameters = self . parameters . parameter_registry
if parameters :
    swagger . setdefault ( 'parameters' , { } )
    merge_dicts ( swagger [ 'parameters' ] , parameters )
responses = self . responses . response_registry
if responses :
    swagger . setdefault ( 'responses' , { } )
    merge_dicts ( swagger [ 'responses' ] , responses )
return swagger


signature:
def _build_paths ( self ) :
    """Build the Swagger " paths " and " tags " attributes from cornice service
 definitions ."""

implementation:
paths = { }
tags = [ ]
for service in self . services :
    path , path_obj = self . _extract_path_from_service ( service )
    service_tags = getattr ( service , 'tags' , [ ] )
    self . _check_tags ( service_tags )
    tags = self . _get_tags ( tags , service_tags )
    for method , view , args in service . definitions :
        if method . lower ( ) in map ( str . lower , self . ignore_methods ) :
            continue
        op = self . _extract_operation_from_view ( view , args )
        if any ( ctype in op . get ( 'consumes' , [ ] ) for ctype in self . ignore_ctypes ) :
            continue
        # XXX : Swagger doesn ' t support different schemas for for a same method
        # with different ctypes as cornice . If this happens , you may ignore one
        # content - type from the documentation otherwise we raise an Exception
        # Related to https : / / github . com / OAI / OpenAPI - Specification / issues / 146
        previous_definition = path_obj . get ( method . lower ( ) )
        if previous_definition :
            raise CorniceSwaggerException ( ( "Swagger doesn't support multiple " "views for a same method. You may " "ignore one." ) )
        # If tag not defined and a default tag is provided
        if 'tags' not in op and self . default_tags :
            if callable ( self . default_tags ) :
                op [ 'tags' ] = self . default_tags ( service , method )
            else :
                op [ 'tags' ] = self . default_tags
        op_tags = op . get ( 'tags' , [ ] )
        self . _check_tags ( op_tags )
        # Add service tags
        if service_tags :
            new_tags = service_tags + op_tags
            op [ 'tags' ] = list ( OrderedDict . fromkeys ( new_tags ) )
        # Add method tags to root tags
        tags = self . _get_tags ( tags , op_tags )
        # If operation id is not defined and a default generator is provided
        if 'operationId' not in op and self . default_op_ids :
            if not callable ( self . default_op_ids ) :
                raise CorniceSwaggerException ( 'default_op_id should be a callable.' )
            op [ 'operationId' ] = self . default_op_ids ( service , method )
        # If security options not defined and default is provided
        if 'security' not in op and self . default_security :
            if callable ( self . default_security ) :
                op [ 'security' ] = self . default_security ( service , method )
            else :
                op [ 'security' ] = self . default_security
        if not isinstance ( op . get ( 'security' , [ ] ) , list ) :
            raise CorniceSwaggerException ( 'security should be a list or callable' )
        path_obj [ method . lower ( ) ] = op
    paths [ path ] = path_obj
return paths , tags


signature:
def _extract_path_from_service ( self , service ) :
    """Extract path object and its parameters from service definitions .
 : param service :
 Cornice service to extract information from .
 : rtype : dict
 : returns : Path definition ."""

implementation:
path_obj = { }
path = service . path
route_name = getattr ( service , 'pyramid_route' , None )
# handle services that don ' t create fresh routes ,
# we still need the paths so we need to grab pyramid introspector to
# extract that information
if route_name : # avoid failure if someone forgets to pass registry
    registry = self . pyramid_registry or get_current_registry ( )
    route_intr = registry . introspector . get ( 'routes' , route_name )
    if route_intr :
        path = route_intr [ 'pattern' ]
    else :
        msg = 'Route `{}` is not found by ' 'pyramid introspector' . format ( route_name )
        raise ValueError ( msg )
# handle traverse and subpath as regular parameters
# docs . pylonsproject . org / projects / pyramid / en / latest / narr / hybrid . html
for subpath_marker in ( '*subpath' , '*traverse' ) :
    path = path . replace ( subpath_marker , '{subpath}' )
# Extract path parameters
parameters = self . parameters . from_path ( path )
if parameters :
    path_obj [ 'parameters' ] = parameters
return path , path_obj


signature:
def _extract_operation_from_view ( self , view , args ) :
    """Extract swagger operation details from colander view definitions .
 : param view :
 View to extract information from .
 : param args :
 Arguments from the view decorator .
 : rtype : dict
 : returns : Operation definition ."""

implementation:
op = { 'responses' : { 'default' : { 'description' : 'UNDOCUMENTED RESPONSE' } } , }
# If ' produces ' are not defined in the view , try get from renderers
renderer = args . get ( 'renderer' , '' )
if "json" in renderer : # allows for " json " or " simplejson "
    produces = [ 'application/json' ]
elif renderer == 'xml' :
    produces = [ 'text/xml' ]
else :
    produces = None
if produces :
    op . setdefault ( 'produces' , produces )
# Get explicit accepted content - types
consumes = args . get ( 'content_type' )
if consumes is not None : # convert to a list , if it ' s not yet one
    consumes = to_list ( consumes )
    # It is possible to add callables for content _ type , so we have to
    # to filter those out , since we cannot evaluate those here .
    consumes = [ x for x in consumes if not callable ( x ) ]
    op [ 'consumes' ] = consumes
# Get parameters from view schema
is_colander = self . _is_colander_schema ( args )
if is_colander :
    schema = self . _extract_transform_colander_schema ( args )
    parameters = self . parameters . from_schema ( schema )
else : # Bail out for now
    parameters = None
if parameters :
    op [ 'parameters' ] = parameters
# Get summary from docstring
if isinstance ( view , six . string_types ) :
    if 'klass' in args :
        ob = args [ 'klass' ]
        view_ = getattr ( ob , view . lower ( ) )
        docstring = trim ( view_ . __doc__ )
else :
    docstring = str ( trim ( view . __doc__ ) )
if docstring and self . summary_docstrings :
    op [ 'summary' ] = docstring
# Get response definitions
if 'response_schemas' in args :
    op [ 'responses' ] = self . responses . from_schema_mapping ( args [ 'response_schemas' ] )
# Get response tags
if 'tags' in args :
    op [ 'tags' ] = args [ 'tags' ]
# Get response operationId
if 'operation_id' in args :
    op [ 'operationId' ] = args [ 'operation_id' ]
# Get security policies
if 'api_security' in args :
    op [ 'security' ] = args [ 'api_security' ]
return op


signature:
def _extract_transform_colander_schema ( self , args ) :
    """Extract schema from view args and transform it using
 the pipeline of schema transformers
 : param args :
 Arguments from the view decorator .
 : rtype : colander . MappingSchema ( )
 : returns : View schema cloned and transformed"""

implementation:
schema = args . get ( 'schema' , colander . MappingSchema ( ) )
if not isinstance ( schema , colander . Schema ) :
    schema = schema ( )
schema = schema . clone ( )
for transformer in self . schema_transformers :
    schema = transformer ( schema , args )
return schema


signature:
def main ( ) :
    """Creates arguments and parses user input"""

implementation:
parser = argparse . ArgumentParser ( description = _ ( 'Uploads selected file to working pomf.se clone' ) )
parser . add_argument ( 'files' , metavar = 'file' , nargs = '*' , type = str , help = _ ( 'Files to upload' ) )
parser . add_argument ( '-c' , metavar = 'host_number' , type = int , dest = 'host' , default = None , help = _ ( 'The number (0-n) of the selected host (default is random)' ) )
parser . add_argument ( '-l' , dest = 'only_link' , action = 'store_const' , const = True , default = False , help = _ ( 'Changes output to just link to the file' ) )
parser . add_argument ( '-e' , dest = 'encrypt' , action = 'store_const' , const = True , default = False , help = _ ( 'Encrypts then uploads the files.' ) )
parser . add_argument ( '-d' , dest = 'decrypt' , action = 'store_const' , const = True , default = False , help = _ ( 'Decrypts files from links with encrypted files' ) )
parser . add_argument ( '-j' , dest = "local_list" , default = False , help = _ ( 'Path to a local list file' ) )
parser . add_argument ( '-s' , dest = "show_list" , action = 'store_const' , const = True , default = False , help = _ ( 'Show the host list (will not upload your files when called)' ) )
parser . add_argument ( '-m' , dest = 'limit_size' , action = 'store_const' , const = True , default = False , help = _ ( 'Do not upload file if it exceeds the certain host limit' ) )
parser . add_argument ( '-nc' , dest = 'no_cloudflare' , action = 'store_const' , const = True , default = False , help = _ ( 'Do not use hosts which use Cloudflare.' ) )
parser . add_argument ( '--log-file' , metavar = "LOGFILE" , dest = "logfile" , default = "~/limf.log" , help = _ ( "The location of log file" ) )
parser . add_argument ( '--log' , dest = 'log' , action = "store_const" , const = True , default = False , help = _ ( "Enables the logging feature, default logfile is ~/limf.log" ) )
args = parser . parse_args ( )
try :
    if args . local_list :
        clone_list = retrieve_local_host_list ( args . local_list )
    else :
        clone_list = retrieve_online_host_list ( )
    if len ( min ( clone_list , key = len ) ) < 5 and ( args . limit_size or args . no_cloudflare ) :
        print ( _ ( "For newer options, please update your host_list." ) )
        exit ( )
    if args . host and not ( 0 <= args . host < len ( clone_list ) ) :
        print ( generate_host_string ( clone_list ) )
        exit ( )
    parse_arguments ( args , clone_list )
except FileNotFoundError :
    print ( _ ( 'Plese enter valid file.' ) )


signature:
def convert ( self , schema_node , definition_handler ) :
    """Convert node schema into a parameter object ."""

implementation:
converted = { 'name' : schema_node . name , 'in' : self . _in , 'required' : schema_node . required }
if schema_node . description :
    converted [ 'description' ] = schema_node . description
if schema_node . default :
    converted [ 'default' ] = schema_node . default
schema = definition_handler ( schema_node )
# Parameters shouldn ' t have a title
schema . pop ( 'title' , None )
converted . update ( schema )
if schema . get ( 'type' ) == 'array' :
    converted [ 'items' ] = { 'type' : schema [ 'items' ] [ 'type' ] }
return converted


signature:
def cornice_enable_openapi_view ( config , api_path = '/api-explorer/swagger.json' , permission = NO_PERMISSION_REQUIRED , route_factory = None , ** kwargs ) :
    """: param config :
 Pyramid configurator object
 : param api _ path :
 where to expose swagger JSON definition view
 : param permission :
 pyramid permission for those views
 : param route _ factory :
 factory for context object for those routes
 : param kwargs :
 kwargs that will be passed to CorniceSwagger ' s ` generate ( ) `
 This registers and configures the view that serves api definitions"""

implementation:
config . registry . settings [ 'cornice_swagger.spec_kwargs' ] = kwargs
config . add_route ( 'cornice_swagger.open_api_path' , api_path , factory = route_factory )
config . add_view ( 'cornice_swagger.views.open_api_json_view' , renderer = 'json' , permission = permission , route_name = 'cornice_swagger.open_api_path' )


signature:
def cornice_enable_openapi_explorer ( config , api_explorer_path = '/api-explorer' , permission = NO_PERMISSION_REQUIRED , route_factory = None , ** kwargs ) :
    """: param config :
 Pyramid configurator object
 : param api _ explorer _ path :
 where to expose Swagger UI interface view
 : param permission :
 pyramid permission for those views
 : param route _ factory :
 factory for context object for those routes
 This registers and configures the view that serves api explorer"""

implementation:
config . add_route ( 'cornice_swagger.api_explorer_path' , api_explorer_path , factory = route_factory )
config . add_view ( 'cornice_swagger.views.swagger_ui_template_view' , permission = permission , route_name = 'cornice_swagger.api_explorer_path' )


signature:
def trim ( docstring ) :
    """Remove the tabs to spaces , and remove the extra spaces / tabs that are in
 front of the text in docstrings .
 Implementation taken from http : / / www . python . org / dev / peps / pep - 0257/"""

implementation:
if not docstring :
    return ''
# Convert tabs to spaces ( following the normal Python rules )
# and split into a list of lines :
lines = six . u ( docstring ) . expandtabs ( ) . splitlines ( )
lines = [ line . strip ( ) for line in lines ]
res = six . u ( '\n' ) . join ( lines )
return res


signature:
def merge_dicts ( base , changes ) :
    """Merge b into a recursively , without overwriting values .
 : param base : the dict that will be altered .
 : param changes : changes to update base ."""

implementation:
for k , v in changes . items ( ) :
    if isinstance ( v , dict ) :
        merge_dicts ( base . setdefault ( k , { } ) , v )
    else :
        base . setdefault ( k , v )


signature:
def get_transition_viewset_method ( transition_name , ** kwargs ) :
    '''Create a viewset method for the provided ` transition _ name `'''

implementation:
@ detail_route ( methods = [ 'post' ] , ** kwargs )
def inner_func ( self , request , pk = None , ** kwargs ) :
    object = self . get_object ( )
    transition_method = getattr ( object , transition_name )
    transition_method ( by = self . request . user )
    if self . save_after_transition :
        object . save ( )
    serializer = self . get_serializer ( object )
    return Response ( serializer . data )
return inner_func


signature:
def get_viewset_transition_action_mixin ( model , ** kwargs ) :
    '''Find all transitions defined on ` model ` , then create a corresponding
 viewset action method for each and apply it to ` Mixin ` . Finally , return
 ` Mixin `'''

implementation:
instance = model ( )
class Mixin ( object ) :
    save_after_transition = True
transitions = instance . get_all_status_transitions ( )
transition_names = set ( x . name for x in transitions )
for transition_name in transition_names :
    setattr ( Mixin , transition_name , get_transition_viewset_method ( transition_name , ** kwargs ) )
return Mixin


signature:
def fresh_cookies ( ctx , mold = '' ) :
    """Refresh the project from the original cookiecutter template ."""

implementation:
mold = mold or "https://github.com/Springerle/py-generic-project.git"
# TODO : URL from config
tmpdir = os . path . join ( tempfile . gettempdir ( ) , "cc-upgrade-pygments-markdown-lexer" )
if os . path . isdir ( '.git' ) : # TODO : Ensure there are no local unstashed changes
    pass
# Make a copy of the new mold version
if os . path . isdir ( tmpdir ) :
    shutil . rmtree ( tmpdir )
if os . path . exists ( mold ) :
    shutil . copytree ( mold , tmpdir , ignore = shutil . ignore_patterns ( ".git" , ".svn" , "*~" , ) )
else :
    ctx . run ( "git clone {} {}" . format ( mold , tmpdir ) )
# Copy recorded " cookiecutter . json " into mold
shutil . copy2 ( "project.d/cookiecutter.json" , tmpdir )
with pushd ( '..' ) :
    ctx . run ( "cookiecutter --no-input {}" . format ( tmpdir ) )
if os . path . exists ( '.git' ) :
    ctx . run ( "git status" )


signature:
def ci ( ctx ) :
    """Perform continuous integration tasks ."""

implementation:
opts = [ '' ]
# ' tox ' makes no sense in Travis
if os . environ . get ( 'TRAVIS' , '' ) . lower ( ) == 'true' :
    opts += [ 'test.pytest' ]
else :
    opts += [ 'test.tox' ]
ctx . run ( "invoke --echo --pty clean --all build --docs check --reports{}" . format ( ' ' . join ( opts ) ) )


signature:
def _build_metadata ( ) : # pylint : disable = too - many - locals , too - many - branches
    "Return project ' s metadata as a dict ."

implementation:
# Handle metadata in package source
expected_keys = ( 'url' , 'version' , 'license' , 'author' , 'author_email' , 'long_description' , 'keywords' )
metadata = { }
with io . open ( srcfile ( 'src' , package_name , '__init__.py' ) , encoding = 'utf-8' ) as handle :
    pkg_init = handle . read ( )
    # Get default long description from docstring
    metadata [ 'long_description' ] = re . search ( r'^"""(.+?)^"""$' , pkg_init , re . DOTALL | re . MULTILINE ) . group ( 1 )
    for line in pkg_init . splitlines ( ) :
        match = re . match ( r"""^__({0})__ += (?P<q>['"])(.+?)(?P=q)$""" . format ( '|' . join ( expected_keys ) ) , line )
        if match :
            metadata [ match . group ( 1 ) ] = match . group ( 3 )
if not all ( i in metadata for i in expected_keys ) :
    raise RuntimeError ( "Missing or bad metadata in '{0}' package: {1}" . format ( name , ', ' . join ( sorted ( set ( expected_keys ) - set ( metadata . keys ( ) ) ) ) , ) )
text = metadata [ 'long_description' ] . strip ( )
if text :
    metadata [ 'description' ] , text = text . split ( '.' , 1 )
    metadata [ 'description' ] = ' ' . join ( metadata [ 'description' ] . split ( ) ) . strip ( ) + '.'
    # normalize whitespace
    metadata [ 'long_description' ] = textwrap . dedent ( text ) . strip ( )
metadata [ 'keywords' ] = metadata [ 'keywords' ] . replace ( ',' , ' ' ) . strip ( ) . split ( )
# Load requirements files
requirements_files = dict ( install = 'requirements.txt' , setup = 'setup-requirements.txt' , test = 'test-requirements.txt' , )
requires = { }
for key , filename in requirements_files . items ( ) :
    requires [ key ] = [ ]
    if os . path . exists ( srcfile ( filename ) ) :
        with io . open ( srcfile ( filename ) , encoding = 'utf-8' ) as handle :
            for line in handle :
                line = line . strip ( )
                if line and not line . startswith ( '#' ) :
                    if any ( line . startswith ( i ) for i in ( '-e' , 'http://' , 'https://' ) ) :
                        line = line . split ( '#egg=' ) [ 1 ]
                    requires [ key ] . append ( line )
if not any ( 'pytest' == re . split ( '[\t ,<=>]' , i . lower ( ) ) [ 0 ] for i in requires [ 'test' ] ) :
    requires [ 'test' ] . append ( 'pytest' )
# add missing requirement
# CLI entry points
console_scripts = [ ]
for path , dirs , files in os . walk ( srcfile ( 'src' , package_name ) ) :
    dirs = [ i for i in dirs if not i . startswith ( '.' ) ]
    if '__main__.py' in files :
        path = path [ len ( srcfile ( 'src' ) + os . sep ) : ]
        appname = path . split ( os . sep ) [ - 1 ]
        with io . open ( srcfile ( 'src' , path , '__main__.py' ) , encoding = 'utf-8' ) as handle :
            for line in handle . readlines ( ) :
                match = re . match ( r"""^__app_name__ += (?P<q>['"])(.+?)(?P=q)$""" , line )
                if match :
                    appname = match . group ( 2 )
        console_scripts . append ( '{0} = {1}.__main__:cli' . format ( appname , path . replace ( os . sep , '.' ) ) )
# Add some common files to EGG - INFO
candidate_files = [ 'LICENSE' , 'NOTICE' , 'README' , 'README.md' , 'README.rst' , 'README.txt' , 'CHANGES' , 'CHANGELOG' , 'debian/changelog' , ]
data_files = defaultdict ( list )
for filename in candidate_files :
    if os . path . exists ( srcfile ( filename ) ) :
        data_files [ 'EGG-INFO' ] . append ( filename )
# Complete project metadata
classifiers = [ ]
for classifiers_txt in ( 'classifiers.txt' , 'project.d/classifiers.txt' ) :
    classifiers_txt = srcfile ( classifiers_txt )
    if os . path . exists ( classifiers_txt ) :
        with io . open ( classifiers_txt , encoding = 'utf-8' ) as handle :
            classifiers = [ i . strip ( ) for i in handle if i . strip ( ) and not i . startswith ( '#' ) ]
        break
entry_points . setdefault ( 'console_scripts' , [ ] ) . extend ( console_scripts )
metadata . update ( dict ( name = name , package_dir = { '' : 'src' } , packages = find_packages ( srcfile ( 'src' ) , exclude = [ 'tests' ] ) , data_files = data_files . items ( ) , zip_safe = False , include_package_data = True , install_requires = requires [ 'install' ] , setup_requires = requires [ 'setup' ] , tests_require = requires [ 'test' ] , classifiers = classifiers , cmdclass = dict ( test = PyTest , ) , entry_points = entry_points , ) )
return metadata


signature:
def py_hash ( key , num_buckets ) :
    """Generate a number in the range [ 0 , num _ buckets ) .
 Args :
 key ( int ) : The key to hash .
 num _ buckets ( int ) : Number of buckets to use .
 Returns :
 The bucket number ` key ` computes to .
 Raises :
 ValueError : If ` num _ buckets ` is not a positive number ."""

implementation:
b , j = - 1 , 0
if num_buckets < 1 :
    raise ValueError ( 'num_buckets must be a positive number' )
while j < num_buckets :
    b = int ( j )
    key = ( ( key * long ( 2862933555777941757 ) ) + 1 ) & 0xffffffffffffffff
    j = float ( b + 1 ) * ( float ( 1 << 31 ) / float ( ( key >> 33 ) + 1 ) )
return int ( b )


signature:
def setup ( app ) :
    """Initializer for Sphinx extension API .
 See http : / / www . sphinx - doc . org / en / stable / extdev / index . html # dev - extensions ."""

implementation:
lexer = MarkdownLexer ( )
for alias in lexer . aliases :
    app . add_lexer ( alias , lexer )
return dict ( version = __version__ )


signature:
def load ( self ) :
    """Return a dict of stats ."""

implementation:
ret = { }
# Read the mdstat file
with open ( self . get_path ( ) , 'r' ) as f : # lines is a list of line ( with \ n )
    lines = f . readlines ( )
# First line : get the personalities
# The " Personalities " line tells you what RAID level the kernel currently supports .
# This can be changed by either changing the raid modules or recompiling the kernel .
# Possible personalities include : [ raid0 ] [ raid1 ] [ raid4 ] [ raid5 ] [ raid6 ] [ linear ] [ multipath ] [ faulty ]
ret [ 'personalities' ] = self . get_personalities ( lines [ 0 ] )
# Second to last before line : Array definition
ret [ 'arrays' ] = self . get_arrays ( lines [ 1 : - 1 ] , ret [ 'personalities' ] )
# Save the file content as it for the _ _ str _ _ method
self . content = reduce ( lambda x , y : x + y , lines )
return ret


signature:
def get_personalities ( self , line ) :
    """Return a list of personalities readed from the input line ."""

implementation: return [ split ( '\W+' , i ) [ 1 ] for i in line . split ( ':' ) [ 1 ] . split ( ' ' ) if i . startswith ( '[' ) ]


signature:
def get_arrays ( self , lines , personalities = [ ] ) :
    """Return a dict of arrays ."""

implementation:
ret = { }
i = 0
while i < len ( lines ) :
    try : # First array line : get the md device
        md_device = self . get_md_device_name ( lines [ i ] )
    except IndexError : # No array detected
        pass
    else : # Array detected
        if md_device is not None : # md device line
            ret [ md_device ] = self . get_md_device ( lines [ i ] , personalities )
            # md config / status line
            i += 1
            ret [ md_device ] . update ( self . get_md_status ( lines [ i ] ) )
    i += 1
return ret


signature:
def get_md_device ( self , line , personalities = [ ] ) :
    """Return a dict of md device define in the line ."""

implementation:
ret = { }
splitted = split ( '\W+' , line )
# Raid status
# Active or ' started ' . An inactive array is usually faulty .
# Stopped arrays aren ' t visible here .
ret [ 'status' ] = splitted [ 1 ]
if splitted [ 2 ] in personalities : # Raid type ( ex : RAID5)
    ret [ 'type' ] = splitted [ 2 ]
    # Array ' s components
    ret [ 'components' ] = self . get_components ( line , with_type = True )
else : # Raid type ( ex : RAID5)
    ret [ 'type' ] = None
    # Array ' s components
    ret [ 'components' ] = self . get_components ( line , with_type = False )
return ret


signature:
def get_md_status ( self , line ) :
    """Return a dict of md status define in the line ."""

implementation:
ret = { }
splitted = split ( '\W+' , line )
if len ( splitted ) < 7 :
    ret [ 'available' ] = None
    ret [ 'used' ] = None
    ret [ 'config' ] = None
else : # The final 2 entries on this line : [ n / m ] [ UUUU _ ]
# [ n / m ] means that ideally the array would have n devices however , currently , m devices are in use .
# Obviously when m > = n then things are good .
    ret [ 'available' ] = splitted [ - 4 ]
    ret [ 'used' ] = splitted [ - 3 ]
    # [ UUUU _ ] represents the status of each device , either U for up or _ for down .
    ret [ 'config' ] = splitted [ - 2 ]
return ret


signature:
def get_components ( self , line , with_type = True ) :
    """Return a dict of components in the line .
 key : device name ( ex : ' sdc1 ' )
 value : device role number"""

implementation:
ret = { }
# Ignore ( F ) ( see test 08)
line2 = reduce ( lambda x , y : x + y , split ( '\(.+\)' , line ) )
if with_type :
    splitted = split ( '\W+' , line2 ) [ 3 : ]
else :
    splitted = split ( '\W+' , line2 ) [ 2 : ]
ret = dict ( zip ( splitted [ 0 : : 2 ] , splitted [ 1 : : 2 ] ) )
return ret


signature:
def register_receivers ( app , config ) :
    """Register signal receivers which send events ."""

implementation:
for event_name , event_config in config . items ( ) :
    event_builders = [ obj_or_import_string ( func ) for func in event_config . get ( 'event_builders' , [ ] ) ]
    signal = obj_or_import_string ( event_config [ 'signal' ] )
    signal . connect ( EventEmmiter ( event_name , event_builders ) , sender = app , weak = False )


signature:
def check ( self , query ) :
    """: param query :"""

implementation:
if query . get_type ( ) != Keyword . DELETE :
    return Ok ( True )
return Err ( "Delete queries are forbidden." )


signature:
def set_scheduled ( self ) :
    """Returns True if state was successfully changed from idle to scheduled ."""

implementation:
with self . _idle_lock :
    if self . _idle :
        self . _idle = False
        return True
return False


signature:
def post ( self , ** kwargs ) :
    """Get statistics ."""

implementation:
data = request . get_json ( force = False )
if data is None :
    data = { }
result = { }
for query_name , config in data . items ( ) :
    if config is None or not isinstance ( config , dict ) or ( set ( config . keys ( ) ) != { 'stat' , 'params' } and set ( config . keys ( ) ) != { 'stat' } ) :
        raise InvalidRequestInputError ( 'Invalid Input. It should be of the form ' '{ STATISTIC_NAME: { "stat": STAT_TYPE, ' '"params": STAT_PARAMS \}}' )
    stat = config [ 'stat' ]
    params = config . get ( 'params' , { } )
    try :
        query_cfg = current_stats . queries [ stat ]
    except KeyError :
        raise UnknownQueryError ( stat )
    permission = current_stats . permission_factory ( stat , params )
    if permission is not None and not permission . can ( ) :
        message = ( 'You do not have a permission to query the ' 'statistic "{}" with those ' 'parameters' . format ( stat ) )
        if current_user . is_authenticated :
            abort ( 403 , message )
        abort ( 401 , message )
    try :
        query = query_cfg . query_class ( ** query_cfg . query_config )
        result [ query_name ] = query . run ( ** params )
    except ValueError as e :
        raise InvalidRequestInputError ( e . args [ 0 ] )
    except NotFoundError as e :
        return None
return self . make_response ( result )


implementation:
if query . get_type ( ) not in { Keyword . SELECT , Keyword . DELETE } : # Only select and delete queries deal with time durations
# All others are not affected by this rule . Bailing out .
    return Ok ( True )
datapoints = query . get_datapoints ( )
if datapoints <= self . max_datapoints :
    return Ok ( True )
return Err ( ( "Expecting {} datapoints from that query, which is above the threshold! " "Set a date range (e.g. where time > now() - 24h), " "increase grouping (e.g. group by time(24h) " "or limit the number of datapoints (e.g. limit 100)" ) . format ( datapoints ) )


signature:
def _get_oldest_event_timestamp ( self ) :
    """Search for the oldest event timestamp ."""

implementation:
# Retrieve the oldest event in order to start aggregation
# from there
query_events = Search ( using = self . client , index = self . event_index ) [ 0 : 1 ] . sort ( { 'timestamp' : { 'order' : 'asc' } } )
result = query_events . execute ( )
# There might not be any events yet if the first event have been
# indexed but the indices have not been refreshed yet .
if len ( result ) == 0 :
    return None
return parser . parse ( result [ 0 ] [ 'timestamp' ] )


signature:
def get_bookmark ( self ) :
    """Get last aggregation date ."""

implementation:
if not Index ( self . aggregation_alias , using = self . client ) . exists ( ) :
    if not Index ( self . event_index , using = self . client ) . exists ( ) :
        return datetime . date . today ( )
    return self . _get_oldest_event_timestamp ( )
# retrieve the oldest bookmark
query_bookmark = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . bookmark_doc_type ) [ 0 : 1 ] . sort ( { 'date' : { 'order' : 'desc' } } )
bookmarks = query_bookmark . execute ( )
# if no bookmark is found but the index exist , the bookmark was somehow
# lost or never written , so restart from the beginning
if len ( bookmarks ) == 0 :
    return self . _get_oldest_event_timestamp ( )
# change it to doc _ id _ suffix
bookmark = datetime . datetime . strptime ( bookmarks [ 0 ] . date , self . doc_id_suffix )
return bookmark


signature:
def set_bookmark ( self ) :
    """Set bookmark for starting next aggregation ."""

implementation:
def _success_date ( ) :
    bookmark = { 'date' : self . new_bookmark or datetime . datetime . utcnow ( ) . strftime ( self . doc_id_suffix ) }
    yield dict ( _index = self . last_index_written , _type = self . bookmark_doc_type , _source = bookmark )
if self . last_index_written :
    bulk ( self . client , _success_date ( ) , stats_only = True )


signature:
def _format_range_dt ( self , d ) :
    """Format range filter datetime to the closest aggregation interval ."""

implementation:
if not isinstance ( d , six . string_types ) :
    d = d . isoformat ( )
return '{0}||/{1}' . format ( d , self . dt_rounding_map [ self . aggregation_interval ] )


signature:
def agg_iter ( self , lower_limit = None , upper_limit = None ) :
    """Aggregate and return dictionary to be indexed in ES ."""

implementation:
lower_limit = lower_limit or self . get_bookmark ( ) . isoformat ( )
upper_limit = upper_limit or ( datetime . datetime . utcnow ( ) . replace ( microsecond = 0 ) . isoformat ( ) )
aggregation_data = { }
self . agg_query = Search ( using = self . client , index = self . event_index ) . filter ( 'range' , timestamp = { 'gte' : self . _format_range_dt ( lower_limit ) , 'lte' : self . _format_range_dt ( upper_limit ) } )
# apply query modifiers
for modifier in self . query_modifiers :
    self . agg_query = modifier ( self . agg_query )
hist = self . agg_query . aggs . bucket ( 'histogram' , 'date_histogram' , field = 'timestamp' , interval = self . aggregation_interval )
terms = hist . bucket ( 'terms' , 'terms' , field = self . aggregation_field , size = 0 )
top = terms . metric ( 'top_hit' , 'top_hits' , size = 1 , sort = { 'timestamp' : 'desc' } )
for dst , ( metric , src , opts ) in self . metric_aggregation_fields . items ( ) :
    terms . metric ( dst , metric , field = src , ** opts )
results = self . agg_query . execute ( )
index_name = None
for interval in results . aggregations [ 'histogram' ] . buckets :
    interval_date = datetime . datetime . strptime ( interval [ 'key_as_string' ] , '%Y-%m-%dT%H:%M:%S' )
    for aggregation in interval [ 'terms' ] . buckets :
        aggregation_data [ 'timestamp' ] = interval_date . isoformat ( )
        aggregation_data [ self . aggregation_field ] = aggregation [ 'key' ]
        aggregation_data [ 'count' ] = aggregation [ 'doc_count' ]
        if self . metric_aggregation_fields :
            for f in self . metric_aggregation_fields :
                aggregation_data [ f ] = aggregation [ f ] [ 'value' ]
        doc = aggregation . top_hit . hits . hits [ 0 ] [ '_source' ]
        for destination , source in self . copy_fields . items ( ) :
            if isinstance ( source , six . string_types ) :
                aggregation_data [ destination ] = doc [ source ]
            else :
                aggregation_data [ destination ] = source ( doc , aggregation_data )
        index_name = 'stats-{0}-{1}' . format ( self . event , interval_date . strftime ( self . index_name_suffix ) )
        self . indices . add ( index_name )
        yield dict ( _id = '{0}-{1}' . format ( aggregation [ 'key' ] , interval_date . strftime ( self . doc_id_suffix ) ) , _index = index_name , _type = self . aggregation_doc_type , _source = aggregation_data )
self . last_index_written = index_name


signature:
def run ( self , start_date = None , end_date = None , update_bookmark = True ) :
    """Calculate statistics aggregations ."""

implementation:
# If no events have been indexed there is nothing to aggregate
if not Index ( self . event_index , using = self . client ) . exists ( ) :
    return
lower_limit = start_date or self . get_bookmark ( )
# Stop here if no bookmark could be estimated .
if lower_limit is None :
    return
upper_limit = min ( end_date or datetime . datetime . max , # ignore if ` None `
datetime . datetime . utcnow ( ) . replace ( microsecond = 0 ) , datetime . datetime . combine ( lower_limit + datetime . timedelta ( self . batch_size ) , datetime . datetime . min . time ( ) ) )
while upper_limit <= datetime . datetime . utcnow ( ) :
    self . indices = set ( )
    self . new_bookmark = upper_limit . strftime ( self . doc_id_suffix )
    bulk ( self . client , self . agg_iter ( lower_limit , upper_limit ) , stats_only = True , chunk_size = 50 )
    # Flush all indices which have been modified
    current_search_client . indices . flush ( index = ',' . join ( self . indices ) , wait_if_ongoing = True )
    if update_bookmark :
        self . set_bookmark ( )
    self . indices = set ( )
    lower_limit = lower_limit + datetime . timedelta ( self . batch_size )
    upper_limit = min ( end_date or datetime . datetime . max , # ignore if ` None ` `
    datetime . datetime . utcnow ( ) . replace ( microsecond = 0 ) , lower_limit + datetime . timedelta ( self . batch_size ) )
    if lower_limit > upper_limit :
        break


signature:
def list_bookmarks ( self , start_date = None , end_date = None , limit = None ) :
    """List the aggregation ' s bookmarks ."""

implementation:
query = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . bookmark_doc_type ) . sort ( { 'date' : { 'order' : 'desc' } } )
range_args = { }
if start_date :
    range_args [ 'gte' ] = self . _format_range_dt ( start_date . replace ( microsecond = 0 ) )
if end_date :
    range_args [ 'lte' ] = self . _format_range_dt ( end_date . replace ( microsecond = 0 ) )
if range_args :
    query = query . filter ( 'range' , date = range_args )
return query [ 0 : limit ] . execute ( ) if limit else query . scan ( )


signature:
def delete ( self , start_date = None , end_date = None ) :
    """Delete aggregation documents ."""

implementation:
aggs_query = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . aggregation_doc_type ) . extra ( _source = False )
range_args = { }
if start_date :
    range_args [ 'gte' ] = self . _format_range_dt ( start_date . replace ( microsecond = 0 ) )
if end_date :
    range_args [ 'lte' ] = self . _format_range_dt ( end_date . replace ( microsecond = 0 ) )
if range_args :
    aggs_query = aggs_query . filter ( 'range' , timestamp = range_args )
bookmarks_query = Search ( using = self . client , index = self . aggregation_alias , doc_type = self . bookmark_doc_type ) . sort ( { 'date' : { 'order' : 'desc' } } )
if range_args :
    bookmarks_query = bookmarks_query . filter ( 'range' , date = range_args )
def _delete_actions ( ) :
    for query in ( aggs_query , bookmarks_query ) :
        affected_indices = set ( )
        for doc in query . scan ( ) :
            affected_indices . add ( doc . meta . index )
            yield dict ( _index = doc . meta . index , _op_type = 'delete' , _id = doc . meta . id , _type = doc . meta . doc_type )
        current_search_client . indices . flush ( index = ',' . join ( affected_indices ) , wait_if_ongoing = True )
bulk ( self . client , _delete_actions ( ) , refresh = True )


signature:
def parse ( self , group_by_stmt ) :
    """Extract the data resolution of a query in seconds
 E . g . " group by time ( 99s ) " = > 99
 : param group _ by _ stmt : A raw InfluxDB group by statement"""

implementation:
if not group_by_stmt :
    return Resolution . MAX_RESOLUTION
m = self . GROUP_BY_TIME_PATTERN . match ( group_by_stmt )
if not m :
    return None
value = int ( m . group ( 1 ) )
unit = m . group ( 2 )
resolution = self . convert_to_seconds ( value , unit )
# We can ' t have a higher resolution than the max resolution
return max ( resolution , Resolution . MAX_RESOLUTION )


signature:
def get ( self , timeout = None ) :
    """Return value on success , or raise exception on failure ."""

implementation:
result = None
try :
    result = self . _result . get ( True , timeout = timeout )
except Empty :
    raise Timeout ( )
if isinstance ( result , Failure ) :
    six . reraise ( * result . exc_info )
else :
    return result


signature:
def _events_process ( event_types = None , eager = False ) :
    """Process stats events ."""

implementation:
event_types = event_types or list ( current_stats . enabled_events )
if eager :
    process_events . apply ( ( event_types , ) , throw = True )
    click . secho ( 'Events processed successfully.' , fg = 'green' )
else :
    process_events . delay ( event_types )
    click . secho ( 'Events processing task sent...' , fg = 'yellow' )


signature:
def _aggregations_process ( aggregation_types = None , start_date = None , end_date = None , update_bookmark = False , eager = False ) :
    """Process stats aggregations ."""

implementation:
aggregation_types = ( aggregation_types or list ( current_stats . enabled_aggregations ) )
if eager :
    aggregate_events . apply ( ( aggregation_types , ) , dict ( start_date = start_date , end_date = end_date , update_bookmark = update_bookmark ) , throw = True )
    click . secho ( 'Aggregations processed successfully.' , fg = 'green' )
else :
    aggregate_events . delay ( aggregation_types , start_date = start_date , end_date = end_date )
    click . secho ( 'Aggregations processing task sent...' , fg = 'yellow' )


signature:
def _aggregations_delete ( aggregation_types = None , start_date = None , end_date = None ) :
    """Delete computed aggregations ."""

implementation:
aggregation_types = ( aggregation_types or list ( current_stats . enabled_aggregations ) )
for a in aggregation_types :
    aggr_cfg = current_stats . aggregations [ a ]
    aggregator = aggr_cfg . aggregator_class ( name = aggr_cfg . name , ** aggr_cfg . aggregator_config )
    aggregator . delete ( start_date , end_date )


signature:
def _aggregations_list_bookmarks ( aggregation_types = None , start_date = None , end_date = None , limit = None ) :
    """List aggregation bookmarks ."""

implementation:
aggregation_types = ( aggregation_types or list ( current_stats . enabled_aggregations ) )
for a in aggregation_types :
    aggr_cfg = current_stats . aggregations [ a ]
    aggregator = aggr_cfg . aggregator_class ( name = aggr_cfg . name , ** aggr_cfg . aggregator_config )
    bookmarks = aggregator . list_bookmarks ( start_date , end_date , limit )
    click . echo ( '{}:' . format ( a ) )
    for b in bookmarks :
        click . echo ( ' - {}' . format ( b . date ) )


signature:
def _events_config ( self ) :
    """Load events configuration ."""

implementation:
# import iter _ entry _ points here so that it can be mocked in tests
result = { }
for ep in iter_entry_points ( group = self . entry_point_group_events ) :
    for cfg in ep . load ( ) ( ) :
        if cfg [ 'event_type' ] not in self . enabled_events :
            continue
        elif cfg [ 'event_type' ] in result :
            raise DuplicateEventError ( 'Duplicate event {0} in entry point ' '{1}' . format ( cfg [ 'event_type' ] , ep . name ) )
        # Update the default configuration with env / overlay config .
        cfg . update ( self . enabled_events [ cfg [ 'event_type' ] ] or { } )
        result [ cfg [ 'event_type' ] ] = cfg
return result


signature:
def _aggregations_config ( self ) :
    """Load aggregation configurations ."""

implementation:
result = { }
for ep in iter_entry_points ( group = self . entry_point_group_aggs ) :
    for cfg in ep . load ( ) ( ) :
        if cfg [ 'aggregation_name' ] not in self . enabled_aggregations :
            continue
        elif cfg [ 'aggregation_name' ] in result :
            raise DuplicateAggregationError ( 'Duplicate aggregation {0} in entry point ' '{1}' . format ( cfg [ 'event_type' ] , ep . name ) )
        # Update the default configuration with env / overlay config .
        cfg . update ( self . enabled_aggregations [ cfg [ 'aggregation_name' ] ] or { } )
        result [ cfg [ 'aggregation_name' ] ] = cfg
return result


signature:
def _queries_config ( self ) :
    """Load queries configuration ."""

implementation:
result = { }
for ep in iter_entry_points ( group = self . entry_point_group_queries ) :
    for cfg in ep . load ( ) ( ) :
        if cfg [ 'query_name' ] not in self . enabled_queries :
            continue
        elif cfg [ 'query_name' ] in result :
            raise DuplicateQueryError ( 'Duplicate query {0} in entry point ' '{1}' . format ( cfg [ 'query' ] , ep . name ) )
        # Update the default configuration with env / overlay config .
        cfg . update ( self . enabled_queries [ cfg [ 'query_name' ] ] or { } )
        result [ cfg [ 'query_name' ] ] = cfg
return result


signature:
def publish ( self , event_type , events ) :
    """Publish events ."""

implementation:
assert event_type in self . events
current_queues . queues [ 'stats-{}' . format ( event_type ) ] . publish ( events )


signature:
def consume ( self , event_type , no_ack = True , payload = True ) :
    """Comsume all pending events ."""

implementation:
assert event_type in self . events
return current_queues . queues [ 'stats-{}' . format ( event_type ) ] . consume ( payload = payload )


signature:
def init_app ( self , app , entry_point_group_events = 'invenio_stats.events' , entry_point_group_aggs = 'invenio_stats.aggregations' , entry_point_group_queries = 'invenio_stats.queries' ) :
    """Flask application initialization ."""

implementation:
self . init_config ( app )
state = _InvenioStatsState ( app , entry_point_group_events = entry_point_group_events , entry_point_group_aggs = entry_point_group_aggs , entry_point_group_queries = entry_point_group_queries )
self . _state = app . extensions [ 'invenio-stats' ] = state
if app . config [ 'STATS_REGISTER_RECEIVERS' ] :
    signal_receivers = { key : value for key , value in app . config . get ( 'STATS_EVENTS' , { } ) . items ( ) if 'signal' in value }
    register_receivers ( app , signal_receivers )
return state