signature:
def parse_comment_telemetry ( text ) :
    """Looks for base91 telemetry found in comment field
 Returns [ remaining _ text , telemetry ]"""

implementation:
parsed = { }
match = re . findall ( r"^(.*?)\|([!-{]{4,14})\|(.*)$" , text )
if match and len ( match [ 0 ] [ 1 ] ) % 2 == 0 :
    text , telemetry , post = match [ 0 ]
    text += post
    temp = [ 0 ] * 7
    for i in range ( 7 ) :
        temp [ i ] = base91 . to_decimal ( telemetry [ i * 2 : i * 2 + 2 ] )
    parsed . update ( { 'telemetry' : { 'seq' : temp [ 0 ] , 'vals' : temp [ 1 : 6 ] } } )
    if temp [ 6 ] != '' :
        parsed [ 'telemetry' ] . update ( { 'bits' : "{0:08b}" . format ( temp [ 6 ] & 0xFF ) [ : : - 1 ] } )
return ( text , parsed )


signature:
def parse ( packet ) :
    """Parses an APRS packet and returns a dict with decoded data
 - All attributes are in metric units"""

implementation:
if not isinstance ( packet , string_type_parse ) :
    raise TypeError ( "Expected packet to be str/unicode/bytes, got %s" , type ( packet ) )
if len ( packet ) == 0 :
    raise ParseError ( "packet is empty" , packet )
# attempt to detect encoding
if isinstance ( packet , bytes ) :
    packet = _unicode_packet ( packet )
packet = packet . rstrip ( "\r\n" )
logger . debug ( "Parsing: %s" , packet )
# split into head and body
try :
    ( head , body ) = packet . split ( ':' , 1 )
except :
    raise ParseError ( "packet has no body" , packet )
if len ( body ) == 0 :
    raise ParseError ( "packet body is empty" , packet )
parsed = { 'raw' : packet , }
# parse head
try :
    parsed . update ( parse_header ( head ) )
except ParseError as msg :
    raise ParseError ( str ( msg ) , packet )
# parse body
packet_type = body [ 0 ]
body = body [ 1 : ]
if len ( body ) == 0 and packet_type != '>' :
    raise ParseError ( "packet body is empty after packet type character" , packet )
# attempt to parse the body
try :
    _try_toparse_body ( packet_type , body , parsed )
# capture ParseErrors and attach the packet
except ( UnknownFormat , ParseError ) as exp :
    exp . packet = packet
    raise
# if we fail all attempts to parse , try beacon packet
if 'format' not in parsed :
    if not re . match ( r"^(AIR.*|ALL.*|AP.*|BEACON|CQ.*|GPS.*|DF.*|DGPS.*|" "DRILL.*|DX.*|ID.*|JAVA.*|MAIL.*|MICE.*|QST.*|QTH.*|" "RTCM.*|SKY.*|SPACE.*|SPC.*|SYM.*|TEL.*|TEST.*|TLM.*|" "WX.*|ZIP.*|UIDIGI)$" , parsed [ 'to' ] ) :
        raise UnknownFormat ( "format is not supported" , packet )
    parsed . update ( { 'format' : 'beacon' , 'text' : packet_type + body , } )
logger . debug ( "Parsed ok." )
return parsed


signature:
def to_decimal ( text ) :
    """Takes a base91 char string and returns decimal"""

implementation:
if not isinstance ( text , string_type ) :
    raise TypeError ( "expected str or unicode, %s given" % type ( text ) )
if findall ( r"[\x00-\x20\x7c-\xff]" , text ) :
    raise ValueError ( "invalid character in sequence" )
text = text . lstrip ( '!' )
decimal = 0
length = len ( text ) - 1
for i , char in enumerate ( text ) :
    decimal += ( ord ( char ) - 33 ) * ( 91 ** ( length - i ) )
return decimal if text != '' else 0


signature:
def from_decimal ( number , width = 1 ) :
    """Takes a decimal and returns base91 char string .
 With optional parameter for fix with output"""

implementation:
text = [ ]
if not isinstance ( number , int_type ) :
    raise TypeError ( "Expected number to be int, got %s" , type ( number ) )
elif not isinstance ( width , int_type ) :
    raise TypeError ( "Expected width to be int, got %s" , type ( number ) )
elif number < 0 :
    raise ValueError ( "Expected number to be positive integer" )
elif number > 0 :
    max_n = ceil ( log ( number ) / log ( 91 ) )
    for n in _range ( int ( max_n ) , - 1 , - 1 ) :
        quotient , number = divmod ( number , 91 ** n )
        text . append ( chr ( 33 + quotient ) )
return "" . join ( text ) . lstrip ( '!' ) . rjust ( max ( 1 , width ) , '!' )


signature:
def passcode ( callsign ) :
    """Takes a CALLSIGN and returns passcode"""

implementation:
assert isinstance ( callsign , str )
callsign = callsign . split ( '-' ) [ 0 ] . upper ( )
code = 0x73e2
for i , char in enumerate ( callsign ) :
    code ^= ord ( char ) << ( 8 if not i % 2 else 0 )
return code & 0x7fff


signature:
def parse_header ( head ) :
    """Parses the header part of packet
 Returns a dict"""

implementation:
try :
    ( fromcall , path ) = head . split ( '>' , 1 )
except :
    raise ParseError ( "invalid packet header" )
if ( not 1 <= len ( fromcall ) <= 9 or not re . findall ( r"^[a-z0-9]{0,9}(\-[a-z0-9]{1,8})?$" , fromcall , re . I ) ) :
    raise ParseError ( "fromcallsign is invalid" )
path = path . split ( ',' )
if len ( path [ 0 ] ) == 0 :
    raise ParseError ( "no tocallsign in header" )
tocall = path [ 0 ]
path = path [ 1 : ]
validate_callsign ( tocall , "tocallsign" )
for digi in path :
    if not re . findall ( r"^[A-Z0-9\-]{1,9}\*?$" , digi , re . I ) :
        raise ParseError ( "invalid callsign in path" )
parsed = { 'from' : fromcall , 'to' : tocall , 'path' : path , }
viacall = ""
if len ( path ) >= 2 and re . match ( r"^q..$" , path [ - 2 ] ) :
    viacall = path [ - 1 ]
parsed . update ( { 'via' : viacall } )
return parsed


signature:
def set_filter ( self , filter_text ) :
    """Set a specified aprs - is filter for this connection"""

implementation:
self . filter = filter_text
self . logger . info ( "Setting filter to: %s" , self . filter )
if self . _connected :
    self . _sendall ( "#filter %s\r\n" % self . filter )


signature:
def set_login ( self , callsign , passwd = "-1" , skip_login = False ) :
    """Set callsign and password"""

implementation: self . __dict__ . update ( locals ( ) )


signature:
def connect ( self , blocking = False , retry = 30 ) :
    """Initiate connection to APRS server and attempt to login
 blocking = False - Should we block until connected and logged - in
 retry = 30 - Retry interval in seconds"""

implementation:
if self . _connected :
    return
while True :
    try :
        self . _connect ( )
        if not self . skip_login :
            self . _send_login ( )
        break
    except ( LoginError , ConnectionError ) :
        if not blocking :
            raise
    self . logger . info ( "Retrying connection is %d seconds." % retry )
    time . sleep ( retry )


signature:
def close ( self ) :
    """Closes the socket
 Called internally when Exceptions are raised"""

implementation:
self . _connected = False
self . buf = b''
if self . sock is not None :
    self . sock . close ( )


signature:
def sendall ( self , line ) :
    """Send a line , or multiple lines sperapted by ' \\ r \\ n '"""

implementation:
if isinstance ( line , APRSPacket ) :
    line = str ( line )
elif not isinstance ( line , string_type ) :
    raise TypeError ( "Expected line to be str or APRSPacket, got %s" , type ( line ) )
if not self . _connected :
    raise ConnectionError ( "not connected" )
if line == "" :
    return
line = line . rstrip ( "\r\n" ) + "\r\n"
try :
    self . sock . setblocking ( 1 )
    self . sock . settimeout ( 5 )
    self . _sendall ( line )
except socket . error as exp :
    self . close ( )
    raise ConnectionError ( str ( exp ) )


signature:
def consumer ( self , callback , blocking = True , immortal = False , raw = False ) :
    """When a position sentence is received , it will be passed to the callback function
 blocking : if true ( default ) , runs forever , otherwise will return after one sentence
 You can still exit the loop , by raising StopIteration in the callback function
 immortal : When true , consumer will try to reconnect and stop propagation of Parse exceptions
 if false ( default ) , consumer will return
 raw : when true , raw packet is passed to callback , otherwise the result from aprs . parse ( )"""

implementation:
if not self . _connected :
    raise ConnectionError ( "not connected to a server" )
line = b''
while True :
    try :
        for line in self . _socket_readlines ( blocking ) :
            if line [ 0 : 1 ] != b'#' :
                if raw :
                    callback ( line )
                else :
                    callback ( self . _parse ( line ) )
            else :
                self . logger . debug ( "Server: %s" , line . decode ( 'utf8' ) )
    except ParseError as exp :
        self . logger . log ( 11 , "%s\n    Packet: %s" , exp . message , exp . packet )
    except UnknownFormat as exp :
        self . logger . log ( 9 , "%s\n    Packet: %s" , exp . message , exp . packet )
    except LoginError as exp :
        self . logger . error ( "%s: %s" , exp . __class__ . __name__ , exp . message )
    except ( KeyboardInterrupt , SystemExit ) :
        raise
    except ( ConnectionDrop , ConnectionError ) :
        self . close ( )
        if not immortal :
            raise
        else :
            self . connect ( blocking = blocking )
            continue
    except GenericError :
        pass
    except StopIteration :
        break
    except :
        self . logger . error ( "APRS Packet: %s" , line )
        raise
    if not blocking :
        break


signature:
def _connect ( self ) :
    """Attemps connection to the server"""

implementation:
self . logger . info ( "Attempting connection to %s:%s" , self . server [ 0 ] , self . server [ 1 ] )
try :
    self . _open_socket ( )
    peer = self . sock . getpeername ( )
    self . logger . info ( "Connected to %s" , str ( peer ) )
    # 5 second timeout to receive server banner
    self . sock . setblocking ( 1 )
    self . sock . settimeout ( 5 )
    self . sock . setsockopt ( socket . SOL_SOCKET , socket . SO_KEEPALIVE , 1 )
    banner = self . sock . recv ( 512 )
    if is_py3 :
        banner = banner . decode ( 'latin-1' )
    if banner [ 0 ] == "#" :
        self . logger . debug ( "Banner: %s" , banner . rstrip ( ) )
    else :
        raise ConnectionError ( "invalid banner from server" )
except ConnectionError as e :
    self . logger . error ( str ( e ) )
    self . close ( )
    raise
except ( socket . error , socket . timeout ) as e :
    self . close ( )
    self . logger . error ( "Socket error: %s" % str ( e ) )
    if str ( e ) == "timed out" :
        raise ConnectionError ( "no banner from server" )
    else :
        raise ConnectionError ( e )
self . _connected = True


signature:
def _send_login ( self ) :
    """Sends login string to server"""

implementation:
login_str = "user {0} pass {1} vers aprslib {3}{2}\r\n"
login_str = login_str . format ( self . callsign , self . passwd , ( " filter " + self . filter ) if self . filter != "" else "" , __version__ )
self . logger . info ( "Sending login information" )
try :
    self . _sendall ( login_str )
    self . sock . settimeout ( 5 )
    test = self . sock . recv ( len ( login_str ) + 100 )
    if is_py3 :
        test = test . decode ( 'latin-1' )
    test = test . rstrip ( )
    self . logger . debug ( "Server: %s" , test )
    _ , _ , callsign , status , _ = test . split ( ' ' , 4 )
    if callsign == "" :
        raise LoginError ( "Server responded with empty callsign???" )
    if callsign != self . callsign :
        raise LoginError ( "Server: %s" % test )
    if status != "verified," and self . passwd != "-1" :
        raise LoginError ( "Password is incorrect" )
    if self . passwd == "-1" :
        self . logger . info ( "Login successful (receive only)" )
    else :
        self . logger . info ( "Login successful" )
except LoginError as e :
    self . logger . error ( str ( e ) )
    self . close ( )
    raise
except :
    self . close ( )
    self . logger . error ( "Failed to login" )
    raise LoginError ( "Failed to login" )


signature:
def _socket_readlines ( self , blocking = False ) :
    """Generator for complete lines , received from the server"""

implementation:
try :
    self . sock . setblocking ( 0 )
except socket . error as e :
    self . logger . error ( "socket error when setblocking(0): %s" % str ( e ) )
    raise ConnectionDrop ( "connection dropped" )
while True :
    short_buf = b''
    newline = b'\r\n'
    select . select ( [ self . sock ] , [ ] , [ ] , None if blocking else 0 )
    try :
        short_buf = self . sock . recv ( 4096 )
        # sock . recv returns empty if the connection drops
        if not short_buf :
            self . logger . error ( "socket.recv(): returned empty" )
            raise ConnectionDrop ( "connection dropped" )
    except socket . error as e :
        self . logger . error ( "socket error on recv(): %s" % str ( e ) )
        if "Resource temporarily unavailable" in str ( e ) :
            if not blocking :
                if len ( self . buf ) == 0 :
                    break
    self . buf += short_buf
    while newline in self . buf :
        line , self . buf = self . buf . split ( newline , 1 )
        yield line


signature:
def db_value ( self , value ) :
    """Convert UUID to binary blob"""

implementation:
# ensure we have a valid UUID
if not isinstance ( value , UUID ) :
    value = UUID ( value )
# reconstruct for optimal indexing
parts = str ( value ) . split ( "-" )
reordered = '' . join ( [ parts [ 2 ] , parts [ 1 ] , parts [ 0 ] , parts [ 3 ] , parts [ 4 ] ] )
value = binascii . unhexlify ( reordered )
return super ( OrderedUUIDField , self ) . db_value ( value )


signature:
def python_value ( self , value ) :
    """Convert binary blob to UUID instance"""

implementation:
value = super ( OrderedUUIDField , self ) . python_value ( value )
u = binascii . b2a_hex ( value )
value = u [ 8 : 16 ] + u [ 4 : 8 ] + u [ 0 : 4 ] + u [ 16 : 22 ] + u [ 22 : 32 ]
return UUID ( value . decode ( ) )


signature:
def db_value ( self , value ) :
    """Convert the python value for storage in the database ."""

implementation:
value = self . transform_value ( value )
return self . hhash . encrypt ( value , salt_size = self . salt_size , rounds = self . rounds )


signature:
def python_value ( self , value ) :
    """Convert the database value to a pythonic value ."""

implementation:
value = coerce_to_bytes ( value )
obj = HashValue ( value )
obj . field = self
return obj


signature:
def register ( self , model_cls ) :
    """Register model ( s ) with app"""

implementation:
assert issubclass ( model_cls , peewee . Model )
assert not hasattr ( model_cls . _meta , 'database_manager' )
if model_cls in self :
    raise RuntimeError ( "Model already registered" )
self . append ( model_cls )
model_cls . _meta . database = self . dbm
return model_cls


signature:
def disconnect ( self ) :
    """Disconnect from all databases"""

implementation:
for name , connection in self . items ( ) :
    if not connection . is_closed ( ) :
        connection . close ( )


signature:
def get_database ( self , model ) :
    """Find matching database router"""

implementation:
for router in self . routers :
    r = router . get_database ( model )
    if r is not None :
        return r
return self . get ( 'default' )


signature:
def to_cursor_ref ( self ) :
    """Returns dict of values to uniquely reference this item"""

implementation:
fields = self . _meta . get_primary_keys ( )
assert fields
values = { field . name : self . __data__ [ field . name ] for field in fields }
return values


signature:
def paginate_query ( self , query , count , offset = None , sort = None ) :
    """Apply pagination to query
 : attr query : Instance of ` peewee . Query `
 : attr count : Max rows to return
 : attr offset : Pagination offset , str / int
 : attr sort : List of tuples , e . g . [ ( ' id ' , ' asc ' ) ]
 : returns : Instance of ` peewee . Query `"""

implementation:
assert isinstance ( query , peewee . Query )
assert isinstance ( count , int )
assert isinstance ( offset , ( str , int , type ( None ) ) )
assert isinstance ( sort , ( list , set , tuple , type ( None ) ) )
# ensure our model has a primary key
fields = query . model . _meta . get_primary_keys ( )
if len ( fields ) == 0 :
    raise peewee . ProgrammingError ( 'Cannot apply pagination on model without primary key' )
# ensure our model doesn ' t use a compound primary key
if len ( fields ) > 1 :
    raise peewee . ProgrammingError ( 'Cannot apply pagination on model with compound primary key' )
# apply offset
if offset is not None :
    query = query . where ( fields [ 0 ] >= offset )
# do we need to apply sorting ?
order_bys = [ ]
if sort :
    for field , direction in sort : # does this field have a valid sort direction ?
        if not isinstance ( direction , str ) :
            raise ValueError ( "Invalid sort direction on field '{}'" . format ( field ) )
        direction = direction . lower ( ) . strip ( )
        if direction not in [ 'asc' , 'desc' ] :
            raise ValueError ( "Invalid sort direction on field '{}'" . format ( field ) )
        # apply sorting
        order_by = peewee . SQL ( field )
        order_by = getattr ( order_by , direction ) ( )
        order_bys += [ order_by ]
# add primary key ordering after user sorting
order_bys += [ fields [ 0 ] . asc ( ) ]
# apply ordering and limits
query = query . order_by ( * order_bys )
query = query . limit ( count )
return query


signature:
def apply_filters ( self , query , filters ) :
    """Apply user specified filters to query"""

implementation:
assert isinstance ( query , peewee . Query )
assert isinstance ( filters , dict )


signature:
def retrieve ( self , cursor ) :
    """Retrieve items from query"""

implementation:
assert isinstance ( cursor , dict ) , "expected cursor type 'dict'"
# look for record in query
query = self . get_query ( )
assert isinstance ( query , peewee . Query )
query
return query . get ( ** cursor )


signature:
def regenerate_signing_key ( self , secret_key = None , region = None , service = None , date = None ) :
    """Regenerate the signing key for this instance . Store the new key in
 signing _ key property .
 Take scope elements of the new key from the equivalent properties
 ( region , service , date ) of the current AWS4Auth instance . Scope
 elements can be overridden for the new key by supplying arguments to
 this function . If overrides are supplied update the current AWS4Auth
 instance ' s equivalent properties to match the new values .
 If secret _ key is not specified use the value of the secret _ key property
 of the current AWS4Auth instance ' s signing key . If the existing signing
 key is not storing its secret key ( i . e . store _ secret _ key was set to
 False at instantiation ) then raise a NoSecretKeyError and do not
 regenerate the key . In order to regenerate a key which is not storing
 its secret key , secret _ key must be supplied to this function .
 Use the value of the existing key ' s store _ secret _ key property when
 generating the new key . If there is no existing key , then default
 to setting store _ secret _ key to True for new key ."""

implementation:
if secret_key is None and ( self . signing_key is None or self . signing_key . secret_key is None ) :
    raise NoSecretKeyError
secret_key = secret_key or self . signing_key . secret_key
region = region or self . region
service = service or self . service
date = date or self . date
if self . signing_key is None :
    store_secret_key = True
else :
    store_secret_key = self . signing_key . store_secret_key
self . signing_key = AWS4SigningKey ( secret_key , region , service , date , store_secret_key )
self . region = region
self . service = service
self . date = self . signing_key . date


signature:
def get_request_date ( cls , req ) :
    """Try to pull a date from the request by looking first at the
 x - amz - date header , and if that ' s not present then the Date header .
 Return a datetime . date object , or None if neither date header
 is found or is in a recognisable format .
 req - - a requests PreparedRequest object"""

implementation:
date = None
for header in [ 'x-amz-date' , 'date' ] :
    if header not in req . headers :
        continue
    try :
        date_str = cls . parse_date ( req . headers [ header ] )
    except DateFormatError :
        continue
    try :
        date = datetime . datetime . strptime ( date_str , '%Y-%m-%d' ) . date ( )
    except ValueError :
        continue
    else :
        break
return date


signature:
def parse_date ( date_str ) :
    """Check if date _ str is in a recognised format and return an ISO
 yyyy - mm - dd format version if so . Raise DateFormatError if not .
 Recognised formats are :
 * RFC 7231 ( e . g . Mon , 09 Sep 2011 23:36:00 GMT )
 * RFC 850 ( e . g . Sunday , 06 - Nov - 94 08:49:37 GMT )
 * C time ( e . g . Wed Dec 4 00:00:00 2002)
 * Amz - Date format ( e . g . 20090325T010101Z )
 * ISO 8601 / RFC 3339 ( e . g . 2009-03-25T10:11:12.13-01:00)
 date _ str - - Str containing a date and optional time"""


signature:
def handle_date_mismatch ( self , req ) :
    """Handle a request whose date doesn ' t match the signing key scope date .
 This AWS4Auth class implementation regenerates the signing key . See
 StrictAWS4Auth class if you would prefer an exception to be raised .
 req - - a requests prepared request object"""

implementation:
req_datetime = self . get_request_date ( req )
new_key_date = req_datetime . strftime ( '%Y%m%d' )
self . regenerate_signing_key ( date = new_key_date )


signature:
def encode_body ( req ) :
    """Encode body of request to bytes and update content - type if required .
 If the body of req is Unicode then encode to the charset found in
 content - type header if present , otherwise UTF - 8 , or ASCII if
 content - type is application / x - www - form - urlencoded . If encoding to UTF - 8
 then add charset to content - type . Modifies req directly , does not
 return a modified copy .
 req - - Requests PreparedRequest object"""

implementation:
if isinstance ( req . body , text_type ) :
    split = req . headers . get ( 'content-type' , 'text/plain' ) . split ( ';' )
    if len ( split ) == 2 :
        ct , cs = split
        cs = cs . split ( '=' ) [ 1 ]
        req . body = req . body . encode ( cs )
    else :
        ct = split [ 0 ]
        if ( ct == 'application/x-www-form-urlencoded' or 'x-amz-' in ct ) :
            req . body = req . body . encode ( )
        else :
            req . body = req . body . encode ( 'utf-8' )
            req . headers [ 'content-type' ] = ct + '; charset=utf-8'


signature:
def get_canonical_request ( self , req , cano_headers , signed_headers ) :
    """Create the AWS authentication Canonical Request string .
 req - - Requests PreparedRequest object . Should already
 include an x - amz - content - sha256 header
 cano _ headers - - Canonical Headers section of Canonical Request , as
 returned by get _ canonical _ headers ( )
 signed _ headers - - Signed Headers , as returned by
 get _ canonical _ headers ( )"""

implementation:
url = urlparse ( req . url )
path = self . amz_cano_path ( url . path )
# AWS handles " extreme " querystrings differently to urlparse
# ( see post - vanilla - query - nonunreserved test in aws _ testsuite )
split = req . url . split ( '?' , 1 )
qs = split [ 1 ] if len ( split ) == 2 else ''
qs = self . amz_cano_querystring ( qs )
payload_hash = req . headers [ 'x-amz-content-sha256' ]
req_parts = [ req . method . upper ( ) , path , qs , cano_headers , signed_headers , payload_hash ]
cano_req = '\n' . join ( req_parts )
return cano_req


signature:
def get_canonical_headers ( cls , req , include = None ) :
    """Generate the Canonical Headers section of the Canonical Request .
 Return the Canonical Headers and the Signed Headers strs as a tuple
 ( canonical _ headers , signed _ headers ) .
 req - - Requests PreparedRequest object
 include - - List of headers to include in the canonical and signed
 headers . It ' s primarily included to allow testing against
 specific examples from Amazon . If omitted or None it
 includes host , content - type and any header starting ' x - amz - '
 except for x - amz - client context , which appears to break
 mobile analytics auth if included . Except for the
 x - amz - client - context exclusion these defaults are per the
 AWS documentation ."""

implementation:
if include is None :
    include = cls . default_include_headers
include = [ x . lower ( ) for x in include ]
headers = req . headers . copy ( )
# Temporarily include the host header - AWS requires it to be included
# in the signed headers , but Requests doesn ' t include it in a
# PreparedRequest
if 'host' not in headers :
    headers [ 'host' ] = urlparse ( req . url ) . netloc . split ( ':' ) [ 0 ]
# Aggregate for upper / lowercase header name collisions in header names ,
# AMZ requires values of colliding headers be concatenated into a
# single header with lowercase name . Although this is not possible with
# Requests , since it uses a case - insensitive dict to hold headers , this
# is here just in case you duck type with a regular dict
cano_headers_dict = { }
for hdr , val in headers . items ( ) :
    hdr = hdr . strip ( ) . lower ( )
    val = cls . amz_norm_whitespace ( val ) . strip ( )
    if ( hdr in include or '*' in include or ( 'x-amz-*' in include and hdr . startswith ( 'x-amz-' ) and not hdr == 'x-amz-client-context' ) ) :
        vals = cano_headers_dict . setdefault ( hdr , [ ] )
        vals . append ( val )
# Flatten cano _ headers dict to string and generate signed _ headers
cano_headers = ''
signed_headers_list = [ ]
for hdr in sorted ( cano_headers_dict ) :
    vals = cano_headers_dict [ hdr ]
    val = ',' . join ( sorted ( vals ) )
    cano_headers += '{}:{}\n' . format ( hdr , val )
    signed_headers_list . append ( hdr )
signed_headers = ';' . join ( signed_headers_list )
return ( cano_headers , signed_headers )


signature:
def get_sig_string ( req , cano_req , scope ) :
    """Generate the AWS4 auth string to sign for the request .
 req - - Requests PreparedRequest object . This should already
 include an x - amz - date header .
 cano _ req - - The Canonical Request , as returned by
 get _ canonical _ request ( )"""

implementation:
amz_date = req . headers [ 'x-amz-date' ]
hsh = hashlib . sha256 ( cano_req . encode ( ) )
sig_items = [ 'AWS4-HMAC-SHA256' , amz_date , scope , hsh . hexdigest ( ) ]
sig_string = '\n' . join ( sig_items )
return sig_string


signature:
def amz_cano_path ( self , path ) :
    """Generate the canonical path as per AWS4 auth requirements .
 Not documented anywhere , determined from aws4 _ testsuite examples ,
 problem reports and testing against the live services .
 path - - request path"""

implementation:
safe_chars = '/~'
qs = ''
fixed_path = path
if '?' in fixed_path :
    fixed_path , qs = fixed_path . split ( '?' , 1 )
fixed_path = posixpath . normpath ( fixed_path )
fixed_path = re . sub ( '/+' , '/' , fixed_path )
if path . endswith ( '/' ) and not fixed_path . endswith ( '/' ) :
    fixed_path += '/'
full_path = fixed_path
# If Python 2 , switch to working entirely in str as quote ( ) has problems
# with Unicode
if PY2 :
    full_path = full_path . encode ( 'utf-8' )
    safe_chars = safe_chars . encode ( 'utf-8' )
    qs = qs . encode ( 'utf-8' )
# S3 seems to require unquoting first . ' host ' service is used in
# amz _ testsuite tests
if self . service in [ 's3' , 'host' ] :
    full_path = unquote ( full_path )
full_path = quote ( full_path , safe = safe_chars )
if qs :
    qm = b'?' if PY2 else '?'
    full_path = qm . join ( ( full_path , qs ) )
if PY2 :
    full_path = unicode ( full_path )
return full_path


signature:
def amz_cano_querystring ( qs ) :
    """Parse and format querystring as per AWS4 auth requirements .
 Perform percent quoting as needed .
 qs - - querystring"""

implementation:
safe_qs_amz_chars = '&=+'
safe_qs_unresvd = '-_.~'
# If Python 2 , switch to working entirely in str
# as quote ( ) has problems with Unicode
if PY2 :
    qs = qs . encode ( 'utf-8' )
    safe_qs_amz_chars = safe_qs_amz_chars . encode ( )
    safe_qs_unresvd = safe_qs_unresvd . encode ( )
qs = unquote ( qs )
space = b' ' if PY2 else ' '
qs = qs . split ( space ) [ 0 ]
qs = quote ( qs , safe = safe_qs_amz_chars )
qs_items = { }
for name , vals in parse_qs ( qs , keep_blank_values = True ) . items ( ) :
    name = quote ( name , safe = safe_qs_unresvd )
    vals = [ quote ( val , safe = safe_qs_unresvd ) for val in vals ]
    qs_items [ name ] = vals
qs_strings = [ ]
for name , vals in qs_items . items ( ) :
    for val in vals :
        qs_strings . append ( '=' . join ( [ name , val ] ) )
qs = '&' . join ( sorted ( qs_strings ) )
if PY2 :
    qs = unicode ( qs )
return qs


signature:
def generate_key ( cls , secret_key , region , service , date , intermediates = False ) :
    """Generate the signing key string as bytes .
 If intermediate is set to True , returns a 4 - tuple containing the key
 and the intermediate keys :
 ( signing _ key , date _ key , region _ key , service _ key )
 The intermediate keys can be used for testing against examples from
 Amazon ."""

implementation:
init_key = ( 'AWS4' + secret_key ) . encode ( 'utf-8' )
date_key = cls . sign_sha256 ( init_key , date )
region_key = cls . sign_sha256 ( date_key , region )
service_key = cls . sign_sha256 ( region_key , service )
key = cls . sign_sha256 ( service_key , 'aws4_request' )
if intermediates :
    return ( key , date_key , region_key , service_key )
else :
    return key


signature:
def sign_sha256 ( key , msg ) :
    """Generate an SHA256 HMAC , encoding msg to UTF - 8 if not
 already encoded .
 key - - signing key . bytes .
 msg - - message to sign . unicode or bytes ."""

implementation:
if isinstance ( msg , text_type ) :
    msg = msg . encode ( 'utf-8' )
return hmac . new ( key , msg , hashlib . sha256 ) . digest ( )


signature:
def _format_datetime ( dttm ) :
    """Convert a datetime object into a valid STIX timestamp string .
 1 . Convert to timezone - aware
 2 . Convert to UTC
 3 . Format in ISO format
 4 . Ensure correct precision
 a . Add subsecond value if non - zero and precision not defined
 5 . Add " Z " """

implementation:
if dttm . tzinfo is None or dttm . tzinfo . utcoffset ( dttm ) is None : # dttm is timezone - naive ; assume UTC
    zoned = pytz . utc . localize ( dttm )
else :
    zoned = dttm . astimezone ( pytz . utc )
ts = zoned . strftime ( "%Y-%m-%dT%H:%M:%S" )
ms = zoned . strftime ( "%f" )
precision = getattr ( dttm , "precision" , None )
if precision == "second" :
    pass
    # Already precise to the second
elif precision == "millisecond" :
    ts = ts + "." + ms [ : 3 ]
elif zoned . microsecond > 0 :
    ts = ts + "." + ms . rstrip ( "0" )
return ts + "Z"


signature:
def _ensure_datetime_to_string ( maybe_dttm ) :
    """If maybe _ dttm is a datetime instance , convert to a STIX - compliant
 string representation . Otherwise return the value unchanged ."""

implementation:
if isinstance ( maybe_dttm , datetime . datetime ) :
    maybe_dttm = _format_datetime ( maybe_dttm )
return maybe_dttm


signature:
def _filter_kwargs_to_query_params ( filter_kwargs ) :
    """Convert API keyword args to a mapping of URL query parameters . Except for
 " added _ after " , all keywords are mapped to match filters , i . e . to a query
 parameter of the form " match [ < kwarg > ] " . " added _ after " is left alone , since
 it ' s a special filter , as defined in the spec .
 Each value can be a single value or iterable of values . " version " and
 " added _ after " get special treatment , since they are timestamp - valued :
 datetime . datetime instances are supported and automatically converted to
 STIX - compliant strings . Other than that , all values must be strings . None
 values , empty lists , etc are silently ignored .
 Args :
 filter _ kwargs : The filter information , as a mapping .
 Returns :
 query _ params ( dict ) : The query parameter map , mapping strings to
 strings ."""

implementation:
query_params = { }
for kwarg , arglist in six . iteritems ( filter_kwargs ) : # If user passes an empty list , None , etc , silently skip ?
    if not arglist :
        continue
    # force iterability , for the sake of code uniformity
    if not hasattr ( arglist , "__iter__" ) or isinstance ( arglist , six . string_types ) :
        arglist = arglist ,
    if kwarg == "version" :
        query_params [ "match[version]" ] = "," . join ( _ensure_datetime_to_string ( val ) for val in arglist )
    elif kwarg == "added_after" :
        if len ( arglist ) > 1 :
            raise InvalidArgumentsError ( "No more than one value for filter" " 'added_after' may be given" )
        query_params [ "added_after" ] = "," . join ( _ensure_datetime_to_string ( val ) for val in arglist )
    else :
        query_params [ "match[" + kwarg + "]" ] = "," . join ( arglist )
return query_params


signature:
def _to_json ( resp ) :
    """Factors out some JSON parse code with error handling , to hopefully improve
 error messages .
 : param resp : A " requests " library response
 : return : Parsed JSON .
 : raises : InvalidJSONError If JSON parsing failed ."""

implementation:
try :
    return resp . json ( )
except ValueError as e : # Maybe better to report the original request URL ?
    six . raise_from ( InvalidJSONError ( "Invalid JSON was received from " + resp . request . url ) , e )


signature:
def refresh ( self , accept = MEDIA_TYPE_TAXII_V20 ) :
    """Updates Status information"""

implementation:
response = self . __raw = self . _conn . get ( self . url , headers = { "Accept" : accept } )
self . _populate_fields ( ** response )


signature:
def wait_until_final ( self , poll_interval = 1 , timeout = 60 ) :
    """It will poll the URL to grab the latest status resource in a given
 timeout and time interval .
 Args :
 poll _ interval ( int ) : how often to poll the status service .
 timeout ( int ) : how long to poll the URL until giving up . Use < = 0
 to wait forever"""

implementation:
start_time = time . time ( )
elapsed = 0
while ( self . status != "complete" and ( timeout <= 0 or elapsed < timeout ) ) :
    time . sleep ( poll_interval )
    self . refresh ( )
    elapsed = time . time ( ) - start_time


signature:
def _validate_status ( self ) :
    """Validates Status information . Raises errors for required
 properties ."""

implementation:
if not self . id :
    msg = "No 'id' in Status for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if not self . status :
    msg = "No 'status' in Status for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . total_count is None :
    msg = "No 'total_count' in Status for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . success_count is None :
    msg = "No 'success_count' in Status for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . failure_count is None :
    msg = "No 'failure_count' in Status for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . pending_count is None :
    msg = "No 'pending_count' in Status for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if len ( self . successes ) != self . success_count :
    msg = "Found successes={}, but success_count={} in status '{}'"
    raise ValidationError ( msg . format ( self . successes , self . success_count , self . id ) )
if len ( self . pendings ) != self . pending_count :
    msg = "Found pendings={}, but pending_count={} in status '{}'"
    raise ValidationError ( msg . format ( self . pendings , self . pending_count , self . id ) )
if len ( self . failures ) != self . failure_count :
    msg = "Found failures={}, but failure_count={} in status '{}'"
    raise ValidationError ( msg . format ( self . failures , self . failure_count , self . id ) )
if ( self . success_count + self . pending_count + self . failure_count != self . total_count ) :
    msg = ( "(success_count={} + pending_count={} + " "failure_count={}) != total_count={} in status '{}'" )
    raise ValidationError ( msg . format ( self . success_count , self . pending_count , self . failure_count , self . total_count , self . id ) )


signature:
def _validate_collection ( self ) :
    """Validates Collection information . Raises errors for required
 properties ."""

implementation:
if not self . _id :
    msg = "No 'id' in Collection for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if not self . _title :
    msg = "No 'title' in Collection for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . _can_read is None :
    msg = "No 'can_read' in Collection for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . _can_write is None :
    msg = "No 'can_write' in Collection for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . _id not in self . url :
    msg = "The collection '{}' does not match the url for queries '{}'"
    raise ValidationError ( msg . format ( self . _id , self . url ) )


signature:
def get_objects ( self , accept = MEDIA_TYPE_STIX_V20 , ** filter_kwargs ) :
    """Implement the ` ` Get Objects ` ` endpoint ( section 5.3)"""

implementation:
self . _verify_can_read ( )
query_params = _filter_kwargs_to_query_params ( filter_kwargs )
return self . _conn . get ( self . objects_url , headers = { "Accept" : accept } , params = query_params )


signature:
def get_object ( self , obj_id , version = None , accept = MEDIA_TYPE_STIX_V20 ) :
    """Implement the ` ` Get an Object ` ` endpoint ( section 5.5)"""

implementation:
self . _verify_can_read ( )
url = self . objects_url + str ( obj_id ) + "/"
query_params = None
if version :
    query_params = _filter_kwargs_to_query_params ( { "version" : version } )
return self . _conn . get ( url , headers = { "Accept" : accept } , params = query_params )


signature:
def add_objects ( self , bundle , wait_for_completion = True , poll_interval = 1 , timeout = 60 , accept = MEDIA_TYPE_TAXII_V20 , content_type = MEDIA_TYPE_STIX_V20 ) :
    """Implement the ` ` Add Objects ` ` endpoint ( section 5.4)
 Add objects to the collection . This may be performed either
 synchronously or asynchronously . To add asynchronously , set
 wait _ for _ completion to False . If False , the latter two args are
 unused . If the caller wishes to monitor the status of the addition ,
 it may do so in its own way . To add synchronously , set
 wait _ for _ completion to True , and optionally set the poll and timeout
 intervals . After initiating the addition , the caller will block ,
 and the TAXII " status " service will be polled until the timeout
 expires , or the operation completes .
 Args :
 bundle : A STIX bundle with the objects to add ( string , dict , binary )
 wait _ for _ completion ( bool ) : Whether to wait for the add operation
 to complete before returning
 poll _ interval ( int ) : If waiting for completion , how often to poll
 the status service ( seconds )
 timeout ( int ) : If waiting for completion , how long to poll until
 giving up ( seconds ) . Use < = 0 to wait forever
 accept ( str ) : media type to include in the ` ` Accept : ` ` header .
 content _ type ( str ) : media type to include in the ` ` Content - Type : ` `
 header .
 Returns :
 If ` ` wait _ for _ completion ` ` is False , a Status object corresponding
 to the initial status data returned from the service , is returned .
 The status may not yet be complete at this point .
 If ` ` wait _ for _ completion ` ` is True , a Status object corresponding
 to the completed operation is returned if it didn ' t time out ;
 otherwise a Status object corresponding to the most recent data
 obtained before the timeout , is returned ."""

implementation:
self . _verify_can_write ( )
headers = { "Accept" : accept , "Content-Type" : content_type , }
if isinstance ( bundle , dict ) :
    json_text = json . dumps ( bundle , ensure_ascii = False )
    data = json_text . encode ( "utf-8" )
elif isinstance ( bundle , six . text_type ) :
    data = bundle . encode ( "utf-8" )
elif isinstance ( bundle , six . binary_type ) :
    data = bundle
else :
    raise TypeError ( "Don't know how to handle type '{}'" . format ( type ( bundle ) . __name__ ) )
status_json = self . _conn . post ( self . objects_url , headers = headers , data = data )
status_url = urlparse . urljoin ( self . url , "../../status/{}" . format ( status_json [ "id" ] ) )
status = Status ( url = status_url , conn = self . _conn , status_info = status_json )
if not wait_for_completion or status . status == "complete" :
    return status
status . wait_until_final ( poll_interval , timeout )
return status


signature:
def get_manifest ( self , accept = MEDIA_TYPE_TAXII_V20 , ** filter_kwargs ) :
    """Implement the ` ` Get Object Manifests ` ` endpoint ( section 5.6 ) ."""

implementation:
self . _verify_can_read ( )
query_params = _filter_kwargs_to_query_params ( filter_kwargs )
return self . _conn . get ( self . url + "manifest/" , headers = { "Accept" : accept } , params = query_params )


signature:
def _validate_api_root ( self ) :
    """Validates API Root information . Raises errors for required
 properties ."""

implementation:
if not self . _title :
    msg = "No 'title' in API Root for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if not self . _versions :
    msg = "No 'versions' in API Root for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )
if self . _max_content_length is None :
    msg = "No 'max_content_length' in API Root for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )


signature:
def refresh ( self , accept = MEDIA_TYPE_TAXII_V20 ) :
    """Update the API Root ' s information and list of Collections"""

implementation:
self . refresh_information ( accept )
self . refresh_collections ( accept )


signature:
def refresh_information ( self , accept = MEDIA_TYPE_TAXII_V20 ) :
    """Update the properties of this API Root .
 This invokes the ` ` Get API Root Information ` ` endpoint ."""

implementation:
response = self . __raw = self . _conn . get ( self . url , headers = { "Accept" : accept } )
self . _populate_fields ( ** response )
self . _loaded_information = True


signature:
def refresh_collections ( self , accept = MEDIA_TYPE_TAXII_V20 ) :
    """Update the list of Collections contained by this API Root .
 This invokes the ` ` Get Collections ` ` endpoint ."""

implementation:
url = self . url + "collections/"
response = self . _conn . get ( url , headers = { "Accept" : accept } )
self . _collections = [ ]
for item in response . get ( "collections" , [ ] ) : # optional
    collection_url = url + item [ "id" ] + "/"
    collection = Collection ( collection_url , conn = self . _conn , collection_info = item )
    self . _collections . append ( collection )
self . _loaded_collections = True


signature:
def _validate_server ( self ) :
    """Validates server information . Raises errors for required properties ."""

implementation:
if not self . _title :
    msg = "No 'title' in Server Discovery for request '{}'"
    raise ValidationError ( msg . format ( self . url ) )


signature:
def refresh ( self ) :
    """Update the Server information and list of API Roots"""

implementation:
response = self . __raw = self . _conn . get ( self . url )
self . _populate_fields ( ** response )
self . _loaded = True


signature:
def valid_content_type ( self , content_type , accept ) :
    """Check that the server is returning a valid Content - Type
 Args :
 content _ type ( str ) : ` ` Content - Type : ` ` header value
 accept ( str ) : media type to include in the ` ` Accept : ` ` header ."""

implementation:
accept_tokens = accept . replace ( ' ' , '' ) . split ( ';' )
content_type_tokens = content_type . replace ( ' ' , '' ) . split ( ';' )
return ( all ( elem in content_type_tokens for elem in accept_tokens ) and ( content_type_tokens [ 0 ] == 'application/vnd.oasis.taxii+json' or content_type_tokens [ 0 ] == 'application/vnd.oasis.stix+json' ) )


signature:
def get ( self , url , headers = None , params = None ) :
    """Perform an HTTP GET , using the saved requests . Session and auth info .
 If " Accept " isn ' t one of the given headers , a default TAXII mime type is
 used . Regardless , the response type is checked against the accept
 header value , and an exception is raised if they don ' t match .
 Args :
 url ( str ) : URL to retrieve
 headers ( dict ) : Any other headers to be added to the request .
 params : dictionary or bytes to be sent in the query string for the
 request . ( optional )"""

implementation:
merged_headers = self . _merge_headers ( headers )
if "Accept" not in merged_headers :
    merged_headers [ "Accept" ] = MEDIA_TYPE_TAXII_V20
accept = merged_headers [ "Accept" ]
resp = self . session . get ( url , headers = merged_headers , params = params )
resp . raise_for_status ( )
content_type = resp . headers [ "Content-Type" ]
if not self . valid_content_type ( content_type = content_type , accept = accept ) :
    msg = "Unexpected Response. Got Content-Type: '{}' for Accept: '{}'"
    raise TAXIIServiceException ( msg . format ( content_type , accept ) )
return _to_json ( resp )


signature:
def post ( self , url , headers = None , params = None , ** kwargs ) :
    """Send a JSON POST request with the given request headers , additional
 URL query parameters , and the given JSON in the request body . The
 extra query parameters are merged with any which already exist in the
 URL . The ' json ' and ' data ' parameters may not both be given .
 Args :
 url ( str ) : URL to retrieve
 headers ( dict ) : Any other headers to be added to the request .
 params : dictionary or bytes to be sent in the query string for the
 request . ( optional )
 json : json to send in the body of the Request . This must be a
 JSON - serializable object . ( optional )
 data : raw request body data . May be a dictionary , list of tuples ,
 bytes , or file - like object to send in the body of the Request .
 ( optional )"""

implementation:
if len ( kwargs ) > 1 :
    raise InvalidArgumentsError ( "Too many extra args ({} > 1)" . format ( len ( kwargs ) ) )
if kwargs :
    kwarg = next ( iter ( kwargs ) )
    if kwarg not in ( "json" , "data" ) :
        raise InvalidArgumentsError ( "Invalid kwarg: " + kwarg )
resp = self . session . post ( url , headers = headers , params = params , ** kwargs )
resp . raise_for_status ( )
return _to_json ( resp )


signature:
def _merge_headers ( self , call_specific_headers ) :
    """Merge headers from different sources together . Headers passed to the
 post / get methods have highest priority , then headers associated with
 the connection object itself have next priority .
 : param call _ specific _ headers : A header dict from the get / post call , or
 None ( the default for those methods ) .
 : return : A key - case - insensitive MutableMapping object which contains
 the merged headers . ( This doesn ' t actually return a dict . )"""

implementation:
# A case - insensitive mapping is necessary here so that there is
# predictable behavior . If a plain dict were used , you ' d get keys in
# the merged dict which differ only in case . The requests library
# would merge them internally , and it would be unpredictable which key
# is chosen for the final set of headers . Another possible approach
# would be to upper / lower - case everything , but this seemed easier . On
# the other hand , I don ' t know if CaseInsensitiveDict is public API . . . ?
# First establish defaults
merged_headers = requests . structures . CaseInsensitiveDict ( { "User-Agent" : self . user_agent } )
# Then overlay with specifics from post / get methods
if call_specific_headers :
    merged_headers . update ( call_specific_headers )
# Special " User - Agent " header check , to ensure one is always sent .
# The call - specific overlay could have null ' d out that header .
if not merged_headers . get ( "User-Agent" ) :
    merged_headers [ "User-Agent" ] = self . user_agent
return merged_headers


signature:
def total_memory ( ) :
    """Returns the the amount of memory available for use .
 The memory is obtained from MemTotal entry in / proc / meminfo .
 Notes
 This function is not very useful and not very portable ."""

implementation:
with file ( '/proc/meminfo' , 'r' ) as f :
    for line in f :
        words = line . split ( )
    if words [ 0 ] . upper ( ) == 'MEMTOTAL:' :
        return int ( words [ 1 ] ) * 1024
raise IOError ( 'MemTotal unknown' )


signature:
def cpu_count ( ) :
    """Returns the default number of slave processes to be spawned .
 The default value is the number of physical cpu cores seen by python .
 : code : ` OMP _ NUM _ THREADS ` environment variable overrides it .
 On PBS / torque systems if OMP _ NUM _ THREADS is empty , we try to
 use the value of : code : ` PBS _ NUM _ PPN ` variable .
 Notes
 On some machines the physical number of cores does not equal
 the number of cpus shall be used . PSC Blacklight for example ."""

implementation:
num = os . getenv ( "OMP_NUM_THREADS" )
if num is None :
    num = os . getenv ( "PBS_NUM_PPN" )
try :
    return int ( num )
except :
    return multiprocessing . cpu_count ( )


signature:
def empty_like ( array , dtype = None ) :
    """Create a shared memory array from the shape of array ."""

implementation:
array = numpy . asarray ( array )
if dtype is None :
    dtype = array . dtype
return anonymousmemmap ( array . shape , dtype )


signature:
def full_like ( array , value , dtype = None ) :
    """Create a shared memory array with the same shape and type as a given array , filled with ` value ` ."""

implementation:
shared = empty_like ( array , dtype )
shared [ : ] = value
return shared


signature:
def full ( shape , value , dtype = 'f8' ) :
    """Create a shared memory array of given shape and type , filled with ` value ` ."""

implementation:
shared = empty ( shape , dtype )
shared [ : ] = value
return shared


signature:
def copy ( a ) :
    """Copy an array to the shared memory .
 Notes
 copy is not always necessary because the private memory is always copy - on - write .
 Use : code : ` a = copy ( a ) ` to immediately dereference the old ' a ' on private memory"""

implementation:
shared = anonymousmemmap ( a . shape , dtype = a . dtype )
shared [ : ] = a [ : ]
return shared


signature:
def get ( self , Q ) :
    """Protected get . Get an item from Q .
 Will block . but if the process group has errors ,
 raise an StopProcessGroup exception .
 A slave process will terminate upon StopProcessGroup .
 The master process shall read the error from the process group ."""

implementation:
while self . Errors . empty ( ) :
    try :
        return Q . get ( timeout = 1 )
    except queue . Empty : # check if the process group is dead
        if not self . is_alive ( ) : # todo : can be graceful , in which
        # case the last item shall have been
        # flushed to Q .
            try :
                return Q . get ( timeout = 0 )
            except queue . Empty :
                raise StopProcessGroup
        else :
            continue
else :
    raise StopProcessGroup


signature:
def wait ( self ) :
    """Wait and join the child process .
 The return value of the function call is returned .
 If any exception occurred it is wrapped and raised ."""

implementation:
e , r = self . result . get ( )
self . slave . join ( )
self . slave = None
self . result = None
if isinstance ( e , Exception ) :
    raise SlaveException ( e , r )
return r


signature:
def map ( self , func , sequence , reduce = None , star = False , minlength = 0 ) :
    """Map - reduce with multile processes .
 Apply func to each item on the sequence , in parallel .
 As the results are collected , reduce is called on the result .
 The reduced result is returned as a list .
 Parameters
 func : callable
 The function to call . It must accept the same number of
 arguments as the length of an item in the sequence .
 . . warning : :
 func is not supposed to use exceptions for flow control .
 In non - debug mode all exceptions will be wrapped into
 a : py : class : ` SlaveException ` .
 sequence : list or array _ like
 The sequence of arguments to be applied to func .
 reduce : callable , optional
 Apply an reduction operation on the
 return values of func . If func returns a tuple , they
 are treated as positional arguments of reduce .
 star : boolean
 if True , the items in sequence are treated as positional
 arguments of reduce .
 minlength : integer
 Minimal length of ` sequence ` to start parallel processing .
 if len ( sequence ) < minlength , fall back to sequential
 processing . This can be used to avoid the overhead of starting
 the worker processes when there is little work .
 Returns
 results : list
 The list of reduced results from the map operation , in
 the order of the arguments of sequence .
 Raises
 SlaveException
 If any of the slave process encounters
 an exception . Inspect : py : attr : ` SlaveException . reason ` for the underlying exception ."""

implementation:
def realreduce ( r ) :
    if reduce :
        if isinstance ( r , tuple ) :
            return reduce ( * r )
        else :
            return reduce ( r )
    return r
def realfunc ( i ) :
    if star :
        return func ( * i )
    else :
        return func ( i )
if len ( sequence ) <= 0 or self . np == 0 or get_debug ( ) : # Do this in serial
    self . local = lambda : None
    self . local . rank = 0
    rt = [ realreduce ( realfunc ( i ) ) for i in sequence ]
    self . local = None
    return rt
# never use more than len ( sequence ) processes
np = min ( [ self . np , len ( sequence ) ] )
Q = self . backend . QueueFactory ( 64 )
R = self . backend . QueueFactory ( 64 )
self . ordered . reset ( )
pg = ProcessGroup ( main = self . _main , np = np , backend = self . backend , args = ( Q , R , sequence , realfunc ) )
pg . start ( )
L = [ ]
N = [ ]
def feeder ( pg , Q , N ) : # will fail silently if any error occurs .
    j = 0
    try :
        for i , work in enumerate ( sequence ) :
            if not hasattr ( sequence , '__getitem__' ) :
                pg . put ( Q , ( i , work ) )
            else :
                pg . put ( Q , ( i , ) )
            j = j + 1
        N . append ( j )
        for i in range ( np ) :
            pg . put ( Q , None )
    except StopProcessGroup :
        return
    finally :
        pass
feeder = threading . Thread ( None , feeder , args = ( pg , Q , N ) )
feeder . start ( )
# we run fetcher on main thread to catch exceptions
# raised by reduce
count = 0
try :
    while True :
        try :
            capsule = pg . get ( R )
        except queue . Empty :
            continue
        except StopProcessGroup :
            raise pg . get_exception ( )
        capsule = capsule [ 0 ] , realreduce ( capsule [ 1 ] )
        heapq . heappush ( L , capsule )
        count = count + 1
        if len ( N ) > 0 and count == N [ 0 ] : # if finished feeding see if all
        # results have been obtained
            break
    rt = [ ]
    # R . close ( )
    # R . join _ thread ( )
    while len ( L ) > 0 :
        rt . append ( heapq . heappop ( L ) [ 1 ] )
    pg . join ( )
    feeder . join ( )
    assert N [ 0 ] == len ( rt )
    return rt
except BaseException as e :
    pg . killall ( )
    pg . join ( )
    feeder . join ( )
    raise


signature:
def savetxt2 ( fname , X , delimiter = ' ' , newline = '\n' , comment_character = '#' , header = '' , save_dtype = False , fmt = { } ) :
    """format of table header :
 # ID [ type ] : name ( index ) . . . . * number of items
 user ' s header is not prefixed by comment _ character
 name of nested dtype elements are split by ."""

implementation:
prefixfmt = { }
for key in fmt :
    prefixfmt [ key ] = fmt [ key ]
olddtype = X . dtype
newdtype = flatten_dtype ( numpy . dtype ( [ ( '' , ( X . dtype , X . shape [ 1 : ] ) ) ] ) )
X = X . view ( dtype = newdtype )
dtype = X . dtype
X = numpy . atleast_1d ( X . squeeze ( ) )
header2 = _mkheader ( dtype )
fmtstr = _mkfmtstr ( dtype , prefixfmt , delimiter , _default_fmt )
if hasattr ( fname , 'write' ) :
    fh = fname
    cleanup = lambda : None
else :
    fh = file ( fname , 'w+' )
    cleanup = lambda : fh . close ( )
try :
    fh . write ( header )
    if header [ : - 1 ] != newline :
        fh . write ( newline )
    fh . write ( comment_character )
    fh . write ( '!' )
    fh . write ( header2 )
    fh . write ( delimiter )
    fh . write ( '*%d' % len ( X ) )
    fh . write ( newline )
    if save_dtype :
        fh . write ( comment_character )
        fh . write ( '?' )
        fh . write ( base64 . b64encode ( pickle . dumps ( olddtype ) ) )
        fh . write ( newline )
    for row in X :
        fh . write ( fmtstr % tuple ( row ) )
        fh . write ( newline )
    if hasattr ( fh , 'flush' ) :
        fh . flush ( )
finally :
    cleanup ( )


signature:
def loadtxt2 ( fname , dtype = None , delimiter = ' ' , newline = '\n' , comment_character = '#' , skiplines = 0 ) :
    """Known issues delimiter and newline is not respected .
 string quotation with space is broken ."""

implementation:
dtypert = [ None , None , None ]
def preparedtype ( dtype ) :
    dtypert [ 0 ] = dtype
    flatten = flatten_dtype ( dtype )
    dtypert [ 1 ] = flatten
    dtypert [ 2 ] = numpy . dtype ( [ ( 'a' , ( numpy . int8 , flatten . itemsize ) ) ] )
    buf = numpy . empty ( ( ) , dtype = dtypert [ 1 ] )
    converters = [ _default_conv [ flatten [ name ] . char ] for name in flatten . names ]
    return buf , converters , flatten . names
def fileiter ( fh ) :
    converters = [ ]
    buf = None
    if dtype is not None :
        buf , converters , names = preparedtype ( dtype )
        yield None
    for lineno , line in enumerate ( fh ) :
        if lineno < skiplines :
            continue
        if line [ 0 ] in comment_character :
            if buf is None and line [ 1 ] == '?' :
                ddtype = pickle . loads ( base64 . b64decode ( line [ 2 : ] ) )
                buf , converters , names = preparedtype ( ddtype )
                yield None
            continue
        for word , c , name in zip ( line . split ( ) , converters , names ) :
            buf [ name ] = c ( word )
        buf2 = buf . copy ( ) . view ( dtype = dtypert [ 2 ] )
        yield buf2
if isinstance ( fname , basestring ) :
    fh = file ( fh , 'r' )
    cleanup = lambda : fh . close ( )
else :
    fh = iter ( fname )
    cleanup = lambda : None
try :
    i = fileiter ( fh )
    i . next ( )
    return numpy . fromiter ( i , dtype = dtypert [ 2 ] ) . view ( dtype = dtypert [ 0 ] )
finally :
    cleanup ( )


signature:
def flatten_dtype ( dtype , _next = None ) :
    """Unpack a structured data - type ."""

implementation:
types = [ ]
if _next is None :
    _next = [ 0 , '' ]
    primary = True
else :
    primary = False
prefix = _next [ 1 ]
if dtype . names is None :
    for i in numpy . ndindex ( dtype . shape ) :
        if dtype . base == dtype :
            types . append ( ( '%s%s' % ( prefix , simplerepr ( i ) ) , dtype ) )
            _next [ 0 ] += 1
        else :
            _next [ 1 ] = '%s%s' % ( prefix , simplerepr ( i ) )
            types . extend ( flatten_dtype ( dtype . base , _next ) )
else :
    for field in dtype . names :
        typ_fields = dtype . fields [ field ]
        if len ( prefix ) > 0 :
            _next [ 1 ] = prefix + '.' + field
        else :
            _next [ 1 ] = '' + field
        flat_dt = flatten_dtype ( typ_fields [ 0 ] , _next )
        types . extend ( flat_dt )
_next [ 1 ] = prefix
if primary :
    return numpy . dtype ( types )
else :
    return types


signature:
def MetaOrdered ( parallel , done , turnstile ) :
    """meta class for Ordered construct ."""

implementation:
class Ordered :
    def __init__ ( self , iterref ) :
        if parallel . master :
            done [ ... ] = 0
        self . iterref = iterref
        parallel . barrier ( )
    @ classmethod
    def abort ( self ) :
        turnstile . release ( )
    def __enter__ ( self ) :
        while self . iterref != done :
            pass
        turnstile . acquire ( )
        return self
    def __exit__ ( self , * args ) :
        done [ ... ] += 1
        turnstile . release ( )
return Ordered


signature:
def kill_all ( self ) :
    """kill all slaves and reap the monitor"""

implementation:
for pid in self . children :
    try :
        os . kill ( pid , signal . SIGTRAP )
    except OSError :
        continue
self . join ( )


signature:
def join ( self ) :
    """master only"""

implementation:
try :
    self . pipe . put ( 'Q' )
    self . thread . join ( )
except :
    pass
finally :
    self . thread = None


signature:
def slaveraise ( self , type , error , traceback ) :
    """slave only"""

implementation:
message = 'E' * 1 + pickle . dumps ( ( type , '' . join ( tb . format_exception ( type , error , traceback ) ) ) )
if self . pipe is not None :
    self . pipe . put ( message )


signature:
def forloop ( self , range , ordered = False , schedule = ( 'static' , 1 ) ) :
    """schedule can be
 ( sch , chunk ) or sch ;
 sch is ' static ' , ' dynamic ' or ' guided ' .
 chunk defaults to 1
 if ordered , create an ordred"""

implementation:
if isinstance ( schedule , tuple ) :
    schedule , chunk = schedule
else :
    chunk = None
if schedule == 'static' :
    return self . _StaticForLoop ( range , ordered , chunk )
elif schedule == 'dynamic' :
    return self . _DynamicForLoop ( range , ordered , chunk , guided = False )
elif schedule == 'guided' :
    return self . _DynamicForLoop ( range , ordered , chunk , guided = True )
else :
    raise "schedule unknown"


signature:
def abort ( self ) :
    """ensure the master exit from Barrier"""

implementation:
self . mutex . release ( )
self . turnstile . release ( )
self . mutex . release ( )
self . turnstile2 . release ( )


signature:
def read ( self , n ) :
    """return at most n array items , move the cursor ."""

implementation:
while len ( self . pool ) < n :
    self . cur = self . files . next ( )
    self . pool = numpy . append ( self . pool , self . fetch ( self . cur ) , axis = 0 )
rt = self . pool [ : n ]
if n == len ( self . pool ) :
    self . pool = self . fetch ( None )
else :
    self . pool = self . pool [ n : ]
return rt


signature:
def call ( self , args , axis = 0 , out = None , chunksize = 1024 * 1024 , ** kwargs ) :
    """axis is the axis to chop it off .
 if self . altreduce is set , the results will
 be reduced with altreduce and returned
 otherwise will be saved to out , then return out ."""

implementation:
if self . altreduce is not None :
    ret = [ None ]
else :
    if out is None :
        if self . outdtype is not None :
            dtype = self . outdtype
        else :
            try :
                dtype = numpy . result_type ( * [ args [ i ] for i in self . ins ] * 2 )
            except :
                dtype = None
        out = sharedmem . empty ( numpy . broadcast ( * [ args [ i ] for i in self . ins ] * 2 ) . shape , dtype = dtype )
if axis != 0 :
    for i in self . ins :
        args [ i ] = numpy . rollaxis ( args [ i ] , axis )
    out = numpy . rollaxis ( out , axis )
size = numpy . max ( [ len ( args [ i ] ) for i in self . ins ] )
with sharedmem . MapReduce ( ) as pool :
    def work ( i ) :
        sl = slice ( i , i + chunksize )
        myargs = args [ : ]
        for j in self . ins :
            try :
                tmp = myargs [ j ] [ sl ]
                a , b , c = sl . indices ( len ( args [ j ] ) )
                myargs [ j ] = tmp
            except Exception as e :
                print tmp
                print j , e
                pass
        if b == a :
            return None
        rt = self . ufunc ( * myargs , ** kwargs )
        if self . altreduce is not None :
            return rt
        else :
            out [ sl ] = rt
    def reduce ( rt ) :
        if self . altreduce is None :
            return
        if ret [ 0 ] is None :
            ret [ 0 ] = rt
        elif rt is not None :
            ret [ 0 ] = self . altreduce ( ret [ 0 ] , rt )
    pool . map ( work , range ( 0 , size , chunksize ) , reduce = reduce )
if self . altreduce is None :
    if axis != 0 :
        out = numpy . rollaxis ( out , 0 , axis + 1 )
    return out
else :
    return ret [ 0 ]


signature:
def adapt ( cls , source , template ) :
    """adapt source to a packarray according to the layout of template"""

implementation:
if not isinstance ( template , packarray ) :
    raise TypeError ( 'template must be a packarray' )
return cls ( source , template . start , template . end )


signature:
def argsort ( data , out = None , chunksize = None , baseargsort = None , argmerge = None , np = None ) :
    """parallel argsort , like numpy . argsort
 use sizeof ( intp ) * len ( data ) as scratch space
 use baseargsort for serial sort
 ind = baseargsort ( data )
 use argmerge to merge
 def argmerge ( data , A , B , out ) :
 ensure data [ out ] is sorted
 and out [ : ] = A join B
 TODO : shall try to use the inplace merge mentioned in
 http : / / keithschwarz . com / interesting / code / ? dir = inplace - merge ."""

implementation:
if baseargsort is None :
    baseargsort = lambda x : x . argsort ( )
if argmerge is None :
    argmerge = default_argmerge
if chunksize is None :
    chunksize = 1024 * 1024 * 16
if out is None :
    arg1 = numpy . empty ( len ( data ) , dtype = 'intp' )
    out = arg1
else :
    assert out . dtype == numpy . dtype ( 'intp' )
    assert len ( out ) == len ( data )
    arg1 = out
if np is None :
    np = sharedmem . cpu_count ( )
if np <= 1 or len ( data ) < chunksize :
    out [ : ] = baseargsort ( data )
    return out
CHK = [ slice ( i , i + chunksize ) for i in range ( 0 , len ( data ) , chunksize ) ]
DUMMY = slice ( len ( data ) , len ( data ) )
if len ( CHK ) % 2 :
    CHK . append ( DUMMY )
with sharedmem . TPool ( ) as pool :
    def work ( i ) :
        C = CHK [ i ]
        start , stop , step = C . indices ( len ( data ) )
        arg1 [ C ] = baseargsort ( data [ C ] )
        arg1 [ C ] += start
    pool . map ( work , range ( len ( CHK ) ) )
arg2 = numpy . empty_like ( arg1 )
flip = 0
while len ( CHK ) > 1 :
    with sharedmem . TPool ( ) as pool :
        def work ( i ) :
            C1 = CHK [ i ]
            C2 = CHK [ i + 1 ]
            start1 , stop1 , step1 = C1 . indices ( len ( data ) )
            start2 , stop2 , step2 = C2 . indices ( len ( data ) )
            # print ' argmerge ' , start1 , stop1 , start2 , stop2
            assert start2 == stop1
            argmerge ( data , arg1 [ C1 ] , arg1 [ C2 ] , arg2 [ start1 : stop2 ] )
            return slice ( start1 , stop2 )
        CHK = pool . map ( work , range ( 0 , len ( CHK ) , 2 ) )
        arg1 , arg2 = arg2 , arg1
        flip = flip + 1
    if len ( CHK ) == 1 :
        break
    if len ( CHK ) % 2 :
        CHK . append ( DUMMY )
if flip % 2 != 0 : # only even flips out ends up pointing to arg2 and needs to be
# copied
    out [ : ] = arg1
return out


signature:
def day_of_week ( abbr = False ) :
    """Return a random ( abbreviated if ` abbr ` ) day of week name ."""

implementation:
if abbr :
    return random . choice ( DAYS_ABBR )
else :
    return random . choice ( DAYS )


signature:
def month ( abbr = False , numerical = False ) :
    """Return a random ( abbreviated if ` abbr ` ) month name or month number if
 ` numerical ` ."""

implementation:
if numerical :
    return random . randint ( 1 , 12 )
else :
    if abbr :
        return random . choice ( MONTHS_ABBR )
    else :
        return random . choice ( MONTHS )


signature:
def year ( past = False , min_delta = 0 , max_delta = 20 ) :
    """Return a random year ."""

implementation: return dt . date . today ( ) . year + _delta ( past , min_delta , max_delta )


signature:
def date ( past = False , min_delta = 0 , max_delta = 20 ) :
    """Return a random ` dt . date ` object . Delta args are days ."""

implementation:
timedelta = dt . timedelta ( days = _delta ( past , min_delta , max_delta ) )
return dt . date . today ( ) + timedelta


signature:
def get_dictionary ( dict_name ) :
    """Load a dictionary file ` ` dict _ name ` ` ( if it ' s not cached ) and return its
 contents as an array of strings ."""

implementation:
global dictionaries_cache
if dict_name not in dictionaries_cache :
    try :
        dictionary_file = codecs . open ( join ( DICTIONARIES_PATH , dict_name ) , 'r' , 'utf-8' )
    except IOError :
        None
    else :
        dictionaries_cache [ dict_name ] = dictionary_file . readlines ( )
        dictionary_file . close ( )
return dictionaries_cache [ dict_name ]


signature:
def check_digit ( num ) :
    """Return a check digit of the given credit card number .
 Check digit calculated using Luhn algorithm ( " modulus 10 " )
 See : http : / / www . darkcoding . net / credit - card / luhn - formula /"""

implementation:
sum = 0
# drop last digit , then reverse the number
digits = str ( num ) [ : - 1 ] [ : : - 1 ]
for i , n in enumerate ( digits ) : # select all digits at odd positions starting from 1
    if ( i + 1 ) % 2 != 0 :
        digit = int ( n ) * 2
        if digit > 9 :
            sum += ( digit - 9 )
        else :
            sum += digit
    else :
        sum += int ( n )
return ( ( divmod ( sum , 10 ) [ 0 ] + 1 ) * 10 - sum ) % 10


signature:
def number ( type = None , length = None , prefixes = None ) :
    """Return a random credit card number .
 : param type : credit card type . Defaults to a random selection .
 : param length : length of the credit card number .
 Defaults to the length for the selected card type .
 : param prefixes : allowed prefixes for the card number .
 Defaults to prefixes for the selected card type .
 : return : credit card randomly generated number ( int )"""

implementation:
# select credit card type
if type and type in CARDS :
    card = type
else :
    card = random . choice ( list ( CARDS . keys ( ) ) )
# select a credit card number ' s prefix
if not prefixes :
    prefixes = CARDS [ card ] [ 'prefixes' ]
prefix = random . choice ( prefixes )
# select length of the credit card number , if it ' s not set
if not length :
    length = CARDS [ card ] [ 'length' ]
# generate all digits but the last one
result = str ( prefix )
for d in range ( length - len ( str ( prefix ) ) ) :
    result += str ( basic . number ( ) )
last_digit = check_digit ( int ( result ) )
return int ( result [ : - 1 ] + str ( last_digit ) )


signature:
def street_number ( ) :
    """Return a random street number ."""

implementation:
length = int ( random . choice ( string . digits [ 1 : 6 ] ) )
return '' . join ( random . sample ( string . digits , length ) )


signature:
def zip_code ( ) :
    """Return a random ZIP code , either in ` # # # # # ` or ` # # # # # - # # # # ` format ."""

implementation:
format = '#####'
if random . random ( ) >= 0.5 :
    format = '#####-####'
result = ''
for item in format :
    if item == '#' :
        result += str ( random . randint ( 0 , 9 ) )
    else :
        result += item
return result


signature:
def phone ( ) :
    """Return a random phone number in ` # - ( # # # ) # # # - # # # # ` format ."""

implementation:
format = '#-(###)###-####'
result = ''
for item in format :
    if item == '#' :
        result += str ( random . randint ( 0 , 9 ) )
    else :
        result += item
return result


signature:
def job_title ( ) :
    """Return a random job title ."""

implementation:
result = random . choice ( get_dictionary ( 'job_titles' ) ) . strip ( )
result = result . replace ( '#{N}' , job_title_suffix ( ) )
return result


signature:
def body ( quantity = 2 , separator = '\n\n' , wrap_start = '' , wrap_end = '' , html = False , sentences_quantity = 3 , as_list = False ) :
    """Return a random email text ."""

implementation: return lorem_ipsum . paragraphs ( quantity = quantity , separator = separator , wrap_start = wrap_start , wrap_end = wrap_end , html = html , sentences_quantity = sentences_quantity , as_list = as_list )


signature:
def money ( min = 0 , max = 10 ) :
    """Return a str of decimal with two digits after a decimal mark ."""

implementation:
value = random . choice ( range ( min * 100 , max * 100 ) )
return "%1.2f" % ( float ( value ) / 100 )


signature:
def words ( quantity = 10 , as_list = False ) :
    """Return random words ."""

implementation:
global _words
if not _words :
    _words = ' ' . join ( get_dictionary ( 'lorem_ipsum' ) ) . lower ( ) . replace ( '\n' , '' )
    _words = re . sub ( r'\.|,|;/' , '' , _words )
    _words = _words . split ( ' ' )
result = random . sample ( _words , quantity )
if as_list :
    return result
else :
    return ' ' . join ( result )


signature:
def title ( words_quantity = 4 ) :
    """Return a random sentence to be used as e . g . an e - mail subject ."""

implementation:
result = words ( quantity = words_quantity )
result += random . choice ( '?.!' )
return result . capitalize ( )


signature:
def sentences ( quantity = 2 , as_list = False ) :
    """Return random sentences ."""

implementation:
result = [ sntc . strip ( ) for sntc in random . sample ( get_dictionary ( 'lorem_ipsum' ) , quantity ) ]
if as_list :
    return result
else :
    return ' ' . join ( result )


signature:
def paragraph ( separator = '\n\n' , wrap_start = '' , wrap_end = '' , html = False , sentences_quantity = 3 ) :
    """Return a random paragraph ."""

implementation: return paragraphs ( quantity = 1 , separator = separator , wrap_start = wrap_start , wrap_end = wrap_end , html = html , sentences_quantity = sentences_quantity )


signature:
def paragraphs ( quantity = 2 , separator = '\n\n' , wrap_start = '' , wrap_end = '' , html = False , sentences_quantity = 3 , as_list = False ) :
    """Return random paragraphs ."""

implementation:
if html :
    wrap_start = '<p>'
    wrap_end = '</p>'
    separator = '\n\n'
result = [ ]
try :
    for _ in xrange ( 0 , quantity ) :
        result . append ( wrap_start + sentences ( sentences_quantity ) + wrap_end )
# Python 3 compatibility
except NameError :
    for _ in range ( 0 , quantity ) :
        result . append ( wrap_start + sentences ( sentences_quantity ) + wrap_end )
if as_list :
    return result
else :
    return separator . join ( result )


signature:
def _to_lower_alpha_only ( s ) :
    """Return a lowercased string with non alphabetic chars removed .
 White spaces are not to be removed ."""

implementation:
s = re . sub ( r'\n' , ' ' , s . lower ( ) )
return re . sub ( r'[^a-z\s]' , '' , s )


signature:
def characters ( quantity = 10 ) :
    """Return random characters ."""

implementation:
line = map ( _to_lower_alpha_only , '' . join ( random . sample ( get_dictionary ( 'lorem_ipsum' ) , quantity ) ) )
return '' . join ( line ) [ : quantity ]


signature:
def text ( what = "sentence" , * args , ** kwargs ) :
    """An aggregator for all above defined public methods ."""

implementation:
if what == "character" :
    return character ( * args , ** kwargs )
elif what == "characters" :
    return characters ( * args , ** kwargs )
elif what == "word" :
    return word ( * args , ** kwargs )
elif what == "words" :
    return words ( * args , ** kwargs )
elif what == "sentence" :
    return sentence ( * args , ** kwargs )
elif what == "sentences" :
    return sentences ( * args , ** kwargs )
elif what == "paragraph" :
    return paragraph ( * args , ** kwargs )
elif what == "paragraphs" :
    return paragraphs ( * args , ** kwargs )
elif what == "title" :
    return title ( * args , ** kwargs )
else :
    raise NameError ( 'No such method' )


signature:
def user_name ( with_num = False ) :
    """Return a random user name .
 Basically it ' s lowercased result of
 : py : func : ` ~ forgery _ py . forgery . name . first _ name ( ) ` with a number appended
 if ` with _ num ` ."""

implementation:
result = first_name ( )
if with_num :
    result += str ( random . randint ( 63 , 94 ) )
return result . lower ( )


signature:
def domain_name ( ) :
    """Return a random domain name .
 Lowercased result of : py : func : ` ~ forgery _ py . forgery . name . company _ name ( ) `
 plus : py : func : ` ~ top _ level _ domain ( ) ` ."""

implementation:
result = random . choice ( get_dictionary ( 'company_names' ) ) . strip ( )
result += '.' + top_level_domain ( )
return result . lower ( )


signature:
def email_address ( user = None ) :
    """Return random e - mail address in a hopefully imaginary domain .
 If ` user ` is ` ` None ` ` : py : func : ` ~ user _ name ( ) ` will be used . Otherwise it
 will be lowercased and will have spaces replaced with ` ` _ ` ` .
 Domain name is created using : py : func : ` ~ domain _ name ( ) ` ."""

implementation:
if not user :
    user = user_name ( )
else :
    user = user . strip ( ) . replace ( ' ' , '_' ) . lower ( )
return user + '@' + domain_name ( )


signature:
def account_number ( ) :
    """Return a random bank account number ."""

implementation:
account = [ random . randint ( 1 , 9 ) for _ in range ( 20 ) ]
return "" . join ( map ( str , account ) )


signature:
def bik ( ) :
    """Return a random bank identification number ."""

implementation: return '04' + '' . join ( [ str ( random . randint ( 1 , 9 ) ) for _ in range ( 5 ) ] ) + str ( random . randint ( 0 , 49 ) + 50 )


signature:
def legal_inn ( ) :
    """Return a random taxation ID number for a company ."""

implementation:
mask = [ 2 , 4 , 10 , 3 , 5 , 9 , 4 , 6 , 8 ]
inn = [ random . randint ( 1 , 9 ) for _ in range ( 10 ) ]
weighted = [ v * mask [ i ] for i , v in enumerate ( inn [ : - 1 ] ) ]
inn [ 9 ] = sum ( weighted ) % 11 % 10
return "" . join ( map ( str , inn ) )


signature:
def legal_ogrn ( ) :
    """Return a random government registration ID for a company ."""

implementation:
ogrn = "" . join ( map ( str , [ random . randint ( 1 , 9 ) for _ in range ( 12 ) ] ) )
ogrn += str ( ( int ( ogrn ) % 11 % 10 ) )
return ogrn


signature:
def person_inn ( ) :
    """Return a random taxation ID number for a natural person ."""

implementation:
mask11 = [ 7 , 2 , 4 , 10 , 3 , 5 , 9 , 4 , 6 , 8 ]
mask12 = [ 3 , 7 , 2 , 4 , 10 , 3 , 5 , 9 , 4 , 6 , 8 ]
inn = [ random . randint ( 1 , 9 ) for _ in range ( 12 ) ]
# get the 11th digit of the INN
weighted11 = [ v * mask11 [ i ] for i , v in enumerate ( inn [ : - 2 ] ) ]
inn [ 10 ] = sum ( weighted11 ) % 11 % 10
# get the 12th digit of the INN
weighted12 = [ v * mask12 [ i ] for i , v in enumerate ( inn [ : - 1 ] ) ]
inn [ 11 ] = sum ( weighted12 ) % 11 % 10
return "" . join ( map ( str , inn ) )


signature:
def encrypt ( password = 'password' , salt = None ) :
    """Return SHA1 hexdigest of a password ( optionally salted with a string ) ."""

implementation:
if not salt :
    salt = str ( datetime . utcnow ( ) )
try : # available for python 2.7.8 and python 3.4 +
    dk = hashlib . pbkdf2_hmac ( 'sha1' , password . encode ( ) , salt . encode ( ) , 100000 )
    hexdigest = binascii . hexlify ( dk ) . decode ( 'utf-8' )
except AttributeError : # see https : / / pymotw . com / 2 / hashlib /
# see https : / / docs . python . org / release / 2.5 / lib / module - hashlib . html
    dk = hashlib . sha1 ( )
    dk . update ( password . encode ( ) + salt . encode ( ) )
    hexdigest = dk . hexdigest ( )
return hexdigest


signature:
def password ( at_least = 6 , at_most = 12 , lowercase = True , uppercase = True , digits = True , spaces = False , punctuation = False ) :
    """Return a random string for use as a password ."""

implementation: return text ( at_least = at_least , at_most = at_most , lowercase = lowercase , uppercase = uppercase , digits = digits , spaces = spaces , punctuation = punctuation )


signature:
def forwards ( self , orm ) :
    "Write your forwards methods here ."

implementation:
# Note : Don ' t use " from appname . models import ModelName " .
# Use orm . ModelName to refer to models in this application ,
# and orm [ ' appname . ModelName ' ] for models in other applications .
print ( "Updating: JednostkaAdministracyjna" )
ja_akt_stan = orm . JednostkaAdministracyjna . objects . all ( ) . aggregate ( Max ( 'stan_na' ) ) [ 'stan_na__max' ]
orm . JednostkaAdministracyjna . objects . filter ( stan_na__exact = ja_akt_stan ) . update ( aktywny = True )
orm . JednostkaAdministracyjna . objects . exclude ( stan_na__exact = ja_akt_stan ) . update ( aktywny = False )
print ( "Updating: Miejscowosc" )
m_akt_stan = orm . Miejscowosc . objects . all ( ) . aggregate ( Max ( 'stan_na' ) ) [ 'stan_na__max' ]
orm . Miejscowosc . objects . filter ( stan_na__exact = m_akt_stan ) . update ( aktywny = True )
orm . Miejscowosc . objects . exclude ( stan_na__exact = m_akt_stan ) . update ( aktywny = False )
print ( "Updating: RodzajMiejsowosci" )
rm_akt_stan = orm . RodzajMiejsowosci . objects . all ( ) . aggregate ( Max ( 'stan_na' ) ) [ 'stan_na__max' ]
orm . RodzajMiejsowosci . objects . filter ( stan_na__exact = rm_akt_stan ) . update ( aktywny = True )
orm . RodzajMiejsowosci . objects . exclude ( stan_na__exact = rm_akt_stan ) . update ( aktywny = False )
print ( "Updating: Ulica" )
u_akt_stan = orm . Ulica . objects . all ( ) . aggregate ( Max ( 'stan_na' ) ) [ 'stan_na__max' ]
orm . Ulica . objects . filter ( stan_na__exact = u_akt_stan ) . update ( aktywny = True )
orm . Ulica . objects . exclude ( stan_na__exact = u_akt_stan ) . update ( aktywny = False )


implementation:
# Note : Don ' t use " from appname . models import ModelName " .
# Use orm . ModelName to refer to models in this application ,
# and orm [ ' appname . ModelName ' ] for models in other applications .
LEN_TYPE = { 7 : 'GMI' , 4 : 'POW' , 2 : 'WOJ' , }
for ja in orm . JednostkaAdministracyjna . objects . all ( ) :
    ja . typ = LEN_TYPE [ len ( ja . id ) ]
    ja . save ( )


signature:
def case ( * , to , ** kwargs ) :
    """Converts an identifier from one case type to another .
 An identifier is an ASCII string consisting of letters , digits and underscores , not starting with a digit .
 The supported case types are camelCase , PascalCase , snake _ case , and CONSTANT _ CASE ,
 identified as camel , pascal , snake , and constant .
 The input identifier is given as a keyword argument with one of these names ,
 and the output type is given as a string in the ` to ` keyword argument .
 If a given string does not conform to the specified case type ( such as underscores in camel or pascal case strings ,
 or double _ _ underscores in general ) , the result may not be as desired ,
 although things like snaKe _ casE or CONStaNT _ CASe will generally work ."""

implementation:
if len ( kwargs ) != 1 :
    raise ValueError ( "expect exactly one source string argument" )
[ ( typ , string ) ] = kwargs . items ( )
types = { 'pascal' , 'camel' , 'snake' , 'constant' }
if typ not in types :
    raise ValueError ( f"source string keyword must be one of {types}" )
if to not in types :
    raise ValueError ( f"\"to\" argument must be one of {types}" )
def pascal_iter ( string ) :
    yield from ( m . group ( 0 ) for m in re . finditer ( r'[A-Z][a-z0-9]*|[a-z0-9]+' , string ) )
def snake_iter ( string ) :
    yield from ( m . group ( 2 ) for m in re . finditer ( r'(^|_)([A-Za-z0-9]+)' , string ) )
inputs = { 'pascal' : pascal_iter , 'camel' : pascal_iter , 'snake' : snake_iter , 'constant' : snake_iter , }
def out_fun ( sep , case = None , case_fst = None ) :
    if case is None :
        case = lambda x : x
    if case_fst is None :
        case_fst = case
    return lambda tokens : sep . join ( case_fst ( token ) if i == 0 else case ( token ) for i , token in enumerate ( tokens ) )
outputs = { 'pascal' : out_fun ( '' , str . capitalize ) , 'camel' : out_fun ( '' , str . capitalize , str . lower ) , 'snake' : out_fun ( '_' , str . lower ) , 'constant' : out_fun ( '_' , str . upper ) , }
tokens = inputs [ typ ] ( string )
return outputs [ to ] ( tokens )


signature:
def read_stream ( schema , stream , * , buffer_size = io . DEFAULT_BUFFER_SIZE ) :
    """Using a schema , deserialize a stream of consecutive Avro values .
 : param str schema : json string representing the Avro schema
 : param file - like stream : a buffered stream of binary input
 : param int buffer _ size : size of bytes to read from the stream each time
 : return : yields a sequence of python data structures deserialized
 from the stream"""

implementation:
reader = _lancaster . Reader ( schema )
buf = stream . read ( buffer_size )
remainder = b''
while len ( buf ) > 0 :
    values , n = reader . read_seq ( buf )
    yield from values
    remainder = buf [ n : ]
    buf = stream . read ( buffer_size )
    if len ( buf ) > 0 and len ( remainder ) > 0 :
        ba = bytearray ( )
        ba . extend ( remainder )
        ba . extend ( buf )
        buf = memoryview ( ba ) . tobytes ( )
if len ( remainder ) > 0 :
    raise EOFError ( '{} bytes remaining but could not continue reading ' 'from stream' . format ( len ( remainder ) ) )


signature:
def parse_user_defined_metric_classes ( config_obj , metric_classes ) :
    """Parse the user defined metric class information
 : param config _ obj : ConfigParser object
 : param metric _ classes : list of metric classes to be updated
 : return :"""

implementation:
user_defined_metric_list = config_obj . get ( 'GLOBAL' , 'user_defined_metrics' ) . split ( )
for udm_string in user_defined_metric_list :
    try :
        metric_name , metric_class_name , metric_file = udm_string . split ( ':' )
    except ValueError :
        logger . error ( 'Bad user defined metric specified' )
        continue
    module_name = os . path . splitext ( os . path . basename ( metric_file ) ) [ 0 ]
    try :
        new_module = imp . load_source ( module_name , metric_file )
        new_class = getattr ( new_module , metric_class_name )
        if metric_name in metric_classes . keys ( ) :
            logger . warn ( 'Overriding pre-defined metric class definition for ' , metric_name )
        metric_classes [ metric_name ] = new_class
    except ImportError :
        logger . error ( 'Something wrong with importing a user defined metric class. Skipping metric: ' , metric_name )
        continue


signature:
def is_valid_url ( url ) :
    """Check if a given string is in the correct URL format or not
 : param str url :
 : return : True or False"""


signature:
def download_file ( url ) :
    """Download a file pointed to by url to a temp file on local disk
 : param str url :
 : return : local _ file"""

implementation:
try :
    ( local_file , headers ) = urllib . urlretrieve ( url )
except :
    sys . exit ( "ERROR: Problem downloading config file. Please check the URL (" + url + "). Exiting..." )
return local_file


signature:
def is_valid_metric_name ( metric_name ) :
    """check the validity of metric _ name in config ; the metric _ name will be used for creation of sub - dir , so only contains : alphabet , digits , ' . ' , ' - ' and ' _ '
 : param str metric _ name : metric _ name
 : return : True if valid"""

implementation:
reg = re . compile ( '^[a-zA-Z0-9\.\-\_]+$' )
if reg . match ( metric_name ) and not metric_name . startswith ( '.' ) :
    return True
else :
    return False


signature:
def get_run_time_period ( run_steps ) :
    """This method finds the time range which covers all the Run _ Steps
 : param run _ steps : list of Run _ Step objects
 : return : tuple of start and end timestamps"""

implementation:
init_ts_start = get_standardized_timestamp ( 'now' , None )
ts_start = init_ts_start
ts_end = '0'
for run_step in run_steps :
    if run_step . ts_start and run_step . ts_end :
        if run_step . ts_start < ts_start :
            ts_start = run_step . ts_start
        if run_step . ts_end > ts_end :
            ts_end = run_step . ts_end
if ts_end == '0' :
    ts_end = None
if ts_start == init_ts_start :
    ts_start = None
logger . info ( 'get_run_time_period range returned ' + str ( ts_start ) + ' to ' + str ( ts_end ) )
return ts_start , ts_end


signature:
def get_rule_strings ( config_obj , section ) :
    """Extract rule strings from a section
 : param config _ obj : ConfigParser object
 : param section : Section name
 : return : the rule strings"""

implementation:
rule_strings = { }
kwargs = dict ( config_obj . items ( section ) )
for key in kwargs . keys ( ) :
    if key . endswith ( '.sla' ) :
        rule_strings [ key . replace ( '.sla' , '' ) ] = kwargs [ key ]
        del kwargs [ key ]
return rule_strings , kwargs


signature:
def extract_diff_sla_from_config_file ( obj , options_file ) :
    """Helper function to parse diff config file , which contains SLA rules for diff comparisons"""

implementation:
rule_strings = { }
config_obj = ConfigParser . ConfigParser ( )
config_obj . optionxform = str
config_obj . read ( options_file )
for section in config_obj . sections ( ) :
    rule_strings , kwargs = get_rule_strings ( config_obj , section )
    for ( key , val ) in rule_strings . iteritems ( ) :
        set_sla ( obj , section , key , val )


signature:
def parse_basic_metric_options ( config_obj , section ) :
    """Parse basic options from metric sections of the config
 : param config _ obj : ConfigParser object
 : param section : Section name
 : return : all the parsed options"""

implementation:
infile = { }
aggr_hosts = None
aggr_metrics = None
ts_start = None
ts_end = None
precision = None
hostname = "localhost"
rule_strings = { }
important_sub_metrics = None
anomaly_detection_metrics = None
try :
    if config_obj . has_option ( section , 'important_sub_metrics' ) :
        important_sub_metrics = config_obj . get ( section , 'important_sub_metrics' ) . split ( )
        config_obj . remove_option ( section , 'important_sub_metrics' )
    if config_obj . has_option ( section , 'hostname' ) :
        hostname = config_obj . get ( section , 'hostname' )
        config_obj . remove_option ( section , 'hostname' )
    # ' infile ' is not mandatory for aggregate metrics
    if config_obj . has_option ( section , 'infile' ) :
        infile = config_obj . get ( section , 'infile' ) . split ( )
        config_obj . remove_option ( section , 'infile' )
    label = sanitize_string_section_name ( section )
    if config_obj . has_option ( section , 'ts_start' ) :
        ts_start = get_standardized_timestamp ( config_obj . get ( section , 'ts_start' ) , None )
        config_obj . remove_option ( section , 'ts_start' )
    if config_obj . has_option ( section , 'ts_end' ) :
        ts_end = get_standardized_timestamp ( config_obj . get ( section , 'ts_end' ) , None )
        config_obj . remove_option ( section , 'ts_end' )
    if config_obj . has_option ( section , 'precision' ) :
        precision = config_obj . get ( section , 'precision' )
        config_obj . remove_option ( section , 'precision' )
    # support aggregate metrics , which take aggr _ hosts and aggr _ metrics
    if config_obj . has_option ( section , 'aggr_hosts' ) :
        aggr_hosts = config_obj . get ( section , 'aggr_hosts' )
        config_obj . remove_option ( section , 'aggr_hosts' )
    if config_obj . has_option ( section , 'aggr_metrics' ) :
        aggr_metrics = config_obj . get ( section , 'aggr_metrics' )
        config_obj . remove_option ( section , 'aggr_metrics' )
    if config_obj . has_option ( section , 'anomaly_detection_metrics' ) :
        anomaly_detection_metrics = config_obj . get ( section , 'anomaly_detection_metrics' ) . split ( )
        config_obj . remove_option ( section , 'anomaly_detection_metrics' )
    rule_strings , other_options = get_rule_strings ( config_obj , section )
except ConfigParser . NoOptionError :
    logger . exception ( "Exiting.... some mandatory options are missing from the config file in section: " + section )
    sys . exit ( )
return ( hostname , infile , aggr_hosts , aggr_metrics , label , ts_start , ts_end , precision , aggr_metrics , other_options , rule_strings , important_sub_metrics , anomaly_detection_metrics )


signature:
def parse_metric_section ( config_obj , section , metric_classes , metrics , aggregate_metric_classes , outdir_default , resource_path ) :
    """Parse a metric section and create a Metric object
 : param config _ obj : ConfigParser object
 : param section : Section name
 : param metric _ classes : List of valid metric types
 : param metrics : List of all regular metric objects ( used by aggregate metric )
 : param aggregate _ metric _ classes : List of all valid aggregate metric types
 : param outdir _ default : Default output directory
 : param resource _ path : Default resource directory
 : return : An initialized Metric object"""

implementation:
( hostname , infile , aggr_hosts , aggr_metrics , label , ts_start , ts_end , precision , aggr_metrics , other_options , rule_strings , important_sub_metrics , anomaly_detection_metrics ) = parse_basic_metric_options ( config_obj , section )
# TODO : Make user specify metric _ type in config and not infer from section
metric_type = section . split ( '-' ) [ 0 ]
if metric_type in aggregate_metric_classes :
    new_metric = initialize_aggregate_metric ( section , aggr_hosts , aggr_metrics , metrics , outdir_default , resource_path , label , ts_start , ts_end , rule_strings , important_sub_metrics , anomaly_detection_metrics , other_options )
else :
    new_metric = initialize_metric ( section , infile , hostname , aggr_metrics , outdir_default , resource_path , label , ts_start , ts_end , rule_strings , important_sub_metrics , anomaly_detection_metrics , other_options )
if config_obj . has_option ( section , 'ignore' ) and config_obj . getint ( section , 'ignore' ) == 1 :
    new_metric . ignore = True
if config_obj . has_option ( section , 'calc_metrics' ) :
    new_metric . calc_metrics = config_obj . get ( section , 'calc_metrics' )
new_metric . precision = precision
return new_metric


signature:
def parse_global_section ( config_obj , section ) :
    """Parse GLOBAL section in the config to return global settings
 : param config _ obj : ConfigParser object
 : param section : Section name
 : return : ts _ start and ts _ end time"""

implementation:
ts_start = None
ts_end = None
if config_obj . has_option ( section , 'ts_start' ) :
    ts_start = get_standardized_timestamp ( config_obj . get ( section , 'ts_start' ) , None )
    config_obj . remove_option ( section , 'ts_start' )
if config_obj . has_option ( section , 'ts_end' ) :
    ts_end = get_standardized_timestamp ( config_obj . get ( section , 'ts_end' ) , None )
    config_obj . remove_option ( section , 'ts_end' )
return ts_start , ts_end


signature:
def parse_run_step_section ( config_obj , section ) :
    """Parse a RUN - STEP section in the config to return a Run _ Step object
 : param config _ obj : ConfigParser objection
 : param section : Section name
 : return : an initialized Run _ Step object"""

implementation:
kill_after_seconds = None
try :
    run_cmd = config_obj . get ( section , 'run_cmd' )
    run_rank = int ( config_obj . get ( section , 'run_rank' ) )
except ConfigParser . NoOptionError :
    logger . exception ( "Exiting.... some mandatory options are missing from the config file in section: " + section )
    sys . exit ( )
except ValueError :
    logger . error ( "Bad run_rank %s specified in section %s, should be integer. Exiting." , config_obj . get ( section , 'run_rank' ) , section )
    sys . exit ( )
if config_obj . has_option ( section , 'run_type' ) :
    run_type = config_obj . get ( section , 'run_type' )
else :
    run_type = CONSTANTS . RUN_TYPE_WORKLOAD
if config_obj . has_option ( section , 'run_order' ) :
    run_order = config_obj . get ( section , 'run_order' )
else :
    run_order = CONSTANTS . PRE_ANALYSIS_RUN
if config_obj . has_option ( section , 'call_type' ) :
    call_type = config_obj . get ( section , 'call_type' )
else :
    call_type = 'local'
if config_obj . has_option ( section , 'kill_after_seconds' ) :
    try :
        kill_after_seconds = int ( config_obj . get ( section , 'kill_after_seconds' ) )
    except ValueError :
        logger . error ( "Bad kill_after_seconds %s specified in section %s, should be integer." , config_obj . get ( section , 'kill_after_seconds' ) , section )
if call_type == 'local' :
    run_step_obj = Local_Cmd ( run_type , run_cmd , call_type , run_order , run_rank , kill_after_seconds = kill_after_seconds )
else :
    logger . error ( 'Unsupported RUN_STEP supplied, call_type should be local' )
    run_step_obj = None
return run_step_obj


signature:
def parse_graph_section ( config_obj , section , outdir_default , indir_default ) :
    """Parse the GRAPH section of the config to extract useful values
 : param config _ obj : ConfigParser object
 : param section : Section name
 : param outdir _ default : Default output directory passed in args
 : param indir _ default : Default input directory passed in args
 : return : List of options extracted from the GRAPH section"""

implementation:
graph_timezone = None
graphing_library = CONSTANTS . DEFAULT_GRAPHING_LIBRARY
crossplots = [ ]
if config_obj . has_option ( section , 'graphing_library' ) :
    graphing_library = config_obj . get ( section , 'graphing_library' )
if config_obj . has_option ( section , 'graphs' ) :
    graphs_string = config_obj . get ( section , 'graphs' )
    crossplots = graphs_string . split ( )
    # Supporting both outdir and output _ dir
if config_obj . has_option ( section , 'outdir' ) :
    outdir_default = config_obj . get ( section , 'outdir' )
if config_obj . has_option ( section , 'output_dir' ) :
    outdir_default = config_obj . get ( section , 'output_dir' )
if config_obj . has_option ( section , 'input_dir' ) :
    indir_default = config_obj . get ( section , 'input_dir' )
if config_obj . has_option ( section , 'graph_timezone' ) :
    graph_timezone = config_obj . get ( section , 'graph_timezone' )
    if graph_timezone not in ( "UTC" , "PST" , "PDT" ) :
        logger . warn ( 'Unsupported timezone ' + graph_timezone + ' specified in option graph_timezone. Will use UTC instead' )
        graph_timezone = "UTC"
return graphing_library , crossplots , outdir_default , indir_default , graph_timezone