signature:
def check_directories ( self ) :
    """Creates base directories for app , virtualenv , and nginx"""

implementation:
self . log . debug ( 'Checking directories' )
if not os . path . exists ( self . _ve_dir ) :
    os . makedirs ( self . _ve_dir )
if not os . path . exists ( self . _app_dir ) :
    os . makedirs ( self . _app_dir )
if not os . path . exists ( self . _conf_dir ) :
    os . makedirs ( self . _conf_dir )
if not os . path . exists ( self . _var_dir ) :
    os . makedirs ( self . _var_dir )
if not os . path . exists ( self . _log_dir ) :
    os . makedirs ( self . _log_dir )
if not os . path . exists ( self . _script_dir ) :
    os . makedirs ( self . _script_dir )
# copy uswgi _ params for nginx
uwsgi_params = '/etc/nginx/uwsgi_params'
if os . path . exists ( uwsgi_params ) :
    shutil . copy ( uwsgi_params , self . _conf_dir )
else :
    logging . warning ( 'Unable to find Nginx uwsgi_params.  You must manually copy this to {0}.' . format ( self . _conf_dir ) )
# copy mime . types for nginx
mime_types = '/etc/nginx/mime.types'
if os . path . exists ( mime_types ) :
    shutil . copy ( mime_types , self . _conf_dir )
    self . _include_mimetypes = True
else :
    logging . warn ( 'Unable to find mime.types for Nginx.  You must manually copy this to {0}.' . format ( self . _conf_dir ) )


signature:
def create_virtualenv ( self ) :
    """Creates the virtualenv for the project"""

implementation:
if check_command ( 'virtualenv' ) :
    ve_dir = os . path . join ( self . _ve_dir , self . _project_name )
    if os . path . exists ( ve_dir ) :
        if self . _force :
            logging . warn ( 'Removing existing virtualenv' )
            shutil . rmtree ( ve_dir )
        else :
            logging . warn ( 'Found existing virtualenv; not creating (use --force to overwrite)' )
            return
    logging . info ( 'Creating virtualenv' )
    p = subprocess . Popen ( 'virtualenv --no-site-packages {0} > /dev/null' . format ( ve_dir ) , shell = True )
    os . waitpid ( p . pid , 0 )
    # install modules
    for m in self . _modules :
        self . log . info ( 'Installing module {0}' . format ( m ) )
        p = subprocess . Popen ( '{0} install {1} > /dev/null' . format ( os . path . join ( self . _ve_dir , self . _project_name ) + os . sep + 'bin' + os . sep + 'pip' , m ) , shell = True )
        os . waitpid ( p . pid , 0 )


signature:
def create_nginx_config ( self ) :
    """Creates the Nginx configuration for the project"""

implementation:
cfg = '# nginx config for {0}\n' . format ( self . _project_name )
if not self . _shared_hosting : # user
    if self . _user :
        cfg += 'user {0};\n' . format ( self . _user )
    # misc nginx config
    cfg += 'worker_processes 1;\nerror_log {0}-errors.log;\n\
pid {1}_    nginx.pid;\n\n' . format ( os . path . join ( self . _log_dir , self . _project_name ) , os . path . join ( self . _var_dir , self . _project_name ) )
    cfg += 'events {\n\tworker_connections 32;\n}\n\n'
    # http section
    cfg += 'http {\n'
    if self . _include_mimetypes :
        cfg += '\tinclude mime.types;\n'
    cfg += '\tdefault_type application/octet-stream;\n'
    cfg += '\tclient_max_body_size 1G;\n'
    cfg += '\tproxy_max_temp_file_size 0;\n'
    cfg += '\tproxy_buffering off;\n'
    cfg += '\taccess_log {0}-access.log;\n' . format ( os . path . join ( self . _log_dir , self . _project_name ) )
    cfg += '\tsendfile on;\n'
    cfg += '\tkeepalive_timeout 65;\n'
    # server section
cfg += '\tserver {\n'
cfg += '\t\tlisten 0.0.0.0:{0};\n' . format ( self . _port )
if self . _server_name :
    cfg += '\t\tserver_name {0};\n' . format ( self . _server_name )
# location section
cfg += '\t\tlocation / {\n'
cfg += '\t\t\tuwsgi_pass unix:///{0}.sock;\n' . format ( os . path . join ( self . _var_dir , self . _project_name ) )
cfg += '\t\t\tinclude uwsgi_params;\n'
cfg += '\t\t}\n\n'
# end location
# error page templates
cfg += '\t\terror_page 500 502 503 504 /50x.html;\n'
cfg += '\t\tlocation = /50x.html {\n'
cfg += '\t\t\troot html;\n'
# end error page section
cfg += '\t\t}\n'
# end server section
cfg += '\t}\n'
if not self . _shared_hosting : # end http section
    cfg += '}\n'
# create conf
f = open ( self . _nginx_config , 'w' )
f . write ( cfg )
f . close ( )


signature:
def create_manage_scripts ( self ) :
    """Creates scripts to start and stop the application"""

implementation:
# create start script
start = '# start script for {0}\n\n' . format ( self . _project_name )
# start uwsgi
start += 'echo \'Starting uWSGI...\'\n'
start += 'sh {0}.uwsgi\n' . format ( os . path . join ( self . _conf_dir , self . _project_name ) )
start += 'sleep 1\n'
# start nginx
start += 'echo \'Starting Nginx...\'\n'
start += 'nginx -c {0}_nginx.conf\n' . format ( os . path . join ( self . _conf_dir , self . _project_name ) )
start += 'sleep 1\n'
start += 'echo \'{0} started\'\n\n' . format ( self . _project_name )
# stop script
stop = '# stop script for {0}\n\n' . format ( self . _project_name )
# stop nginx
stop += 'if [ -e {0}_nginx.pid ]; then nginx -c {1}_nginx.conf -s stop ; fi\n' . format ( os . path . join ( self . _var_dir , self . _project_name ) , os . path . join ( self . _conf_dir , self . _project_name ) )
# stop uwsgi
stop += 'if [ -e {0}_uwsgi.pid ]; then kill -9 `cat {0}_uwsgi.pid` ; rm {0}_uwsgi.pid 2>&1 > /dev/null ; fi\n' . format ( os . path . join ( self . _var_dir , self . _project_name ) )
stop += 'echo \'{0} stopped\'\n' . format ( self . _project_name )
# write scripts
start_file = '{0}_start.sh' . format ( os . path . join ( self . _script_dir , self . _project_name ) )
stop_file = '{0}_stop.sh' . format ( os . path . join ( self . _script_dir , self . _project_name ) )
f = open ( start_file , 'w' )
f . write ( start )
f . close ( )
f = open ( stop_file , 'w' )
f . write ( stop )
f . close ( )
# make executable
os . chmod ( start_file , 0754 )
os . chmod ( stop_file , 0754 )


signature:
def create ( self ) :
    """Creates the full project"""

implementation:
# create virtualenv
self . create_virtualenv ( )
# create project
self . create_project ( )
# generate uwsgi script
self . create_uwsgi_script ( )
# generate nginx config
self . create_nginx_config ( )
# generate management scripts
self . create_manage_scripts ( )
logging . info ( '** Make sure to set proper permissions for the webserver user account on the var and log directories in the project root' )


signature:
def dasherize ( value ) :
    """Dasherizes the passed value ."""

implementation:
value = value . strip ( )
value = re . sub ( r'([A-Z])' , r'-\1' , value )
value = re . sub ( r'[-_\s]+' , r'-' , value )
value = re . sub ( r'^-' , r'' , value )
value = value . lower ( )
return value


signature:
def redirect ( cls , request , response ) :
    """Redirect to the canonical URI for this resource ."""

implementation:
if cls . meta . legacy_redirect :
    if request . method in ( 'GET' , 'HEAD' , ) : # A SAFE request is allowed to redirect using a 301
        response . status = http . client . MOVED_PERMANENTLY
    else : # All other requests must use a 307
        response . status = http . client . TEMPORARY_REDIRECT
else : # Modern redirects are allowed . Let ' s have some fun .
# Hopefully you ' re client supports this .
# The RFC explicitly discourages UserAgent sniffing .
    response . status = http . client . PERMANENT_REDIRECT
# Terminate the connection .
response . close ( )


signature:
def view ( cls , request , response ) :
    """Entry - point of the request / response cycle ; Handles resource creation
 and delegation .
 @ param [ in ] requset
 The HTTP request object ; containing accessors for information
 about the request .
 @ param [ in ] response
 The HTTP response object ; contains accessors for modifying
 the information that will be sent to the client ."""

implementation:
# Determine if we need to redirect .
test = cls . meta . trailing_slash
if test ^ request . path . endswith ( '/' ) : # Construct a new URL by removing or adding the trailing slash .
    path = request . path + '/' if test else request . path [ : - 1 ]
    response [ 'Location' ] = '{}://{}{}{}{}' . format ( request . protocol . lower ( ) , request . host , request . mount_point , path , '?' + request . query if request . query else '' )
    # Redirect to the version with the correct trailing slash .
    return cls . redirect ( request , response )
try : # Instantiate the resource .
    obj = cls ( request , response )
    # Bind the request and response objects to the constructed
    # resource .
    request . bind ( obj )
    response . bind ( obj )
    # Bind the request object to the resource .
    # This is used to facilitate the serializer and deserializer .
    obj . _request = request
    # Initiate the dispatch cycle and handle its result on
    # synchronous requests .
    result = obj . dispatch ( request , response )
    if not response . asynchronous : # There is several things that dispatch is allowed to return .
        if ( isinstance ( result , collections . Iterable ) and not isinstance ( result , six . string_types ) ) : # Return the stream generator .
            return cls . stream ( response , result )
        else : # Leave it up to the response to throw or write whatever
        # we got back .
            response . end ( result )
            if response . body : # Return the body if there was any set .
                return response . body
except http . exceptions . BaseHTTPException as e : # Something that we can handle and return properly happened .
# Set response properties from the exception .
    response . status = e . status
    response . headers . update ( e . headers )
    if e . content : # Write the exception body if present and close
    # the response .
    # TODO : Use the plain - text encoder .
        response . send ( e . content , serialize = True , format = 'json' )
    # Terminate the connection and return the body .
    response . close ( )
    if response . body :
        return response . body
except Exception : # Something unexpected happened .
# Log error message to the logger .
    logger . exception ( 'Internal server error' )
    # Write a debug message for the client .
    if not response . streaming and not response . closed :
        response . status = http . client . INTERNAL_SERVER_ERROR
        response . headers . clear ( )
        response . close ( )


signature:
def parse ( cls , path ) :
    """Parses out parameters and separates them out of the path .
 This uses one of the many defined patterns on the options class . But ,
 it defaults to a no - op if there are no defined patterns ."""

implementation:
# Iterate through the available patterns .
for resource , pattern in cls . meta . patterns : # Attempt to match the path .
    match = re . match ( pattern , path )
    if match is not None : # Found something .
        return resource , match . groupdict ( ) , match . string [ match . end ( ) : ]
# No patterns at all ; return unsuccessful .
return None if not cls . meta . patterns else False


signature:
def traverse ( cls , request , params = None ) :
    """Traverses down the path and determines the accessed resource .
 This makes use of the patterns array to implement simple traversal .
 This defaults to a no - op if there are no defined patterns ."""

implementation:
# Attempt to parse the path using a pattern .
result = cls . parse ( request . path )
if result is None : # No parsing was requested ; no - op .
    return cls , { }
elif not result : # Parsing failed ; raise 404.
    raise http . exceptions . NotFound ( )
# Partition out the result .
resource , data , rest = result
if params : # Append params to data .
    data . update ( params )
if resource is None : # No traversal ; return parameters .
    return cls , data
# Modify the path appropriately .
if data . get ( 'path' ) is not None :
    request . path = data . pop ( 'path' )
elif rest is not None :
    request . path = rest
# Send us through traversal again .
result = resource . traverse ( request , params = data )
return result


signature:
def stream ( cls , response , sequence ) :
    """Helper method used in conjunction with the view handler to
 stream responses to the client ."""

implementation:
# Construct the iterator and run the sequence once in order
# to capture any headers and status codes set .
iterator = iter ( sequence )
data = { 'chunk' : next ( iterator ) }
response . streaming = True
def streamer ( ) : # Iterate through the iterator and yield its content
    while True :
        if response . asynchronous : # Yield our current chunk .
            yield data [ 'chunk' ]
        else : # Write the chunk to the response
            response . send ( data [ 'chunk' ] )
            # Yield its body
            yield response . body
            # Unset the body .
            response . body = None
        try : # Get the next chunk .
            data [ 'chunk' ] = next ( iterator )
        except StopIteration : # Get out of the loop .
            break
    if not response . asynchronous : # Close the response .
        response . close ( )
# Return the streaming function .
return streamer ( )


signature:
def deserialize ( self , request = None , text = None , format = None ) :
    """Deserializes the text using a determined deserializer .
 @ param [ in ] request
 The request object to pull information from ; normally used to
 determine the deserialization format ( when ` format ` is
 not provided ) .
 @ param [ in ] text
 The text to be deserialized . Can be left blank and the
 request will be read .
 @ param [ in ] format
 A specific format to deserialize in ; if provided , no detection is
 done . If not provided , the content - type header is looked at to
 determine an appropriate deserializer .
 @ returns
 A tuple of the deserialized data and an instance of the
 deserializer used ."""

implementation:
if isinstance ( self , Resource ) :
    if not request : # Ensure we have a response object .
        request = self . _request
Deserializer = None
if format : # An explicit format was given ; do not attempt to auto - detect
# a deserializer .
    Deserializer = self . meta . deserializers [ format ]
if not Deserializer : # Determine an appropriate deserializer to use by
# introspecting the request object and looking at
# the ` Content - Type ` header .
    media_ranges = request . get ( 'Content-Type' )
    if media_ranges : # Parse the media ranges and determine the deserializer
    # that is the closest match .
        media_types = six . iterkeys ( self . _deserializer_map )
        media_type = mimeparse . best_match ( media_types , media_ranges )
        if media_type :
            format = self . _deserializer_map [ media_type ]
            Deserializer = self . meta . deserializers [ format ]
    else : # Client didn ' t provide a content - type ; we ' re supposed
    # to auto - detect .
    # TODO : Implement this .
        pass
if Deserializer :
    try : # Attempt to deserialize the data using the determined
    # deserializer .
        deserializer = Deserializer ( )
        data = deserializer . deserialize ( request = request , text = text )
        return data , deserializer
    except ValueError : # Failed to deserialize the data .
        pass
# Failed to determine a deserializer ; or failed to deserialize .
raise http . exceptions . UnsupportedMediaType ( )


signature:
def serialize ( self , data , response = None , request = None , format = None ) :
    """Serializes the data using a determined serializer .
 @ param [ in ] data
 The data to be serialized .
 @ param [ in ] response
 The response object to serialize the data to .
 If this method is invoked as an instance method , the response
 object can be omitted and it will be taken from the instance .
 @ param [ in ] request
 The request object to pull information from ; normally used to
 determine the serialization format ( when ` format ` is not provided ) .
 May be used by some serializers as well to pull additional headers .
 If this method is invoked as an instance method , the request
 object can be omitted and it will be taken from the instance .
 @ param [ in ] format
 A specific format to serialize in ; if provided , no detection is
 done . If not provided , the accept header ( as well as the URL
 extension ) is looked at to determine an appropriate serializer .
 @ returns
 A tuple of the serialized text and an instance of the
 serializer used ."""

implementation:
if isinstance ( self , Resource ) :
    if not request : # Ensure we have a response object .
        request = self . _request
Serializer = None
if format : # An explicit format was given ; do not attempt to auto - detect
# a serializer .
    Serializer = self . meta . serializers [ format ]
if not Serializer : # Determine an appropriate serializer to use by
# introspecting the request object and looking at the ` Accept `
# header .
    media_ranges = ( request . get ( 'Accept' ) or '*/*' ) . strip ( )
    if not media_ranges : # Default the media ranges to * / *
        media_ranges = '*/*'
    if media_ranges != '*/*' : # Parse the media ranges and determine the serializer
    # that is the closest match .
        media_types = six . iterkeys ( self . _serializer_map )
        media_type = mimeparse . best_match ( media_types , media_ranges )
        if media_type :
            format = self . _serializer_map [ media_type ]
            Serializer = self . meta . serializers [ format ]
    else : # Client indicated no preference ; use the default .
        default = self . meta . default_serializer
        Serializer = self . meta . serializers [ default ]
if Serializer :
    try : # Attempt to serialize the data using the determined
    # serializer .
        serializer = Serializer ( request , response )
        return serializer . serialize ( data ) , serializer
    except ValueError : # Failed to serialize the data .
        pass
# Either failed to determine a serializer or failed to serialize
# the data ; construct a list of available and valid encoders .
available = { }
for name in self . meta . allowed_serializers :
    Serializer = self . meta . serializers [ name ]
    instance = Serializer ( request , None )
    if instance . can_serialize ( data ) :
        available [ name ] = Serializer . media_types [ 0 ]
# Raise a Not Acceptable exception .
raise http . exceptions . NotAcceptable ( available )


signature:
def _process_cross_domain_request ( cls , request , response ) :
    """Facilitate Cross - Origin Requests ( CORs ) ."""

implementation:
# Step 1
# Check for Origin header .
origin = request . get ( 'Origin' )
if not origin :
    return
# Step 2
# Check if the origin is in the list of allowed origins .
if not ( origin in cls . meta . http_allowed_origins or '*' == cls . meta . http_allowed_origins ) :
    return
# Step 3
# Try to parse the Request - Method header if it exists .
method = request . get ( 'Access-Control-Request-Method' )
if method and method not in cls . meta . http_allowed_methods :
    return
# Step 4
# Try to parse the Request - Header header if it exists .
headers = request . get ( 'Access-Control-Request-Headers' , ( ) )
if headers :
    headers = [ h . strip ( ) for h in headers . split ( ',' ) ]
# Step 5
# Check if the headers are allowed on this resource .
allowed_headers = [ h . lower ( ) for h in cls . meta . http_allowed_headers ]
if any ( h . lower ( ) not in allowed_headers for h in headers ) :
    return
# Step 6
# Always add the origin .
response [ 'Access-Control-Allow-Origin' ] = origin
# TODO : Check if we can provide credentials .
response [ 'Access-Control-Allow-Credentials' ] = 'true'
# Step 7
# TODO : Optionally add Max - Age header .
# Step 8
# Add the allowed methods .
allowed_methods = ', ' . join ( cls . meta . http_allowed_methods )
response [ 'Access-Control-Allow-Methods' ] = allowed_methods
# Step 9
# Add any allowed headers .
allowed_headers = ', ' . join ( cls . meta . http_allowed_headers )
if allowed_headers :
    response [ 'Access-Control-Allow-Headers' ] = allowed_headers
# Step 10
# Add any exposed headers .
exposed_headers = ', ' . join ( cls . meta . http_exposed_headers )
if exposed_headers :
    response [ 'Access-Control-Expose-Headers' ] = exposed_headers


signature:
def dispatch ( self , request , response ) :
    """Entry - point of the dispatch cycle for this resource .
 Performs common work such as authentication , decoding , etc . before
 handing complete control of the result to a function with the
 same name as the request method ."""

implementation:
# Assert authentication and attempt to get a valid user object .
self . require_authentication ( request )
# Assert accessibiltiy of the resource in question .
self . require_accessibility ( request . user , request . method )
# Facilitate CORS by applying various headers .
# This must be done on every request .
# TODO : Provide cross _ domain configuration that turns this off .
self . _process_cross_domain_request ( request , response )
# Route the HTTP / 1.1 request to an appropriate method .
return self . route ( request , response )


signature:
def require_authentication ( self , request ) :
    """Ensure we are authenticated ."""

implementation:
request . user = user = None
if request . method == 'OPTIONS' : # Authentication should not be checked on an OPTIONS request .
    return
for auth in self . meta . authentication :
    user = auth . authenticate ( request )
    if user is False : # Authentication protocol failed to authenticate ;
    # pass the baton .
        continue
    if user is None and not auth . allow_anonymous : # Authentication protocol determined the user is
    # unauthenticated .
        auth . unauthenticated ( )
    # Authentication protocol determined the user is indeed
    # authenticated ( or not ) ; Store the user for later reference .
    request . user = user
    return
if not user and not auth . allow_anonymous : # No authenticated user found and protocol doesn ' t allow
# anonymous users .
    auth . unauthenticated ( )


signature:
def require_accessibility ( self , user , method ) :
    """Ensure we are allowed to access this resource ."""

implementation:
if method == 'OPTIONS' : # Authorization should not be checked on an OPTIONS request .
    return
authz = self . meta . authorization
if not authz . is_accessible ( user , method , self ) : # User is not authorized ; raise an appropriate message .
    authz . unaccessible ( )


signature:
def require_http_allowed_method ( cls , request ) :
    """Ensure that we ' re allowed to use this HTTP method ."""

implementation:
allowed = cls . meta . http_allowed_methods
if request . method not in allowed : # The specified method is not allowed for the resource
# identified by the request URI .
# RFC 2616 § 10.4.6 — 405 Method Not Allowed
    raise http . exceptions . MethodNotAllowed ( allowed )


signature:
def route ( self , request , response ) :
    """Processes every request .
 Directs control flow to the appropriate HTTP / 1.1 method ."""

implementation:
# Ensure that we ' re allowed to use this HTTP method .
self . require_http_allowed_method ( request )
# Retrieve the function corresponding to this HTTP method .
function = getattr ( self , request . method . lower ( ) , None )
if function is None : # Server is not capable of supporting it .
    raise http . exceptions . NotImplemented ( )
# Delegate to the determined function to process the request .
return function ( request , response )


signature:
def options ( self , request , response ) :
    """Process an ` OPTIONS ` request .
 Used to initiate a cross - origin request . All handling specific to
 CORS requests is done on every request however this method also
 returns a list of available methods ."""

implementation:
# Gather a list available HTTP / 1.1 methods for this URI .
response [ 'Allowed' ] = ', ' . join ( self . meta . http_allowed_methods )
# All CORS handling is done for every HTTP / 1.1 method .
# No more handling is neccesary ; set the response to 200 and return .
response . status = http . client . OK


signature:
def resource ( ** kwargs ) :
    """Wraps the decorated function in a lightweight resource ."""

implementation:
def inner ( function ) :
    name = kwargs . pop ( 'name' , None )
    if name is None :
        name = utils . dasherize ( function . __name__ )
    methods = kwargs . pop ( 'methods' , None )
    if isinstance ( methods , six . string_types ) : # Tuple - ify the method if we got just a string .
        methods = methods ,
    # Construct a handler .
    handler = ( function , methods )
    if name not in _resources : # Initiate the handlers list .
        _handlers [ name ] = [ ]
        # Construct a light - weight resource using the passed kwargs
        # as the arguments for the meta .
        from armet import resources
        kwargs [ 'name' ] = name
        class LightweightResource ( resources . Resource ) :
            Meta = type ( str ( 'Meta' ) , ( ) , kwargs )
            def route ( self , request , response ) :
                for handler , methods in _handlers [ name ] :
                    if methods is None or request . method in methods :
                        return handler ( request , response )
                resources . Resource . route ( self )
        # Construct and add this resource .
        _resources [ name ] = LightweightResource
    # Add this to the handlers .
    _handlers [ name ] . append ( handler )
    # Return the resource .
    return _resources [ name ]
# Return the inner method .
return inner


signature:
def threewise ( iterable ) :
    """s - > ( None , s0 , s1 ) , ( s0 , s1 , s2 ) , . . . ( sn - 1 , sn , None )
 example :
 for ( last , cur , next ) in threewise ( l ) :"""

implementation:
a , b , c = itertools . tee ( iterable , 3 )
def prepend ( val , l ) :
    yield val
    for i in l :
        yield i
def postpend ( val , l ) :
    for i in l :
        yield i
    yield val
next ( c , None )
for _xa , _xb , _xc in six . moves . zip ( prepend ( None , a ) , b , postpend ( None , c ) ) :
    yield ( _xa , _xb , _xc )


signature:
def lines2less ( lines ) :
    """input : lines = list / iterator of strings
 eg : lines = [ " This is the first line " , " This is the second line " ]
 output : print those lines to stdout if the output is short + narrow
 otherwise print the lines to less"""

implementation:
lines = iter ( lines )
# cast list to iterator
# print output to stdout if small , otherwise to less
has_term = True
terminal_cols = 100
try :
    terminal_cols = terminal_size ( )
except : # getting terminal info failed - - maybe it ' s a
# weird situation like running through cron
    has_term = False
MAX_CAT_ROWS = 20
# if there are < = this many rows then print to screen
first_rows = list ( itertools . islice ( lines , 0 , MAX_CAT_ROWS ) )
wide = any ( len ( l ) > terminal_cols for l in first_rows )
use_less = False
if has_term and ( wide or len ( first_rows ) == MAX_CAT_ROWS ) :
    use_less = True
lines = itertools . chain ( first_rows , lines )
lines = six . moves . map ( lambda x : x + '\n' , lines )
if use_less :
    lesspager ( lines )
else :
    for l in lines :
        sys . stdout . write ( l )


signature:
def argmax ( l , f = None ) :
    """http : / / stackoverflow . com / questions / 5098580 / implementing - argmax - in - python"""

implementation:
if f :
    l = [ f ( i ) for i in l ]
return max ( enumerate ( l ) , key = lambda x : x [ 1 ] ) [ 0 ]


signature:
def render_to_string ( self ) :
    """Render to cookie strings ."""

implementation:
values = ''
for key , value in self . items ( ) :
    values += '{}={};' . format ( key , value )
return values


signature:
def from_cookie_string ( self , cookie_string ) :
    """update self with cookie _ string ."""

implementation:
for key_value in cookie_string . split ( ';' ) :
    if '=' in key_value :
        key , value = key_value . split ( '=' , 1 )
    else :
        key = key_value
    strip_key = key . strip ( )
    if strip_key and strip_key . lower ( ) not in COOKIE_ATTRIBUTE_NAMES :
        self [ strip_key ] = value . strip ( )


signature:
def _add_method ( self , effect , verb , resource , conditions ) :
    """Adds a method to the internal lists of allowed or denied methods .
 Each object in the internal list contains a resource ARN and a
 condition statement . The condition statement can be null ."""

implementation:
if verb != '*' and not hasattr ( HttpVerb , verb ) :
    raise NameError ( 'Invalid HTTP verb ' + verb + '. Allowed verbs in HttpVerb class' )
resource_pattern = re . compile ( self . path_regex )
if not resource_pattern . match ( resource ) :
    raise NameError ( 'Invalid resource path: ' + resource + '. Path should match ' + self . path_regex )
if resource [ : 1 ] == '/' :
    resource = resource [ 1 : ]
resource_arn = ( 'arn:aws:execute-api:' + self . region + ':' + self . aws_account_id + ':' + self . rest_api_id + '/' + self . stage + '/' + verb + '/' + resource )
if effect . lower ( ) == 'allow' :
    self . allowMethods . append ( { 'resource_arn' : resource_arn , 'conditions' : conditions } )
elif effect . lower ( ) == 'deny' :
    self . denyMethods . append ( { 'resource_arn' : resource_arn , 'conditions' : conditions } )


signature:
def _get_effect_statement ( self , effect , methods ) :
    """This function loops over an array of objects containing
 a resourceArn and conditions statement and generates
 the array of statements for the policy ."""

implementation:
statements = [ ]
if len ( methods ) > 0 :
    statement = self . _get_empty_statement ( effect )
    for method in methods :
        if ( method [ 'conditions' ] is None or len ( method [ 'conditions' ] ) == 0 ) :
            statement [ 'Resource' ] . append ( method [ 'resource_arn' ] )
        else :
            cond_statement = self . _get_empty_statement ( effect )
            cond_statement [ 'Resource' ] . append ( method [ 'resource_arn' ] )
            cond_statement [ 'Condition' ] = method [ 'conditions' ]
            statements . append ( cond_statement )
    statements . append ( statement )
return statements


signature:
def allow_method_with_conditions ( self , verb , resource , conditions ) :
    """Adds an API Gateway method ( Http verb + Resource path ) to the
 list of allowed methods and includes a condition for the policy
 statement . More on AWS policy conditions here :
 http : / / docs . aws . amazon . com / IAM / latest / UserGuide /
 reference _ policies _ elements . html # Condition"""

implementation: self . _add_method ( 'Allow' , verb , resource , conditions )


signature:
def deny_method_with_conditions ( self , verb , resource , conditions ) :
    """Adds an API Gateway method ( Http verb + Resource path ) to the
 list of denied methods and includes a condition for the policy
 statement . More on AWS policy conditions here :
 http : / / docs . aws . amazon . com / IAM / latest / UserGuide /
 reference _ policies _ elements . html # Condition"""

implementation: self . _add_method ( 'Deny' , verb , resource , conditions )


signature:
def build ( self ) :
    """Generates the policy document based on the internal lists of
 allowed and denied conditions . This will generate a policy with
 two main statements for the effect : one statement for Allow and
 one statement for Deny . Methods that includes conditions will
 have their own statement in the policy ."""

implementation:
if ( ( self . allowMethods is None or len ( self . allowMethods ) == 0 ) and ( self . denyMethods is None or len ( self . denyMethods ) == 0 ) ) :
    raise NameError ( 'No statements defined for the policy' )
policy = { 'principalId' : self . principal_id , 'policyDocument' : { 'Version' : self . version , 'Statement' : [ ] } }
policy [ 'policyDocument' ] [ 'Statement' ] . extend ( self . _get_effect_statement ( 'Allow' , self . allowMethods ) )
policy [ 'policyDocument' ] [ 'Statement' ] . extend ( self . _get_effect_statement ( 'Deny' , self . denyMethods ) )
return policy


signature:
def deref ( self , data ) :
    """AWS doesn ' t quite have Swagger 2.0 validation right and will fail
 on some refs . So , we need to convert to deref before
 upload ."""

implementation:
# We have to make a deepcopy here to create a proper JSON
# compatible object , otherwise ` json . dumps ` fails when it
# hits jsonref . JsonRef objects .
deref = copy . deepcopy ( jsonref . JsonRef . replace_refs ( data ) )
# Write out JSON version because we might want this .
self . write_template ( deref , filename = 'swagger.json' )
return deref


signature:
def check_pre_requirements ( pre_requirements ) :
    """Check all necessary system requirements to exist .
 : param pre _ requirements :
 Sequence of pre - requirements to check by running
 ` ` where < pre _ requirement > ` ` on Windows and ` ` which . . . ` ` elsewhere ."""

implementation:
pre_requirements = set ( pre_requirements or [ ] )
pre_requirements . add ( 'virtualenv' )
for requirement in pre_requirements :
    if not which ( requirement ) :
        print_error ( 'Requirement {0!r} is not found in system' . format ( requirement ) )
        return False
return True


signature:
def config_to_args ( config ) :
    """Convert config dict to arguments list .
 : param config : Configuration dict ."""

implementation:
result = [ ]
for key , value in iteritems ( config ) :
    if value is False :
        continue
    key = '--{0}' . format ( key . replace ( '_' , '-' ) )
    if isinstance ( value , ( list , set , tuple ) ) :
        for item in value :
            result . extend ( ( key , smart_str ( item ) ) )
    elif value is not True :
        result . extend ( ( key , smart_str ( value ) ) )
    else :
        result . append ( key )
return tuple ( result )


signature:
def create_env ( env , args , recreate = False , ignore_activated = False , quiet = False ) :
    """Create virtual environment .
 : param env : Virtual environment name .
 : param args : Pass given arguments to ` ` virtualenv ` ` script .
 : param recerate : Recreate virtual environment ? By default : False
 : param ignore _ activated :
 Ignore already activated virtual environment and create new one . By
 default : False
 : param quiet : Do not output messages into terminal . By default : False"""

implementation:
cmd = None
result = True
inside_env = hasattr ( sys , 'real_prefix' ) or os . environ . get ( 'VIRTUAL_ENV' )
env_exists = os . path . isdir ( env )
if not quiet :
    print_message ( '== Step 1. Create virtual environment ==' )
if ( recreate or ( not inside_env and not env_exists ) ) or ( ignore_activated and not env_exists ) :
    cmd = ( 'virtualenv' , ) + args + ( env , )
if not cmd and not quiet :
    if inside_env :
        message = 'Working inside of virtual environment, done...'
    else :
        message = 'Virtual environment {0!r} already created, done...'
    print_message ( message . format ( env ) )
if cmd :
    with disable_error_handler ( ) :
        result = not run_cmd ( cmd , echo = not quiet )
if not quiet :
    print_message ( )
return result


signature:
def install ( env , requirements , args , ignore_activated = False , install_dev_requirements = False , quiet = False ) :
    """Install library or project into virtual environment .
 : param env : Use given virtual environment name .
 : param requirements : Use given requirements file for pip .
 : param args : Pass given arguments to pip script .
 : param ignore _ activated :
 Do not run pip inside already activated virtual environment . By
 default : False
 : param install _ dev _ requirements :
 When enabled install prefixed or suffixed dev requirements after
 original installation process completed . By default : False
 : param quiet : Do not output message to terminal . By default : False"""

implementation:
if os . path . isfile ( requirements ) :
    args += ( '-r' , requirements )
    label = 'project'
else :
    args += ( '-U' , '-e' , '.' )
    label = 'library'
# Attempt to install development requirements
if install_dev_requirements :
    dev_requirements = None
    dirname = os . path . dirname ( requirements )
    basename , ext = os . path . splitext ( os . path . basename ( requirements ) )
    # Possible dev requirements files :
    # * < requirements > - dev . < ext >
    # * dev - < requirements > . < ext >
    # * < requirements > _ dev . < ext >
    # * dev _ < requirements > . < ext >
    # * < requirements > dev . < ext >
    # * dev < requirements > . < ext >
    # Where < requirements > is basename of given requirements file to use
    # and < ext > is its extension .
    for delimiter in ( '-' , '_' , '' ) :
        filename = os . path . join ( dirname , '' . join ( ( basename , delimiter , 'dev' , ext ) ) )
        if os . path . isfile ( filename ) :
            dev_requirements = filename
            break
        filename = os . path . join ( dirname , '' . join ( ( 'dev' , delimiter , basename , ext ) ) )
        if os . path . isfile ( filename ) :
            dev_requirements = filename
            break
    # If at least one dev requirements file found , install dev requirements
    if dev_requirements :
        args += ( '-r' , dev_requirements )
if not quiet :
    print_message ( '== Step 2. Install {0} ==' . format ( label ) )
result = not pip_cmd ( env , ( 'install' , ) + args , ignore_activated , echo = not quiet )
if not quiet :
    print_message ( )
return result


signature:
def iteritems ( data , ** kwargs ) :
    """Iterate over dict items ."""

implementation: return iter ( data . items ( ** kwargs ) ) if IS_PY3 else data . iteritems ( ** kwargs )


signature:
def iterkeys ( data , ** kwargs ) :
    """Iterate over dict keys ."""

implementation: return iter ( data . keys ( ** kwargs ) ) if IS_PY3 else data . iterkeys ( ** kwargs )


signature:
def main ( * args ) :
    r"""Bootstrap Python projects and libraries with virtualenv and pip .
 Also check system requirements before bootstrap and run post bootstrap
 hook if any .
 : param \ * args : Command line arguments list ."""

implementation:
# Create parser , read arguments from direct input or command line
with disable_error_handler ( ) :
    args = parse_args ( args or sys . argv [ 1 : ] )
# Read current config from file and command line arguments
config = read_config ( args . config , args )
if config is None :
    return True
bootstrap = config [ __script__ ]
# Check pre - requirements
if not check_pre_requirements ( bootstrap [ 'pre_requirements' ] ) :
    return True
# Create virtual environment
env_args = prepare_args ( config [ 'virtualenv' ] , bootstrap )
if not create_env ( bootstrap [ 'env' ] , env_args , bootstrap [ 'recreate' ] , bootstrap [ 'ignore_activated' ] , bootstrap [ 'quiet' ] ) : # Exit if couldn ' t create virtual environment
    return True
# And install library or project here
pip_args = prepare_args ( config [ 'pip' ] , bootstrap )
if not install ( bootstrap [ 'env' ] , bootstrap [ 'requirements' ] , pip_args , bootstrap [ 'ignore_activated' ] , bootstrap [ 'install_dev_requirements' ] , bootstrap [ 'quiet' ] ) : # Exist if couldn ' t install requirements into venv
    return True
# Run post - bootstrap hook
run_hook ( bootstrap [ 'hook' ] , bootstrap , bootstrap [ 'quiet' ] )
# All OK !
if not bootstrap [ 'quiet' ] :
    print_message ( 'All OK!' )
# False means everything went alright , exit code : 0
return False


signature:
def parse_args ( args ) :
    """Parse args from command line by creating argument parser instance and
 process it .
 : param args : Command line arguments list ."""

implementation:
from argparse import ArgumentParser
description = ( 'Bootstrap Python projects and libraries with virtualenv ' 'and pip.' )
parser = ArgumentParser ( description = description )
parser . add_argument ( '--version' , action = 'version' , version = __version__ )
parser . add_argument ( '-c' , '--config' , default = DEFAULT_CONFIG , help = 'Path to config file. By default: {0}' . format ( DEFAULT_CONFIG ) )
parser . add_argument ( '-p' , '--pre-requirements' , default = [ ] , nargs = '+' , help = 'List of pre-requirements to check, separated by space.' )
parser . add_argument ( '-e' , '--env' , help = 'Virtual environment name. By default: {0}' . format ( CONFIG [ __script__ ] [ 'env' ] ) )
parser . add_argument ( '-r' , '--requirements' , help = 'Path to requirements file. By default: {0}' . format ( CONFIG [ __script__ ] [ 'requirements' ] ) )
parser . add_argument ( '-d' , '--install-dev-requirements' , action = 'store_true' , default = None , help = 'Install prefixed or suffixed "dev" requirements after ' 'installation of original requirements file or library completed ' 'without errors.' )
parser . add_argument ( '-C' , '--hook' , help = 'Execute this hook after bootstrap process.' )
parser . add_argument ( '--ignore-activated' , action = 'store_true' , default = None , help = 'Ignore pre-activated virtualenv, like on Travis CI.' )
parser . add_argument ( '--recreate' , action = 'store_true' , default = None , help = 'Recreate virtualenv on every run.' )
parser . add_argument ( '-q' , '--quiet' , action = 'store_true' , default = None , help = 'Minimize output, show only error messages.' )
return parser . parse_args ( args )


signature:
def pip_cmd ( env , cmd , ignore_activated = False , ** kwargs ) :
    r"""Run pip command in given or activated virtual environment .
 : param env : Virtual environment name .
 : param cmd : Pip subcommand to run .
 : param ignore _ activated :
 Ignore activated virtual environment and use given venv instead . By
 default : False
 : param \ * \ * kwargs :
 Additional keyword arguments to be passed to : func : ` ~ run _ cmd `"""

implementation:
cmd = tuple ( cmd )
dirname = safe_path ( env )
if not ignore_activated :
    activated_env = os . environ . get ( 'VIRTUAL_ENV' )
    if hasattr ( sys , 'real_prefix' ) :
        dirname = sys . prefix
    elif activated_env :
        dirname = activated_env
pip_path = os . path . join ( dirname , 'Scripts' if IS_WINDOWS else 'bin' , 'pip' )
if kwargs . pop ( 'return_path' , False ) :
    return pip_path
if not os . path . isfile ( pip_path ) :
    raise OSError ( 'No pip found at {0!r}' . format ( pip_path ) )
# Disable pip version check in tests
if BOOTSTRAPPER_TEST_KEY in os . environ and cmd [ 0 ] == 'install' :
    cmd = list ( cmd )
    cmd . insert ( 1 , '--disable-pip-version-check' )
    cmd = tuple ( cmd )
with disable_error_handler ( ) :
    return run_cmd ( ( pip_path , ) + cmd , ** kwargs )


signature:
def prepare_args ( config , bootstrap ) :
    """Convert config dict to command line args line .
 : param config : Configuration dict .
 : param bootstrap : Bootstrapper configuration dict ."""

implementation:
config = copy . deepcopy ( config )
environ = dict ( copy . deepcopy ( os . environ ) )
data = { 'env' : bootstrap [ 'env' ] , 'pip' : pip_cmd ( bootstrap [ 'env' ] , '' , return_path = True ) , 'requirements' : bootstrap [ 'requirements' ] }
environ . update ( data )
if isinstance ( config , string_types ) :
    return config . format ( ** environ )
for key , value in iteritems ( config ) :
    if not isinstance ( value , string_types ) :
        continue
    config [ key ] = value . format ( ** environ )
return config_to_args ( config )


signature:
def print_error ( message , wrap = True ) :
    """Print error message to stderr , using ANSI - colors .
 : param message : Message to print
 : param wrap :
 Wrap message into ` ` ERROR : < message > . Exit . . . ` ` template . By default :
 True"""

implementation:
if wrap :
    message = 'ERROR: {0}. Exit...' . format ( message . rstrip ( '.' ) )
colorizer = ( _color_wrap ( colorama . Fore . RED ) if colorama else lambda message : message )
return print ( colorizer ( message ) , file = sys . stderr )


signature:
def print_message ( message = None ) :
    """Print message via ` ` subprocess . call ` ` function .
 This helps to ensure consistent output and avoid situations where print
 messages actually shown after messages from all inner threads .
 : param message : Text message to print ."""

implementation:
kwargs = { 'stdout' : sys . stdout , 'stderr' : sys . stderr , 'shell' : True }
return subprocess . call ( 'echo "{0}"' . format ( message or '' ) , ** kwargs )


signature:
def read_config ( filename , args ) :
    """Read and parse configuration file . By default , ` ` filename ` ` is relative
 path to current work directory .
 If no config file found , default ` ` CONFIG ` ` would be used .
 : param filename : Read config from given filename .
 : param args : Parsed command line arguments ."""

implementation:
# Initial vars
config = defaultdict ( dict )
splitter = operator . methodcaller ( 'split' , ' ' )
converters = { __script__ : { 'env' : safe_path , 'pre_requirements' : splitter , } , 'pip' : { 'allow_external' : splitter , 'allow_unverified' : splitter , } }
default = copy . deepcopy ( CONFIG )
sections = set ( iterkeys ( default ) )
# Append download - cache for old pip versions
if int ( getattr ( pip , '__version__' , '1.x' ) . split ( '.' ) [ 0 ] ) < 6 :
    default [ 'pip' ] [ 'download_cache' ] = safe_path ( os . path . expanduser ( os . path . join ( '~' , '.{0}' . format ( __script__ ) , 'pip-cache' ) ) )
# Expand user and environ vars in config filename
is_default = filename == DEFAULT_CONFIG
filename = os . path . expandvars ( os . path . expanduser ( filename ) )
# Read config if it exists on disk
if not is_default and not os . path . isfile ( filename ) :
    print_error ( 'Config file does not exist at {0!r}' . format ( filename ) )
    return None
parser = ConfigParser ( )
try :
    parser . read ( filename )
except ConfigParserError :
    print_error ( 'Cannot parse config file at {0!r}' . format ( filename ) )
    return None
# Apply config for each possible section
for section in sections :
    if not parser . has_section ( section ) :
        continue
    items = parser . items ( section )
    # Make auto convert here for integers and boolean values
    for key , value in items :
        try :
            value = int ( value )
        except ( TypeError , ValueError ) :
            try :
                value = bool ( strtobool ( value ) )
            except ValueError :
                pass
        if section in converters and key in converters [ section ] :
            value = converters [ section ] [ key ] ( value )
        config [ section ] [ key ] = value
# Update config with default values if necessary
for section , data in iteritems ( default ) :
    if section not in config :
        config [ section ] = data
    else :
        for key , value in iteritems ( data ) :
            config [ section ] . setdefault ( key , value )
# Update bootstrap config from parsed args
keys = set ( ( 'env' , 'hook' , 'install_dev_requirements' , 'ignore_activated' , 'pre_requirements' , 'quiet' , 'recreate' , 'requirements' ) )
for key in keys :
    value = getattr ( args , key )
    config [ __script__ ] . setdefault ( key , value )
    if key == 'pre_requirements' and not value :
        continue
    if value is not None :
        config [ __script__ ] [ key ] = value
return config


signature:
def run_cmd ( cmd , echo = False , fail_silently = False , ** kwargs ) :
    r"""Call given command with ` ` subprocess . call ` ` function .
 : param cmd : Command to run .
 : type cmd : tuple or str
 : param echo :
 If enabled show command to call and its output in STDOUT , otherwise
 hide all output . By default : False
 : param fail _ silently : Do not raise exception on error . By default : False
 : param \ * \ * kwargs :
 Additional keyword arguments to be passed to ` ` subprocess . call ` `
 function . STDOUT and STDERR streams would be setup inside of function
 to ensure hiding command output in case of disabling ` ` echo ` ` ."""

implementation:
out , err = None , None
if echo :
    cmd_str = cmd if isinstance ( cmd , string_types ) else ' ' . join ( cmd )
    kwargs [ 'stdout' ] , kwargs [ 'stderr' ] = sys . stdout , sys . stderr
    print_message ( '$ {0}' . format ( cmd_str ) )
else :
    out , err = get_temp_streams ( )
    kwargs [ 'stdout' ] , kwargs [ 'stderr' ] = out , err
try :
    retcode = subprocess . call ( cmd , ** kwargs )
except subprocess . CalledProcessError as err :
    if fail_silently :
        return False
    print_error ( str ( err ) if IS_PY3 else unicode ( err ) )
    # noqa
finally :
    if out :
        out . close ( )
    if err :
        err . close ( )
if retcode and echo and not fail_silently :
    print_error ( 'Command {0!r} returned non-zero exit status {1}' . format ( cmd_str , retcode ) )
return retcode


signature:
def run_hook ( hook , config , quiet = False ) :
    """Run post - bootstrap hook if any .
 : param hook : Hook to run .
 : param config : Configuration dict .
 : param quiet : Do not output messages to STDOUT / STDERR . By default : False"""

implementation:
if not hook :
    return True
if not quiet :
    print_message ( '== Step 3. Run post-bootstrap hook ==' )
result = not run_cmd ( prepare_args ( hook , config ) , echo = not quiet , fail_silently = True , shell = True )
if not quiet :
    print_message ( )
return result


signature:
def save_traceback ( err ) :
    """Save error traceback to bootstrapper log file .
 : param err : Catched exception ."""

implementation:
# Store logs to ~ / . bootstrapper directory
dirname = safe_path ( os . path . expanduser ( os . path . join ( '~' , '.{0}' . format ( __script__ ) ) ) )
# But ensure that directory exists
if not os . path . isdir ( dirname ) :
    os . mkdir ( dirname )
# Now we ready to put traceback to log file
filename = os . path . join ( dirname , '{0}.log' . format ( __script__ ) )
with open ( filename , 'a+' ) as handler :
    traceback . print_exc ( file = handler )
# And show colorized message
message = ( 'User aborted workflow' if isinstance ( err , KeyboardInterrupt ) else 'Unexpected error catched' )
print_error ( message )
print_error ( 'Full log stored to {0}' . format ( filename ) , False )
return True


signature:
def smart_str ( value , encoding = 'utf-8' , errors = 'strict' ) :
    """Convert Python object to string .
 : param value : Python object to convert .
 : param encoding : Encoding to use if in Python 2 given object is unicode .
 : param errors : Errors mode to use if in Python 2 given object is unicode ."""

implementation:
if not IS_PY3 and isinstance ( value , unicode ) : # noqa
    return value . encode ( encoding , errors )
return str ( value )


signature:
def copy_w_ext ( srcfile , destdir , basename ) :
    """Copy ` srcfile ` in ` destdir ` with name ` basename + get _ extension ( srcfile ) ` .
 Add pluses to the destination path basename if a file with the same name already
 exists in ` destdir ` .
 Parameters
 srcfile : str
 destdir : str
 basename : str
 Returns
 dstpath : str"""

implementation:
ext = get_extension ( op . basename ( srcfile ) )
dstpath = op . join ( destdir , basename + ext )
return copy_w_plus ( srcfile , dstpath )


signature:
def copy_w_plus ( src , dst ) :
    """Copy file from ` src ` path to ` dst ` path . If ` dst ` already exists , will add ' + ' characters
 to the end of the basename without extension .
 Parameters
 src : str
 dst : str
 Returns
 dstpath : str"""

implementation:
dst_ext = get_extension ( dst )
dst_pre = remove_ext ( dst )
while op . exists ( dst_pre + dst_ext ) :
    dst_pre += '+'
shutil . copy ( src , dst_pre + dst_ext )
return dst_pre + dst_ext


signature:
def get_abspath ( folderpath ) :
    """Returns the absolute path of folderpath .
 If the path does not exist , will raise IOError ."""

implementation:
if not op . exists ( folderpath ) :
    raise FolderNotFound ( folderpath )
return op . abspath ( folderpath )


signature:
def get_extension ( filepath , check_if_exists = False , allowed_exts = ALLOWED_EXTS ) :
    """Return the extension of fpath .
 Parameters
 fpath : string
 File name or path
 check _ if _ exists : bool
 allowed _ exts : dict
 Dictionary of strings , where the key if the last part of a complex ( ' . ' separated ) extension
 and the value is the previous part .
 For example : for the ' . nii . gz ' extension I would have a dict as { ' . gz ' : [ ' . nii ' , ] }
 Returns
 str
 The extension of the file name or path"""

implementation:
if check_if_exists :
    if not op . exists ( filepath ) :
        raise IOError ( 'File not found: ' + filepath )
rest , ext = op . splitext ( filepath )
if ext in allowed_exts :
    alloweds = allowed_exts [ ext ]
    _ , ext2 = op . splitext ( rest )
    if ext2 in alloweds :
        ext = ext2 + ext
return ext


signature:
def add_extension_if_needed ( filepath , ext , check_if_exists = False ) :
    """Add the extension ext to fpath if it doesn ' t have it .
 Parameters
 filepath : str
 File name or path
 ext : str
 File extension
 check _ if _ exists : bool
 Returns
 File name or path with extension added , if needed ."""

implementation:
if not filepath . endswith ( ext ) :
    filepath += ext
if check_if_exists :
    if not op . exists ( filepath ) :
        raise IOError ( 'File not found: ' + filepath )
return filepath


signature:
def parse_subjects_list ( filepath , datadir = '' , split = ':' , labelsf = None ) :
    """Parses a file with a list of : < subject _ file > : < subject _ class _ label > .
 Parameters
 filepath : str
 Path to file with a list of : < subject _ file > : < subject _ class _ label > .
 Where ' : ' can be any split character
 datadir : str
 String to be path prefix of each line of the fname content ,
 only in case the lines are relative file paths .
 split : str
 Split character for each line
 labelsf : str
 Path to file with a list of the labels if it is not included in
 fname . It will overwrite the labels from fname .
 Returns
 [ labels , subjs ] where labels is a list of labels and subjs a list of
 filepaths"""

implementation:
labels = [ ]
subjs = [ ]
if datadir :
    datadir += op . sep
with open ( filepath , 'r' ) as f :
    for s in f :
        line = s . strip ( ) . split ( split )
        if len ( line ) == 2 :
            labels . append ( np . float ( line [ 1 ] ) )
            subjf = line [ 0 ] . strip ( )
        else :
            subjf = line . strip ( )
        if not op . isabs ( subjf ) :
            subjs . append ( datadir + subjf )
        else :
            subjs . append ( subjf )
if labelsf is not None :
    labels = np . loadtxt ( labelsf )
return [ labels , subjs ]


signature:
def create_subjects_file ( filelist , labels , output_file , split = ':' ) :
    """Creates a file where each line is < subject _ file > : < subject _ class _ label > .
 Parameters
 filelist : list of str
 List of filepaths
 labels : list of int , str or labels that can be transformed with str ( )
 List of labels
 output _ file : str
 Output file path
 split : str
 Split character for each line"""

implementation:
if len ( filelist ) != len ( labels ) :
    raise ValueError ( 'Expected `filelist` and `labels` to have the same length.' 'Got {} and {}.' . format ( len ( filelist ) , len ( labels ) ) )
lines = [ ]
for i , subj in enumerate ( filelist ) :
    lab = labels [ i ]
    line = subj + split + str ( lab )
    lines . append ( line )
lines = np . array ( lines )
np . savetxt ( output_file , lines , fmt = '%s' )


signature:
def join_path_to_filelist ( path , filelist ) :
    """Joins path to each line in filelist
 Parameters
 path : str
 filelist : list of str
 Returns
 list of filepaths"""

implementation: return [ op . join ( path , str ( item ) ) for item in filelist ]


signature:
def remove_all ( filelist , folder = '' ) :
    """Deletes all files in filelist
 Parameters
 filelist : list of str
 List of the file paths to be removed
 folder : str
 Path to be used as common directory for all file paths in filelist"""

implementation:
if not folder :
    for f in filelist :
        os . remove ( f )
else :
    for f in filelist :
        os . remove ( op . join ( folder , f ) )


signature:
def get_folder_subpath ( path , folder_depth ) :
    """Returns a folder path of path with depth given by folder _ dept :
 Parameters
 path : str
 folder _ depth : int > 0
 Returns
 A folder path
 Example
 > > > get _ folder _ subpath ( ' / home / user / mydoc / work / notes . txt ' , 3)
 > > > ' / home / user / mydoc '"""

implementation:
if path [ 0 ] == op . sep :
    folder_depth += 1
return op . sep . join ( path . split ( op . sep ) [ 0 : folder_depth ] )


signature:
def get_temp_dir ( prefix = None , basepath = None ) :
    """Uses tempfile to create a TemporaryDirectory using
 the default arguments .
 The folder is created using tempfile . mkdtemp ( ) function .
 Parameters
 prefix : str
 Name prefix for the temporary folder .
 basepath : str
 Directory where the new folder must be created .
 The default directory is chosen from a platform - dependent
 list , but the user of the application can control the
 directory location by setting the TMPDIR , TEMP or TMP
 environment variables .
 Returns
 folder object"""

implementation:
if basepath is None :
    return tempfile . TemporaryDirectory ( dir = basepath )
else :
    return tempfile . TemporaryDirectory ( prefix = prefix , dir = basepath )


signature:
def ux_file_len ( filepath ) :
    """Returns the length of the file using the ' wc ' GNU command
 Parameters
 filepath : str
 Returns
 float"""

implementation:
p = subprocess . Popen ( [ 'wc' , '-l' , filepath ] , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
result , err = p . communicate ( )
if p . returncode != 0 :
    raise IOError ( err )
l = result . strip ( )
l = int ( l . split ( ) [ 0 ] )
return l


signature:
def merge ( dict_1 , dict_2 ) :
    """Merge two dictionaries .
 Values that evaluate to true take priority over falsy values .
 ` dict _ 1 ` takes priority over ` dict _ 2 ` ."""

implementation: return dict ( ( str ( key ) , dict_1 . get ( key ) or dict_2 . get ( key ) ) for key in set ( dict_2 ) | set ( dict_1 ) )


signature:
def get_sys_path ( rcpath , app_name , section_name = None ) :
    """Return a folder path if it exists .
 First will check if it is an existing system path , if it is , will return it
 expanded and absoluted .
 If this fails will look for the rcpath variable in the app _ name rcfiles or
 exclusively within the given section _ name , if given .
 Parameters
 rcpath : str
 Existing folder path or variable name in app _ name rcfile with an
 existing one .
 section _ name : str
 Name of a section in the app _ name rcfile to look exclusively there for
 variable names .
 app _ name : str
 Name of the application to look for rcfile configuration files .
 Returns
 sys _ path : str
 A expanded absolute file or folder path if the path exists .
 Raises
 IOError if the proposed sys _ path does not exist ."""

implementation:
# first check if it is an existing path
if op . exists ( rcpath ) :
    return op . realpath ( op . expanduser ( rcpath ) )
# look for the rcfile
try :
    settings = rcfile ( app_name , section_name )
except :
    raise
# look for the variable within the rcfile configutarions
try :
    sys_path = op . expanduser ( settings [ rcpath ] )
except KeyError :
    raise IOError ( 'Could not find an existing variable with name {0} in' ' section {1} of {2}rc config setup. Maybe it is a ' ' folder that could not be found.' . format ( rcpath , section_name , app_name ) )
# found the variable , now check if it is an existing path
else :
    if not op . exists ( sys_path ) :
        raise IOError ( 'Could not find the path {3} indicated by the ' 'variable {0} in section {1} of {2}rc config ' 'setup.' . format ( rcpath , section_name , app_name , sys_path ) )
    # expand the path and return
    return op . realpath ( op . expanduser ( sys_path ) )


signature:
def rcfile ( appname , section = None , args = { } , strip_dashes = True ) :
    """Read environment variables and config files and return them merged with
 predefined list of arguments .
 Parameters
 appname : str
 Application name , used for config files and environment variable
 names .
 section : str
 Name of the section to be read . If this is not set : appname .
 args :
 arguments from command line ( optparse , docopt , etc ) .
 strip _ dashes : bool
 Strip dashes prefixing key names from args dict .
 Returns
 dict
 containing the merged variables of environment variables , config
 files and args .
 Raises
 IOError
 In case the return value is empty .
 Notes
 Environment variables are read if they start with appname in uppercase
 with underscore , for example :
 TEST _ VAR = 1
 Config files compatible with ConfigParser are read and the section name
 appname is read , example :
 [ appname ]
 var = 1
 We can also have host - dependent configuration values , which have
 priority over the default appname values .
 [ appname ]
 var = 1
 [ appname : mylinux ]
 var = 3
 For boolean flags do not try to use : ' True ' or ' False ' ,
 ' on ' or ' off ' ,
 '1 ' or ' 0 ' .
 Unless you are willing to parse this values by yourself .
 We recommend commenting the variables out with ' # ' if you want to set a
 flag to False and check if it is in the rcfile cfg dict , i . e . :
 flag _ value = ' flag _ variable ' in cfg
 Files are read from : / etc / appname / config ,
 / etc / appfilerc ,
 ~ / . config / appname / config ,
 ~ / . config / appname ,
 ~ / . appname / config ,
 ~ / . appnamerc ,
 appnamerc ,
 . appnamerc ,
 appnamerc file found in ' path ' folder variable in args ,
 . appnamerc file found in ' path ' folder variable in args ,
 file provided by ' config ' variable in args .
 Example
 args = rcfile ( _ _ name _ _ , docopt ( _ _ doc _ _ , version = _ _ version _ _ ) )"""

implementation:
if strip_dashes :
    for k in args . keys ( ) :
        args [ k . lstrip ( '-' ) ] = args . pop ( k )
environ = get_environment ( appname )
if section is None :
    section = appname
config = get_config ( appname , section , args . get ( 'config' , '' ) , args . get ( 'path' , '' ) )
config = merge ( merge ( args , config ) , environ )
if not config :
    raise IOError ( 'Could not find any rcfile for application ' '{}.' . format ( appname ) )
return config


signature:
def get_rcfile_section ( app_name , section_name ) :
    """Return the dictionary containing the rcfile section configuration
 variables .
 Parameters
 section _ name : str
 Name of the section in the rcfiles .
 app _ name : str
 Name of the application to look for its rcfiles .
 Returns
 settings : dict
 Dict with variable values"""

implementation:
try :
    settings = rcfile ( app_name , section_name )
except IOError :
    raise
except :
    raise KeyError ( 'Error looking for section {} in {} ' ' rcfiles.' . format ( section_name , app_name ) )
else :
    return settings


signature:
def get_rcfile_variable_value ( var_name , app_name , section_name = None ) :
    """Return the value of the variable in the section _ name section of the
 app _ name rc file .
 Parameters
 var _ name : str
 Name of the variable to be searched for .
 section _ name : str
 Name of the section in the rcfiles .
 app _ name : str
 Name of the application to look for its rcfiles .
 Returns
 var _ value : str
 The value of the variable with given var _ name ."""

implementation:
cfg = get_rcfile_section ( app_name , section_name )
if var_name in cfg :
    raise KeyError ( 'Option {} not found in {} ' 'section.' . format ( var_name , section_name ) )
return cfg [ var_name ]


signature:
def find_in_sections ( var_name , app_name ) :
    """Return the section and the value of the variable where the first
 var _ name is found in the app _ name rcfiles .
 Parameters
 var _ name : str
 Name of the variable to be searched for .
 app _ name : str
 Name of the application to look for its rcfiles .
 Returns
 section _ name : str
 Name of the section in the rcfiles where var _ name was first found .
 var _ value : str
 The value of the first variable with given var _ name ."""

implementation:
sections = get_sections ( app_name )
if not sections :
    raise ValueError ( 'No sections found in {} rcfiles.' . format ( app_name ) )
for s in sections :
    try :
        var_value = get_rcfile_variable_value ( var_name , section_name = s , app_name = app_name )
    except :
        pass
    else :
        return s , var_value
raise KeyError ( 'No variable {} has been found in {} ' 'rcfiles.' . format ( var_name , app_name ) )


signature:
def filter_list ( lst , pattern ) :
    """Filters the lst using pattern .
 If pattern starts with ' ( ' it will be considered a re regular expression ,
 otherwise it will use fnmatch filter .
 : param lst : list of strings
 : param pattern : string
 : return : list of strings
 Filtered list of strings"""

implementation:
if is_fnmatch_regex ( pattern ) and not is_regex ( pattern ) : # use fnmatch
    log . info ( 'Using fnmatch for {0}' . format ( pattern ) )
    filst = fnmatch . filter ( lst , pattern )
else : # use re
    log . info ( 'Using regex match for {0}' . format ( pattern ) )
    filst = match_list ( lst , pattern )
if filst :
    filst . sort ( )
return filst


signature:
def get_subdict ( adict , path , sep = os . sep ) :
    """Given a nested dictionary adict .
 This returns its childen just below the path .
 The path is a string composed of adict keys separated by sep .
 : param adict : nested dict
 : param path : str
 : param sep : str
 : return : dict or list or leaf of treemap"""

implementation: return reduce ( adict . __class__ . get , [ p for p in op . split ( sep ) if p ] , adict )


signature:
def get_dict_leaves ( data ) :
    """Given a nested dictionary , this returns all its leave elements in a list .
 : param adict :
 : return : list"""

implementation:
result = [ ]
if isinstance ( data , dict ) :
    for item in data . values ( ) :
        result . extend ( get_dict_leaves ( item ) )
elif isinstance ( data , list ) :
    result . extend ( data )
else :
    result . append ( data )
return result


signature:
def get_possible_paths ( base_path , path_regex ) :
    """Looks for path _ regex within base _ path . Each match is append
 in the returned list .
 path _ regex may contain subfolder structure .
 If any part of the folder structure is a
 : param base _ path : str
 : param path _ regex : str
 : return list of strings"""

implementation:
if not path_regex :
    return [ ]
if len ( path_regex ) < 1 :
    return [ ]
if path_regex [ 0 ] == os . sep :
    path_regex = path_regex [ 1 : ]
rest_files = ''
if os . sep in path_regex : # split by os . sep
    node_names = path_regex . partition ( os . sep )
    first_node = node_names [ 0 ]
    rest_nodes = node_names [ 2 ]
    folder_names = filter_list ( os . listdir ( base_path ) , first_node )
    for nom in folder_names :
        new_base = op . join ( base_path , nom )
        if op . isdir ( new_base ) :
            rest_files = get_possible_paths ( new_base , rest_nodes )
else :
    rest_files = filter_list ( os . listdir ( base_path ) , path_regex )
files = [ ]
if rest_files :
    files = [ op . join ( base_path , f ) for f in rest_files ]
return files


signature:
def create_folder ( dirpath , overwrite = False ) :
    """Will create dirpath folder . If dirpath already exists and overwrite is False ,
 will append a ' + ' suffix to dirpath until dirpath does not exist ."""

implementation:
if not overwrite :
    while op . exists ( dirpath ) :
        dirpath += '+'
os . makedirs ( dirpath , exist_ok = overwrite )
return dirpath


signature:
def _import_config ( filepath ) :
    """Imports filetree and root _ path variable values from the filepath .
 : param filepath :
 : return : root _ path and filetree"""

implementation:
if not op . isfile ( filepath ) :
    raise IOError ( 'Data config file not found. ' 'Got: {0}' . format ( filepath ) )
cfg = import_pyfile ( filepath )
if not hasattr ( cfg , 'root_path' ) :
    raise KeyError ( 'Config file root_path key not found.' )
if not hasattr ( cfg , 'filetree' ) :
    raise KeyError ( 'Config file filetree key not found.' )
return cfg . root_path , cfg . filetree


signature:
def remove_nodes ( self , pattern , adict ) :
    """Remove the nodes that match the pattern ."""

implementation:
mydict = self . _filetree if adict is None else adict
if isinstance ( mydict , dict ) :
    for nom in mydict . keys ( ) :
        if isinstance ( mydict [ nom ] , dict ) :
            matchs = filter_list ( mydict [ nom ] , pattern )
            for nom in matchs :
                mydict = self . remove_nodes ( pattern , mydict [ nom ] )
                mydict . pop ( nom )
        else :
            mydict [ nom ] = filter_list ( mydict [ nom ] , pattern )
else :
    matchs = set ( filter_list ( mydict , pattern ) )
    mydict = set ( mydict ) - matchs
return mydict


signature:
def count_node_match ( self , pattern , adict = None ) :
    """Return the number of nodes that match the pattern .
 : param pattern :
 : param adict :
 : return : int"""

implementation:
mydict = self . _filetree if adict is None else adict
k = 0
if isinstance ( mydict , dict ) :
    names = mydict . keys ( )
    k += len ( filter_list ( names , pattern ) )
    for nom in names :
        k += self . count_node_match ( pattern , mydict [ nom ] )
else :
    k = len ( filter_list ( mydict , pattern ) )
return k


signature:
def as_float_array ( X , copy = True , force_all_finite = True ) :
    """Converts an array - like to an array of floats
 The new dtype will be np . float32 or np . float64 , depending on the original
 type . The function can create a copy or modify the argument depending
 on the argument copy .
 Parameters
 X : { array - like , sparse matrix }
 copy : bool , optional
 If True , a copy of X will be created . If False , a copy may still be
 returned if X ' s dtype is not a floating point type .
 Returns
 XT : { array , sparse matrix }
 An array of type np . float"""

implementation:
if isinstance ( X , np . matrix ) or ( not isinstance ( X , np . ndarray ) and not sp . issparse ( X ) ) :
    return check_array ( X , [ 'csr' , 'csc' , 'coo' ] , dtype = np . float64 , copy = copy , force_all_finite = force_all_finite , ensure_2d = False )
elif sp . issparse ( X ) and X . dtype in [ np . float32 , np . float64 ] :
    return X . copy ( ) if copy else X
elif X . dtype in [ np . float32 , np . float64 ] : # is numpy array
    return X . copy ( 'F' if X . flags [ 'F_CONTIGUOUS' ] else 'C' ) if copy else X
else :
    return X . astype ( np . float32 if X . dtype == np . int32 else np . float64 )


signature:
def _num_samples ( x ) :
    """Return number of samples in array - like x ."""

implementation:
if not hasattr ( x , '__len__' ) and not hasattr ( x , 'shape' ) :
    if hasattr ( x , '__array__' ) :
        x = np . asarray ( x )
    else :
        raise TypeError ( "Expected sequence or array-like, got %r" % x )
return x . shape [ 0 ] if hasattr ( x , 'shape' ) else len ( x )


signature:
def check_consistent_length ( * arrays ) :
    """Check that all arrays have consistent first dimensions .
 Checks whether all objects in arrays have the same shape or length .
 Parameters
 arrays : list or tuple of input objects .
 Objects that will be checked for consistent length ."""

implementation:
uniques = np . unique ( [ _num_samples ( X ) for X in arrays if X is not None ] )
if len ( uniques ) > 1 :
    raise ValueError ( "Found arrays with inconsistent numbers of samples: %s" % str ( uniques ) )


signature:
def indexable ( * iterables ) :
    """Make arrays indexable for cross - validation .
 Checks consistent length , passes through None , and ensures that everything
 can be indexed by converting sparse matrices to csr and converting
 non - interable objects to arrays .
 Parameters
 iterables : lists , dataframes , arrays , sparse matrices
 List of objects to ensure sliceability ."""

implementation:
result = [ ]
for X in iterables :
    if sp . issparse ( X ) :
        result . append ( X . tocsr ( ) )
    elif hasattr ( X , "__getitem__" ) or hasattr ( X , "iloc" ) :
        result . append ( X )
    elif X is None :
        result . append ( X )
    else :
        result . append ( np . array ( X ) )
check_consistent_length ( * result )
return result


signature:
def check_array ( array , accept_sparse = None , dtype = None , order = None , copy = False , force_all_finite = True , ensure_2d = True , allow_nd = False ) :
    """Input validation on an array , list , sparse matrix or similar .
 By default , the input is converted to an at least 2nd numpy array .
 Parameters
 array : object
 Input object to check / convert .
 accept _ sparse : string , list of string or None ( default = None )
 String [ s ] representing allowed sparse matrix formats , such as ' csc ' ,
 ' csr ' , etc . None means that sparse matrix input will raise an error .
 If the input is sparse but not in the allowed format , it will be
 converted to the first listed format .
 order : ' F ' , ' C ' or None ( default = None )
 Whether an array will be forced to be fortran or c - style .
 copy : boolean ( default = False )
 Whether a forced copy will be triggered . If copy = False , a copy might
 be triggered by a conversion .
 force _ all _ finite : boolean ( default = True )
 Whether to raise an error on np . inf and np . nan in X .
 ensure _ 2d : boolean ( default = True )
 Whether to make X at least 2d .
 allow _ nd : boolean ( default = False )
 Whether to allow X . ndim > 2.
 Returns
 X _ converted : object
 The converted and validated X ."""

implementation:
if isinstance ( accept_sparse , str ) :
    accept_sparse = [ accept_sparse ]
if sp . issparse ( array ) :
    array = _ensure_sparse_format ( array , accept_sparse , dtype , order , copy , force_all_finite )
else :
    if ensure_2d :
        array = np . atleast_2d ( array )
    array = np . array ( array , dtype = dtype , order = order , copy = copy )
    if not allow_nd and array . ndim >= 3 :
        raise ValueError ( "Found array with dim %d. Expected <= 2" % array . ndim )
    if force_all_finite :
        _assert_all_finite ( array )
return array


signature:
def check_X_y ( X , y , accept_sparse = None , dtype = None , order = None , copy = False , force_all_finite = True , ensure_2d = True , allow_nd = False , multi_output = False ) :
    """Input validation for standard estimators .
 Checks X and y for consistent length , enforces X 2d and y 1d .
 Standard input checks are only applied to y . For multi - label y ,
 set multi _ ouput = True to allow 2d and sparse y .
 Parameters
 X : nd - array , list or sparse matrix
 Input data .
 y : nd - array , list or sparse matrix
 Labels .
 accept _ sparse : string , list of string or None ( default = None )
 String [ s ] representing allowed sparse matrix formats , such as ' csc ' ,
 ' csr ' , etc . None means that sparse matrix input will raise an error .
 If the input is sparse but not in the allowed format , it will be
 converted to the first listed format .
 order : ' F ' , ' C ' or None ( default = None )
 Whether an array will be forced to be fortran or c - style .
 copy : boolean ( default = False )
 Whether a forced copy will be triggered . If copy = False , a copy might
 be triggered by a conversion .
 force _ all _ finite : boolean ( default = True )
 Whether to raise an error on np . inf and np . nan in X .
 ensure _ 2d : boolean ( default = True )
 Whether to make X at least 2d .
 allow _ nd : boolean ( default = False )
 Whether to allow X . ndim > 2.
 Returns
 X _ converted : object
 The converted and validated X ."""

implementation:
X = check_array ( X , accept_sparse , dtype , order , copy , force_all_finite , ensure_2d , allow_nd )
if multi_output :
    y = check_array ( y , 'csr' , force_all_finite = True , ensure_2d = False )
else :
    y = column_or_1d ( y , warn = True )
    _assert_all_finite ( y )
check_consistent_length ( X , y )
return X , y


signature:
def column_or_1d ( y , warn = False ) :
    """Ravel column or 1d numpy array , else raises an error
 Parameters
 y : array - like
 Returns
 y : array"""

implementation:
shape = np . shape ( y )
if len ( shape ) == 1 :
    return np . ravel ( y )
if len ( shape ) == 2 and shape [ 1 ] == 1 :
    if warn :
        warnings . warn ( "A column-vector y was passed when a 1d array was" " expected. Please change the shape of y to " "(n_samples, ), for example using ravel()." , DataConversionWarning , stacklevel = 2 )
    return np . ravel ( y )
raise ValueError ( "bad input shape {0}" . format ( shape ) )


signature:
def warn_if_not_float ( X , estimator = 'This algorithm' ) :
    """Warning utility function to check that data type is floating point .
 Returns True if a warning was raised ( i . e . the input is not float ) and
 False otherwise , for easier input validation ."""

implementation:
if not isinstance ( estimator , str ) :
    estimator = estimator . __class__ . __name__
if X . dtype . kind != 'f' :
    warnings . warn ( "%s assumes floating point values as input, " "got %s" % ( estimator , X . dtype ) )
    return True
return False


signature:
def as_ndarray ( arr , copy = False , dtype = None , order = 'K' ) :
    """Convert an arbitrary array to numpy . ndarray .
 In the case of a memmap array , a copy is automatically made to break the
 link with the underlying file ( whatever the value of the " copy " keyword ) .
 The purpose of this function is mainly to get rid of memmap objects , but
 it can be used for other purposes . In particular , combining copying and
 casting can lead to performance improvements in some cases , by avoiding
 unnecessary copies .
 If not specified , input array order is preserved , in all cases , even when
 a copy is requested .
 Caveat : this function does not copy during bool to / from 1 - byte dtype
 conversions . This can lead to some surprising results in some rare cases .
 Example :
 a = numpy . asarray ( [ 0 , 1 , 2 ] , dtype = numpy . int8)
 b = as _ ndarray ( a , dtype = bool ) # array ( [ False , True , True ] , dtype = bool )
 c = as _ ndarray ( b , dtype = numpy . int8 ) # array ( [ 0 , 1 , 2 ] , dtype = numpy . int8)
 The usually expected result for the last line would be array ( [ 0 , 1 , 1 ] )
 because True evaluates to 1 . Since there is no copy made here , the original
 array is recovered .
 Parameters
 arr : array - like
 input array . Any value accepted by numpy . asarray is valid .
 copy : bool
 if True , force a copy of the array . Always True when arr is a memmap .
 dtype : any numpy dtype
 dtype of the returned array . Performing copy and type conversion at the
 same time can in some cases avoid an additional copy .
 order : string
 gives the order of the returned array .
 Valid values are : " C " , " F " , " A " , " K " , None .
 default is " K " . See ndarray . copy ( ) for more information .
 Returns
 ret : np . ndarray
 Numpy array containing the same data as arr , always of class
 numpy . ndarray , and with no link to any underlying file ."""

implementation:
if order not in ( 'C' , 'F' , 'A' , 'K' , None ) :
    raise ValueError ( "Invalid value for 'order': {}" . format ( str ( order ) ) )
if isinstance ( arr , np . memmap ) :
    if dtype is None :
        if order in ( 'K' , 'A' , None ) :
            ret = np . array ( np . asarray ( arr ) , copy = True )
        else :
            ret = np . array ( np . asarray ( arr ) , copy = True , order = order )
    else :
        if order in ( 'K' , 'A' , None ) : # always copy ( even when dtype does not change )
            ret = np . asarray ( arr ) . astype ( dtype )
        else : # load data from disk without changing order
        # Changing order while reading through a memmap is incredibly
        # inefficient .
            ret = _asarray ( np . array ( arr , copy = True ) , dtype = dtype , order = order )
elif isinstance ( arr , np . ndarray ) :
    ret = _asarray ( arr , dtype = dtype , order = order )
    # In the present cas , np . may _ share _ memory result is always reliable .
    if np . may_share_memory ( ret , arr ) and copy : # order - preserving copy
        ret = ret . T . copy ( ) . T if ret . flags [ 'F_CONTIGUOUS' ] else ret . copy ( )
elif isinstance ( arr , ( list , tuple ) ) :
    if order in ( "A" , "K" ) :
        ret = np . asarray ( arr , dtype = dtype )
    else :
        ret = np . asarray ( arr , dtype = dtype , order = order )
else :
    raise ValueError ( "Type not handled: {}" . format ( arr . __class__ ) )
return ret


signature:
def xfm_atlas_to_functional ( atlas_filepath , anatbrain_filepath , meanfunc_filepath , atlas2anat_nonlin_xfm_filepath , is_atlas2anat_inverted , anat2func_lin_xfm_filepath , atlasinanat_out_filepath , atlasinfunc_out_filepath , interp = 'nn' , rewrite = True , parallel = False ) :
    """Call FSL tools to apply transformations to a given atlas to a functional image .
 Given the transformation matrices .
 Parameters
 atlas _ filepath : str
 Path to the 3D atlas volume file .
 anatbrain _ filepath : str
 Path to the anatomical brain volume file ( skull - stripped and registered to the same space as the atlas ,
 e . g . , MNI ) .
 meanfunc _ filepath : str
 Path to the average functional image to be used as reference in the last applywarp step .
 atlas2anat _ nonlin _ xfm _ filepath : str
 Path to the atlas to anatomical brain linear transformation . mat file .
 If you have the inverse transformation , i . e . , anatomical brain to atlas , set is _ atlas2anat _ inverted to True .
 is _ atlas2anat _ inverted : bool
 If False will have to calculate the inverse atlas2anat transformation to apply the transformations .
 This step will be performed with FSL invwarp .
 anat2func _ lin _ xfm _ filepath : str
 Path to the anatomical to functional . mat linear transformation file .
 atlasinanat _ out _ filepath : str
 Path to output file which will contain the 3D atlas in the subject anatomical space .
 atlasinfunc _ out _ filepath : str
 Path to output file which will contain the 3D atlas in the subject functional space .
 verbose : bool
 If verbose will show DEBUG log info .
 rewrite : bool
 If True will re - run all the commands overwriting any existing file . Otherwise will check if
 each file exists and if it does won ' t run the command .
 parallel : bool
 If True will launch the commands using $ { FSLDIR } / fsl _ sub to use the cluster infrastructure you have setup
 with FSL ( SGE or HTCondor ) ."""

implementation:
if is_atlas2anat_inverted : # I already have the inverted fields I need
    anat_to_mni_nl_inv = atlas2anat_nonlin_xfm_filepath
else : # I am creating the inverted fields then . . . need output file path :
    output_dir = op . abspath ( op . dirname ( atlasinanat_out_filepath ) )
    ext = get_extension ( atlas2anat_nonlin_xfm_filepath )
    anat_to_mni_nl_inv = op . join ( output_dir , remove_ext ( op . basename ( atlas2anat_nonlin_xfm_filepath ) ) + '_inv' + ext )
# setup the commands to be called
invwarp_cmd = op . join ( '${FSLDIR}' , 'bin' , 'invwarp' )
applywarp_cmd = op . join ( '${FSLDIR}' , 'bin' , 'applywarp' )
fslsub_cmd = op . join ( '${FSLDIR}' , 'bin' , 'fsl_sub' )
# add fsl _ sub before the commands
if parallel :
    invwarp_cmd = fslsub_cmd + ' ' + invwarp_cmd
    applywarp_cmd = fslsub_cmd + ' ' + applywarp_cmd
# create the inverse fields
if rewrite or ( not is_atlas2anat_inverted and not op . exists ( anat_to_mni_nl_inv ) ) :
    log . debug ( 'Creating {}.\n' . format ( anat_to_mni_nl_inv ) )
    cmd = invwarp_cmd + ' '
    cmd += '-w {} ' . format ( atlas2anat_nonlin_xfm_filepath )
    cmd += '-o {} ' . format ( anat_to_mni_nl_inv )
    cmd += '-r {} ' . format ( anatbrain_filepath )
    log . debug ( 'Running {}' . format ( cmd ) )
    check_call ( cmd )
# transform the atlas to anatomical space
if rewrite or not op . exists ( atlasinanat_out_filepath ) :
    log . debug ( 'Creating {}.\n' . format ( atlasinanat_out_filepath ) )
    cmd = applywarp_cmd + ' '
    cmd += '--in={}     ' . format ( atlas_filepath )
    cmd += '--ref={}    ' . format ( anatbrain_filepath )
    cmd += '--warp={}   ' . format ( anat_to_mni_nl_inv )
    cmd += '--interp={} ' . format ( interp )
    cmd += '--out={}    ' . format ( atlasinanat_out_filepath )
    log . debug ( 'Running {}' . format ( cmd ) )
    check_call ( cmd )
# transform the atlas to functional space
if rewrite or not op . exists ( atlasinfunc_out_filepath ) :
    log . debug ( 'Creating {}.\n' . format ( atlasinfunc_out_filepath ) )
    cmd = applywarp_cmd + ' '
    cmd += '--in={}     ' . format ( atlasinanat_out_filepath )
    cmd += '--ref={}    ' . format ( meanfunc_filepath )
    cmd += '--premat={} ' . format ( anat2func_lin_xfm_filepath )
    cmd += '--interp={} ' . format ( interp )
    cmd += '--out={}    ' . format ( atlasinfunc_out_filepath )
    log . debug ( 'Running {}' . format ( cmd ) )
    check_call ( cmd )


signature:
def fwhm2sigma ( fwhm ) :
    """Convert a FWHM value to sigma in a Gaussian kernel .
 Parameters
 fwhm : float or numpy . array
 fwhm value or values
 Returns
 fwhm : float or numpy . array
 sigma values"""

implementation:
fwhm = np . asarray ( fwhm )
return fwhm / np . sqrt ( 8 * np . log ( 2 ) )


signature:
def sigma2fwhm ( sigma ) :
    """Convert a sigma in a Gaussian kernel to a FWHM value .
 Parameters
 sigma : float or numpy . array
 sigma value or values
 Returns
 fwhm : float or numpy . array
 fwhm values corresponding to ` sigma ` values"""

implementation:
sigma = np . asarray ( sigma )
return np . sqrt ( 8 * np . log ( 2 ) ) * sigma


signature:
def _smooth_data_array ( arr , affine , fwhm , copy = True ) :
    """Smooth images with a a Gaussian filter .
 Apply a Gaussian filter along the three first dimensions of arr .
 Parameters
 arr : numpy . ndarray
 3D or 4D array , with image number as last dimension .
 affine : numpy . ndarray
 Image affine transformation matrix for image .
 fwhm : scalar , numpy . ndarray
 Smoothing kernel size , as Full - Width at Half Maximum ( FWHM ) in millimeters .
 If a scalar is given , kernel width is identical on all three directions .
 A numpy . ndarray must have 3 elements , giving the FWHM along each axis .
 copy : bool
 if True , will make a copy of the input array . Otherwise will directly smooth the input array .
 Returns
 smooth _ arr : numpy . ndarray"""

implementation:
if arr . dtype . kind == 'i' :
    if arr . dtype == np . int64 :
        arr = arr . astype ( np . float64 )
    else :
        arr = arr . astype ( np . float32 )
if copy :
    arr = arr . copy ( )
# Zeroe possible NaNs and Inf in the image .
arr [ np . logical_not ( np . isfinite ( arr ) ) ] = 0
try : # Keep the 3D part of the affine .
    affine = affine [ : 3 , : 3 ]
    # Convert from FWHM in mm to a sigma .
    fwhm_sigma_ratio = np . sqrt ( 8 * np . log ( 2 ) )
    vox_size = np . sqrt ( np . sum ( affine ** 2 , axis = 0 ) )
    sigma = fwhm / ( fwhm_sigma_ratio * vox_size )
    for n , s in enumerate ( sigma ) :
        ndimage . gaussian_filter1d ( arr , s , output = arr , axis = n )
except :
    raise ValueError ( 'Error smoothing the array.' )
else :
    return arr


signature:
def smooth_imgs ( images , fwhm ) :
    """Smooth images using a Gaussian filter .
 Apply a Gaussian filter along the three first dimensions of each image in images .
 In all cases , non - finite values in input are zeroed .
 Parameters
 imgs : str or img - like object or iterable of img - like objects
 See boyle . nifti . read . read _ img
 Image ( s ) to smooth .
 fwhm : scalar or numpy . ndarray
 Smoothing kernel size , as Full - Width at Half Maximum ( FWHM ) in millimeters .
 If a scalar is given , kernel width is identical on all three directions .
 A numpy . ndarray must have 3 elements , giving the FWHM along each axis .
 Returns
 smooth _ imgs : nibabel . Nifti1Image or list of .
 Smooth input image / s ."""

implementation:
if fwhm <= 0 :
    return images
if not isinstance ( images , string_types ) and hasattr ( images , '__iter__' ) :
    only_one = False
else :
    only_one = True
    images = [ images ]
result = [ ]
for img in images :
    img = check_img ( img )
    affine = img . get_affine ( )
    smooth = _smooth_data_array ( img . get_data ( ) , affine , fwhm = fwhm , copy = True )
    result . append ( nib . Nifti1Image ( smooth , affine ) )
if only_one :
    return result [ 0 ]
else :
    return result


signature:
def _smooth_array ( arr , affine , fwhm = None , ensure_finite = True , copy = True , ** kwargs ) :
    """Smooth images by applying a Gaussian filter .
 Apply a Gaussian filter along the three first dimensions of arr .
 This is copied and slightly modified from nilearn :
 https : / / github . com / nilearn / nilearn / blob / master / nilearn / image / image . py
 Added the * * kwargs argument .
 Parameters
 arr : numpy . ndarray
 4D array , with image number as last dimension . 3D arrays are also
 accepted .
 affine : numpy . ndarray
 (4 , 4 ) matrix , giving affine transformation for image . ( 3 , 3 ) matrices
 are also accepted ( only these coefficients are used ) .
 If fwhm = ' fast ' , the affine is not used and can be None
 fwhm : scalar , numpy . ndarray , ' fast ' or None
 Smoothing strength , as a full - width at half maximum , in millimeters .
 If a scalar is given , width is identical on all three directions .
 A numpy . ndarray must have 3 elements , giving the FWHM along each axis .
 If fwhm = = ' fast ' , a fast smoothing will be performed with
 a filter [ 0.2 , 1 , 0.2 ] in each direction and a normalisation
 to preserve the local average value .
 If fwhm is None , no filtering is performed ( useful when just removal
 of non - finite values is needed ) .
 ensure _ finite : bool
 if True , replace every non - finite values ( like NaNs ) by zero before
 filtering .
 copy : bool
 if True , input array is not modified . False by default : the filtering
 is performed in - place .
 kwargs : keyword - arguments
 Arguments for the ndimage . gaussian _ filter1d function .
 Returns
 filtered _ arr : numpy . ndarray
 arr , filtered .
 Notes
 This function is most efficient with arr in C order ."""

implementation:
if arr . dtype . kind == 'i' :
    if arr . dtype == np . int64 :
        arr = arr . astype ( np . float64 )
    else : # We don ' t need crazy precision
        arr = arr . astype ( np . float32 )
if copy :
    arr = arr . copy ( )
if ensure_finite : # SPM tends to put NaNs in the data outside the brain
    arr [ np . logical_not ( np . isfinite ( arr ) ) ] = 0
if fwhm == 'fast' :
    arr = _fast_smooth_array ( arr )
elif fwhm is not None : # Keep only the scale part .
    affine = affine [ : 3 , : 3 ]
    # Convert from a FWHM to a sigma :
    fwhm_over_sigma_ratio = np . sqrt ( 8 * np . log ( 2 ) )
    vox_size = np . sqrt ( np . sum ( affine ** 2 , axis = 0 ) )
    sigma = fwhm / ( fwhm_over_sigma_ratio * vox_size )
    for n , s in enumerate ( sigma ) :
        ndimage . gaussian_filter1d ( arr , s , output = arr , axis = n , ** kwargs )
return arr


signature:
def smooth_img ( imgs , fwhm , ** kwargs ) :
    """Smooth images by applying a Gaussian filter .
 Apply a Gaussian filter along the three first dimensions of arr .
 In all cases , non - finite values in input image are replaced by zeros .
 This is copied and slightly modified from nilearn :
 https : / / github . com / nilearn / nilearn / blob / master / nilearn / image / image . py
 Added the * * kwargs argument .
 Parameters
 imgs : Niimg - like object or iterable of Niimg - like objects
 See http : / / nilearn . github . io / manipulating _ images / manipulating _ images . html # niimg .
 Image ( s ) to smooth .
 fwhm : scalar , numpy . ndarray , ' fast ' or None
 Smoothing strength , as a Full - Width at Half Maximum , in millimeters .
 If a scalar is given , width is identical on all three directions .
 A numpy . ndarray must have 3 elements , giving the FWHM along each axis .
 If fwhm = = ' fast ' , a fast smoothing will be performed with
 a filter [ 0.2 , 1 , 0.2 ] in each direction and a normalisation
 to preserve the scale .
 If fwhm is None , no filtering is performed ( useful when just removal
 of non - finite values is needed )
 Returns
 filtered _ img : nibabel . Nifti1Image or list of .
 Input image , filtered . If imgs is an iterable , then filtered _ img is a
 list ."""

implementation:
# Use hasattr ( ) instead of isinstance to workaround a Python 2.6/2.7 bug
# See http : / / bugs . python . org / issue7624
if hasattr ( imgs , "__iter__" ) and not isinstance ( imgs , string_types ) :
    single_img = False
else :
    single_img = True
    imgs = [ imgs ]
ret = [ ]
for img in imgs :
    img = check_niimg ( img )
    affine = img . get_affine ( )
    filtered = _smooth_array ( img . get_data ( ) , affine , fwhm = fwhm , ensure_finite = True , copy = True , ** kwargs )
    ret . append ( new_img_like ( img , filtered , affine , copy_header = True ) )
if single_img :
    return ret [ 0 ]
else :
    return ret


signature:
def signed_session ( self , session = None ) :
    """Create requests session with any required auth headers
 applied .
 : rtype : requests . Session ."""

implementation:
if session :
    session = super ( ClientCertAuthentication , self ) . signed_session ( session )
else :
    session = super ( ClientCertAuthentication , self ) . signed_session ( )
if self . cert is not None :
    session . cert = self . cert
if self . ca_cert is not None :
    session . verify = self . ca_cert
if self . no_verify :
    session . verify = False
return session


signature:
def signed_session ( self , session = None ) :
    """Create requests session with AAD auth headers
 : rtype : requests . Session ."""

implementation:
from sfctl . config import ( aad_metadata , aad_cache )
if session :
    session = super ( AdalAuthentication , self ) . signed_session ( session )
else :
    session = super ( AdalAuthentication , self ) . signed_session ( )
if self . no_verify :
    session . verify = False
authority_uri , cluster_id , client_id = aad_metadata ( )
existing_token , existing_cache = aad_cache ( )
context = adal . AuthenticationContext ( authority_uri , cache = existing_cache )
new_token = context . acquire_token ( cluster_id , existing_token [ 'userId' ] , client_id )
header = "{} {}" . format ( "Bearer" , new_token [ 'accessToken' ] )
session . headers [ 'Authorization' ] = header
return session


signature:
def voxspace_to_mmspace ( img ) :
    """Return a grid with coordinates in 3D physical space for ` img ` ."""

implementation:
shape , affine = img . shape [ : 3 ] , img . affine
coords = np . array ( np . meshgrid ( * ( range ( i ) for i in shape ) , indexing = 'ij' ) )
coords = np . rollaxis ( coords , 0 , len ( shape ) + 1 )
mm_coords = nib . affines . apply_affine ( affine , coords )
return mm_coords


signature:
def voxcoord_to_mm ( cm , i , j , k ) :
    '''Parameters
 cm : nipy . core . reference . coordinate _ map . CoordinateMap
 i , j , k : floats
 Voxel coordinates
 Returns
 Triplet with real 3D world coordinates'''

implementation:
try :
    mm = cm ( [ i , j , k ] )
except Exception as exc :
    raise Exception ( 'Error on converting coordinates.' ) from exc
else :
    return mm


signature:
def mm_to_voxcoord ( cm , x , y , z ) :
    '''Parameters
 cm : nipy . core . reference . coordinate _ map . CoordinateMap
 x , y , z : floats
 Physical coordinates
 Returns
 Triplet with 3D voxel coordinates'''

implementation:
try :
    vox = cm . inverse ( ) ( [ x , y , z ] )
except Exception as exc :
    raise Exception ( 'Error on converting coordinates' ) from exc
else :
    return vox


signature:
def get_3D_coordmap ( img ) :
    '''Gets a 3D CoordinateMap from img .
 Parameters
 img : nib . Nifti1Image or nipy Image
 Returns
 nipy . core . reference . coordinate _ map . CoordinateMap'''

implementation:
if isinstance ( img , nib . Nifti1Image ) :
    img = nifti2nipy ( img )
if img . ndim == 4 :
    from nipy . core . reference . coordinate_map import drop_io_dim
    cm = drop_io_dim ( img . coordmap , 3 )
else :
    cm = img . coordmap
return cm


signature:
def get_img_info ( image ) :
    """Return the header and affine matrix from a Nifti file .
 Parameters
 image : img - like object or str
 Can either be :
 - a file path to a Nifti image
 - any object with get _ data ( ) and get _ affine ( ) methods , e . g . , nibabel . Nifti1Image .
 If niimg is a string , consider it as a path to Nifti image and
 call nibabel . load on it . If it is an object , check if get _ data ( )
 and get _ affine ( ) methods are present , raise TypeError otherwise .
 Returns
 hdr , aff"""

implementation:
try :
    img = check_img ( image )
except Exception as exc :
    raise Exception ( 'Error reading file {0}.' . format ( repr_imgs ( image ) ) ) from exc
else :
    return img . get_header ( ) , img . get_affine ( )


signature:
def get_img_data ( image , copy = True ) :
    """Return the voxel matrix of the Nifti file .
 If safe _ mode will make a copy of the img before returning the data , so the input image is not modified .
 Parameters
 image : img - like object or str
 Can either be :
 - a file path to a Nifti image
 - any object with get _ data ( ) and get _ affine ( ) methods , e . g . , nibabel . Nifti1Image .
 If niimg is a string , consider it as a path to Nifti image and
 call nibabel . load on it . If it is an object , check if get _ data ( )
 and get _ affine ( ) methods are present , raise TypeError otherwise .
 copy : bool
 If safe _ mode will make a copy of the img before returning the data , so the input image is not modified .
 Returns
 array _ like"""

implementation:
try :
    img = check_img ( image )
    if copy :
        return get_data ( img )
    else :
        return img . get_data ( )
except Exception as exc :
    raise Exception ( 'Error when reading file {0}.' . format ( repr_imgs ( image ) ) ) from exc


signature:
def load_nipy_img ( nii_file ) :
    """Read a Nifti file and return as nipy . Image
 Parameters
 param nii _ file : str
 Nifti file path
 Returns
 nipy . Image"""

implementation:
# delayed import because could not install nipy on Python 3 on OSX
import nipy
if not os . path . exists ( nii_file ) :
    raise FileNotFound ( nii_file )
try :
    return nipy . load_image ( nii_file )
except Exception as exc :
    raise Exception ( 'Reading file {0}.' . format ( repr_imgs ( nii_file ) ) ) from exc


signature:
def niftilist_to_array ( img_filelist , outdtype = None ) :
    """From the list of absolute paths to nifti files , creates a Numpy array
 with the data .
 Parameters
 img _ filelist : list of str
 List of absolute file paths to nifti files . All nifti files must have
 the same shape .
 outdtype : dtype
 Type of the elements of the array , if not set will obtain the dtype from
 the first nifti file .
 Returns
 outmat : Numpy array with shape N x prod ( vol . shape )
 containing the N files as flat vectors .
 vol _ shape : Tuple with shape of the volumes , for reshaping ."""

implementation:
try :
    first_img = img_filelist [ 0 ]
    vol = get_img_data ( first_img )
except IndexError as ie :
    raise Exception ( 'Error getting the first item of img_filelis: {}' . format ( repr_imgs ( img_filelist [ 0 ] ) ) ) from ie
if not outdtype :
    outdtype = vol . dtype
outmat = np . zeros ( ( len ( img_filelist ) , np . prod ( vol . shape ) ) , dtype = outdtype )
try :
    for i , img_file in enumerate ( img_filelist ) :
        vol = get_img_data ( img_file )
        outmat [ i , : ] = vol . flatten ( )
except Exception as exc :
    raise Exception ( 'Error on reading file {0}.' . format ( img_file ) ) from exc
return outmat , vol . shape