signature:
def get_trip_stop_coordinates ( self , trip_I ) :
    """Get coordinates for a given trip _ I
 Parameters
 trip _ I : int
 the integer id of the trip
 Returns
 stop _ coords : pandas . DataFrame
 with columns " lats " and " lons " """

implementation:
query = """SELECT lat, lon
                    FROM stop_times
                    JOIN stops
                    USING(stop_I)
                        WHERE trip_I={trip_I}
                    ORDER BY stop_times.seq""" . format ( trip_I = trip_I )
stop_coords = pd . read_sql ( query , self . conn )
return stop_coords


signature:
def get_trip_stop_time_data ( self , trip_I , day_start_ut ) :
    """Obtain from the ( standard ) GTFS database , trip stop data
 ( departure time in ut , lat , lon , seq , shape _ break ) as a pandas DataFrame
 Some filtering could be applied here , if only e . g . departure times
 corresponding within some time interval should be considered .
 Parameters
 trip _ I : int
 integer index of the trip
 day _ start _ ut : int
 the start time of the day in unix time ( seconds )
 Returns
 df : pandas . DataFrame
 df has the following columns
 ' departure _ time _ ut , lat , lon , seq , shape _ break '"""

implementation:
to_select = "stop_I, " + str ( day_start_ut ) + "+dep_time_ds AS dep_time_ut, lat, lon, seq, shape_break"
str_to_run = "SELECT " + to_select + """
                        FROM stop_times JOIN stops USING(stop_I)
                        WHERE (trip_I ={trip_I}) ORDER BY seq
                      """
str_to_run = str_to_run . format ( trip_I = trip_I )
return pd . read_sql_query ( str_to_run , self . conn )


signature:
def get_events_by_tripI_and_dsut ( self , trip_I , day_start_ut , start_ut = None , end_ut = None ) :
    """Get trip data as a list of events ( i . e . dicts ) .
 Parameters
 trip _ I : int
 shorthand index of the trip .
 day _ start _ ut : int
 the start time of the day in unix time ( seconds )
 start _ ut : int , optional
 consider only events that start after this time
 If not specified , this filtering is not applied .
 end _ ut : int , optional
 Consider only events that end before this time
 If not specified , this filtering is not applied .
 Returns
 events : list of dicts
 each element contains the following data :
 from _ stop : int ( stop _ I )
 to _ stop : int ( stop _ I )
 dep _ time _ ut : int ( in unix time )
 arr _ time _ ut : int ( in unix time )"""

implementation:
# for checking input :
assert day_start_ut <= start_ut
assert day_start_ut <= end_ut
assert start_ut <= end_ut
events = [ ]
# check that trip takes place on that day :
if not self . tripI_takes_place_on_dsut ( trip_I , day_start_ut ) :
    return events
query = """SELECT stop_I, arr_time_ds+?, dep_time_ds+?
                    FROM stop_times JOIN stops USING(stop_I)
                    WHERE
                        (trip_I = ?)
                """
params = [ day_start_ut , day_start_ut , trip_I ]
if start_ut :
    query += "AND (dep_time_ds > ?-?)"
    params += [ start_ut , day_start_ut ]
if end_ut :
    query += "AND (arr_time_ds < ?-?)"
    params += [ end_ut , day_start_ut ]
query += "ORDER BY arr_time_ds"
cur = self . conn . cursor ( )
rows = cur . execute ( query , params )
stop_data = list ( rows )
for i in range ( len ( stop_data ) - 1 ) :
    event = { "from_stop" : stop_data [ i ] [ 0 ] , "to_stop" : stop_data [ i + 1 ] [ 0 ] , "dep_time_ut" : stop_data [ i ] [ 2 ] , "arr_time_ut" : stop_data [ i + 1 ] [ 1 ] }
    events . append ( event )
return events


signature:
def tripI_takes_place_on_dsut ( self , trip_I , day_start_ut ) :
    """Check that a trip takes place during a day
 Parameters
 trip _ I : int
 index of the trip in the gtfs data base
 day _ start _ ut : int
 the starting time of the day in unix time ( seconds )
 Returns
 takes _ place : bool
 boolean value describing whether the trip takes place during
 the given day or not"""

implementation:
query = "SELECT * FROM days WHERE trip_I=? AND day_start_ut=?"
params = ( trip_I , day_start_ut )
cur = self . conn . cursor ( )
rows = list ( cur . execute ( query , params ) )
if len ( rows ) == 0 :
    return False
else :
    assert len ( rows ) == 1 , 'On a day, a trip_I should be present at most once'
    return True


signature:
def day_start_ut ( self , ut ) :
    """Convert unixtime to unixtime on GTFS start - of - day .
 GTFS defines the start of a day as " noon minus 12 hours " to solve
 most DST - related problems . This means that on DST - changing days ,
 the day start isn ' t midnight . This function isn ' t idempotent .
 Running it twice on the " move clocks backwards " day will result in
 being one day too early .
 Parameters
 ut : int
 Unixtime
 Returns
 ut : int
 Unixtime corresponding to start of day"""

implementation:
# set timezone to the one of gtfs
old_tz = self . set_current_process_time_zone ( )
ut = time . mktime ( time . localtime ( ut ) [ : 3 ] + ( 12 , 00 , 0 , 0 , 0 , - 1 ) ) - 43200
set_process_timezone ( old_tz )
return ut


signature:
def increment_day_start_ut ( self , day_start_ut , n_days = 1 ) :
    """Increment the GTFS - definition of " day start " .
 Parameters
 day _ start _ ut : int
 unixtime of the previous start of day . If this time is between
 12:00 or greater , there * will * be bugs . To solve this , run the
 input through day _ start _ ut first .
 n _ days : int
 number of days to increment"""

implementation:
old_tz = self . set_current_process_time_zone ( )
day0 = time . localtime ( day_start_ut + 43200 )
# time of noon
dayN = time . mktime ( day0 [ : 2 ] + # YYYY , MM
( day0 [ 2 ] + n_days , ) + # DD
( 12 , 00 , 0 , 0 , 0 , - 1 ) ) - 43200
# HHMM , etc . Minus 12 hours .
set_process_timezone ( old_tz )
return dayN


signature:
def _get_possible_day_starts ( self , start_ut , end_ut , max_time_overnight = None ) :
    """Get all possible day start times between start _ ut and end _ ut
 Currently this function is used only by get _ tripIs _ within _ range _ by _ dsut
 Parameters
 start _ ut : list < int >
 start time in unix time
 end _ ut : list < int >
 end time in unix time
 max _ time _ overnight : list < int >
 the maximum length of time that a trip can take place on
 during the next day ( i . e . after midnight run times like 25:35)
 Returns
 day _ start _ times _ ut : list
 list of ints ( unix times in seconds ) for returning all possible day
 start times
 start _ times _ ds : list
 list of ints ( unix times in seconds ) stating the valid start time in
 day seconds
 end _ times _ ds : list
 list of ints ( unix times in seconds ) stating the valid end times in
 day _ seconds"""

implementation:
if max_time_overnight is None : # 7 hours :
    max_time_overnight = 7 * 60 * 60
# sanity checks for the timezone parameter
# assert timezone < 14
# assert timezone > - 14
# tz _ seconds = int ( timezone * 3600)
assert start_ut < end_ut
start_day_ut = self . day_start_ut ( start_ut )
# start _ day _ ds = int ( start _ ut + tz _ seconds ) % seconds _ in _ a _ day # ? ? ? needed ?
start_day_ds = start_ut - start_day_ut
# assert ( start _ day _ ut + tz _ seconds ) % seconds _ in _ a _ day = = 0
end_day_ut = self . day_start_ut ( end_ut )
# end _ day _ ds = int ( end _ ut + tz _ seconds ) % seconds _ in _ a _ day # ? ? ? needed ?
# end _ day _ ds = end _ ut - end _ day _ ut
# assert ( end _ day _ ut + tz _ seconds ) % seconds _ in _ a _ day = = 0
# If we are early enough in a day that we might have trips from
# the previous day still running , decrement the start day .
if start_day_ds < max_time_overnight :
    start_day_ut = self . increment_day_start_ut ( start_day_ut , n_days = - 1 )
# day _ start _ times _ ut = range ( start _ day _ ut , end _ day _ ut + seconds _ in _ a _ day , seconds _ in _ a _ day )
# Create a list of all possible day start times . This is roughly
# range ( day _ start _ ut , day _ end _ ut + 1day , 1day ) .
day_start_times_ut = [ start_day_ut ]
while day_start_times_ut [ - 1 ] < end_day_ut :
    day_start_times_ut . append ( self . increment_day_start_ut ( day_start_times_ut [ - 1 ] ) )
start_times_ds = [ ]
end_times_ds = [ ]
# For every possible day start :
for dsut in day_start_times_ut : # start day _ seconds starts at either zero , or time - daystart
    day_start_ut = max ( 0 , start_ut - dsut )
    start_times_ds . append ( day_start_ut )
    # end day _ seconds is time - day _ start
    day_end_ut = end_ut - dsut
    end_times_ds . append ( day_end_ut )
# Return three tuples which can be zip : ped together .
return day_start_times_ut , start_times_ds , end_times_ds


signature:
def get_tripIs_within_range_by_dsut ( self , start_time_ut , end_time_ut ) :
    """Obtain a list of trip _ Is that take place during a time interval .
 The trip needs to be only partially overlapping with the given time interval .
 The grouping by dsut ( day _ start _ ut ) is required as same trip _ I could
 take place on multiple days .
 Parameters
 start _ time _ ut : int
 start of the time interval in unix time ( seconds )
 end _ time _ ut : int
 end of the time interval in unix time ( seconds )
 Returns
 trip _ I _ dict : dict
 keys : day _ start _ times to list of integers ( trip _ Is )"""

implementation:
cur = self . conn . cursor ( )
assert start_time_ut <= end_time_ut
dst_ut , st_ds , et_ds = self . _get_possible_day_starts ( start_time_ut , end_time_ut , 7 )
# noinspection PyTypeChecker
assert len ( dst_ut ) >= 0
trip_I_dict = { }
for day_start_ut , start_ds , end_ds in zip ( dst_ut , st_ds , et_ds ) :
    query = """
                        SELECT distinct(trip_I)
                        FROM days
                            JOIN trips
                            USING(trip_I)
                        WHERE
                            (days.day_start_ut == ?)
                            AND (
                                    (trips.start_time_ds <= ?)
                                    AND
                                    (trips.end_time_ds >= ?)
                                )
                        """
    params = ( day_start_ut , end_ds , start_ds )
    trip_Is = [ el [ 0 ] for el in cur . execute ( query , params ) ]
    if len ( trip_Is ) > 0 :
        trip_I_dict [ day_start_ut ] = trip_Is
return trip_I_dict


signature:
def stop ( self , stop_I ) :
    """Get all stop data as a pandas DataFrame for all stops , or an individual stop '
 Parameters
 stop _ I : int
 stop index
 Returns
 stop : pandas . DataFrame"""

implementation: return pd . read_sql_query ( "SELECT * FROM stops WHERE stop_I={stop_I}" . format ( stop_I = stop_I ) , self . conn )


signature:
def get_stops_for_route_type ( self , route_type ) :
    """Parameters
 route _ type : int
 Returns
 stops : pandas . DataFrame"""

implementation:
if route_type is WALK :
    return self . stops ( )
else :
    return pd . read_sql_query ( "SELECT DISTINCT stops.* " "FROM stops JOIN stop_times ON stops.stop_I == stop_times.stop_I " "           JOIN trips ON stop_times.trip_I = trips.trip_I" "           JOIN routes ON trips.route_I == routes.route_I " "WHERE routes.type=(?)" , self . conn , params = ( route_type , ) )


signature:
def generate_routable_transit_events ( self , start_time_ut = None , end_time_ut = None , route_type = None ) :
    """Generates events that take place during a time interval [ start _ time _ ut , end _ time _ ut ] .
 Each event needs to be only partially overlap the given time interval .
 Does not include walking events .
 This is just a quick and dirty implementation to get a way of quickly get a
 method for generating events compatible with the routing algorithm
 Parameters
 start _ time _ ut : int
 end _ time _ ut : int
 route _ type : ?
 Yields
 event : namedtuple
 containing :
 dep _ time _ ut : int
 arr _ time _ ut : int
 from _ stop _ I : int
 to _ stop _ I : int
 trip _ I : int
 route _ type : int
 seq : int"""

implementation:
from gtfspy . networks import temporal_network
df = temporal_network ( self , start_time_ut = start_time_ut , end_time_ut = end_time_ut , route_type = route_type )
df . sort_values ( "dep_time_ut" , ascending = False , inplace = True )
for row in df . itertuples ( ) :
    yield row


signature:
def get_transit_events ( self , start_time_ut = None , end_time_ut = None , route_type = None ) :
    """Obtain a list of events that take place during a time interval .
 Each event needs to be only partially overlap the given time interval .
 Does not include walking events .
 Parameters
 start _ time _ ut : int
 start of the time interval in unix time ( seconds )
 end _ time _ ut : int
 end of the time interval in unix time ( seconds )
 route _ type : int
 consider only events for this route _ type
 Returns
 events : pandas . DataFrame
 with the following columns and types
 dep _ time _ ut : int
 arr _ time _ ut : int
 from _ stop _ I : int
 to _ stop _ I : int
 trip _ I : int
 shape _ id : int
 route _ type : int
 See also
 get _ transit _ events _ in _ time _ span : an older version of the same thing"""

implementation:
table_name = self . _get_day_trips_table_name ( )
event_query = "SELECT stop_I, seq, trip_I, route_I, routes.route_id AS route_id, routes.type AS route_type, " "shape_id, day_start_ut+dep_time_ds AS dep_time_ut, day_start_ut+arr_time_ds AS arr_time_ut " "FROM " + table_name + " " "JOIN trips USING(trip_I) " "JOIN routes USING(route_I) " "JOIN stop_times USING(trip_I)"
where_clauses = [ ]
if end_time_ut :
    where_clauses . append ( table_name + ".start_time_ut< {end_time_ut}" . format ( end_time_ut = end_time_ut ) )
    where_clauses . append ( "dep_time_ut  <={end_time_ut}" . format ( end_time_ut = end_time_ut ) )
if start_time_ut :
    where_clauses . append ( table_name + ".end_time_ut  > {start_time_ut}" . format ( start_time_ut = start_time_ut ) )
    where_clauses . append ( "arr_time_ut  >={start_time_ut}" . format ( start_time_ut = start_time_ut ) )
if route_type is not None :
    assert route_type in ALL_ROUTE_TYPES
    where_clauses . append ( "routes.type={route_type}" . format ( route_type = route_type ) )
if len ( where_clauses ) > 0 :
    event_query += " WHERE "
    for i , where_clause in enumerate ( where_clauses ) :
        if i is not 0 :
            event_query += " AND "
        event_query += where_clause
# ordering is required for later stages
event_query += " ORDER BY trip_I, day_start_ut+dep_time_ds;"
events_result = pd . read_sql_query ( event_query , self . conn )
# ' filter ' results so that only real " events " are taken into account
from_indices = numpy . nonzero ( ( events_result [ 'trip_I' ] [ : - 1 ] . values == events_result [ 'trip_I' ] [ 1 : ] . values ) * ( events_result [ 'seq' ] [ : - 1 ] . values < events_result [ 'seq' ] [ 1 : ] . values ) ) [ 0 ]
to_indices = from_indices + 1
# these should have same trip _ ids
assert ( events_result [ 'trip_I' ] [ from_indices ] . values == events_result [ 'trip_I' ] [ to_indices ] . values ) . all ( )
trip_Is = events_result [ 'trip_I' ] [ from_indices ]
from_stops = events_result [ 'stop_I' ] [ from_indices ]
to_stops = events_result [ 'stop_I' ] [ to_indices ]
shape_ids = events_result [ 'shape_id' ] [ from_indices ]
dep_times = events_result [ 'dep_time_ut' ] [ from_indices ]
arr_times = events_result [ 'arr_time_ut' ] [ to_indices ]
route_types = events_result [ 'route_type' ] [ from_indices ]
route_ids = events_result [ 'route_id' ] [ from_indices ]
route_Is = events_result [ 'route_I' ] [ from_indices ]
durations = arr_times . values - dep_times . values
assert ( durations >= 0 ) . all ( )
from_seqs = events_result [ 'seq' ] [ from_indices ]
to_seqs = events_result [ 'seq' ] [ to_indices ]
data_tuples = zip ( from_stops , to_stops , dep_times , arr_times , shape_ids , route_types , route_ids , trip_Is , durations , from_seqs , to_seqs , route_Is )
columns = [ "from_stop_I" , "to_stop_I" , "dep_time_ut" , "arr_time_ut" , "shape_id" , "route_type" , "route_id" , "trip_I" , "duration" , "from_seq" , "to_seq" , "route_I" ]
df = pd . DataFrame . from_records ( data_tuples , columns = columns )
return df


signature:
def get_route_difference_with_other_db ( self , other_gtfs , start_time , end_time , uniqueness_threshold = None , uniqueness_ratio = None ) :
    """Compares the routes based on stops in the schedule with the routes in another db and returns the ones without match .
 Uniqueness thresholds or ratio can be used to allow small differences
 : param uniqueness _ threshold :
 : param uniqueness _ ratio :
 : return :"""

implementation:
from gtfspy . stats import frequencies_by_generated_route
this_df = frequencies_by_generated_route ( self , start_time , end_time )
other_df = frequencies_by_generated_route ( other_gtfs , start_time , end_time )
this_routes = { x : set ( x . split ( ',' ) ) for x in this_df [ "route" ] }
other_routes = { x : set ( x . split ( ',' ) ) for x in other_df [ "route" ] }
# this _ df [ " route _ set " ] = this _ df . apply ( lambda x : set ( x . route . split ( ' , ' ) ) , axis = 1)
# other _ df [ " route _ set " ] = other _ df . apply ( lambda x : set ( x . route . split ( ' , ' ) ) , axis = 1)
this_uniques = list ( this_routes . keys ( ) )
other_uniques = list ( other_routes . keys ( ) )
print ( "initial routes A:" , len ( this_uniques ) )
print ( "initial routes B:" , len ( other_uniques ) )
for i_key , i in this_routes . items ( ) :
    for j_key , j in other_routes . items ( ) :
        union = i | j
        intersection = i & j
        symmetric_difference = i ^ j
        if uniqueness_ratio :
            if len ( intersection ) / len ( union ) >= uniqueness_ratio :
                try :
                    this_uniques . remove ( i_key )
                    this_df = this_df [ this_df [ "route" ] != i_key ]
                except ValueError :
                    pass
                try :
                    other_uniques . remove ( j_key )
                    other_df = other_df [ other_df [ "route" ] != j_key ]
                except ValueError :
                    pass
print ( "unique routes A" , len ( this_df ) )
print ( "unique routes B" , len ( other_df ) )
return this_df , other_df


signature:
def get_straight_line_transfer_distances ( self , stop_I = None ) :
    """Get ( straight line ) distances to stations that can be transferred to .
 Parameters
 stop _ I : int , optional
 If not specified return all possible transfer distances
 Returns
 distances : pandas . DataFrame
 each row has the following items
 from _ stop _ I : int
 to _ stop _ I : int
 d : float or int # distance in meters"""

implementation:
if stop_I is not None :
    query = u""" SELECT from_stop_I, to_stop_I, d
                        FROM stop_distances
                            WHERE
                                from_stop_I=?
                    """
    params = ( u"{stop_I}" . format ( stop_I = stop_I ) , )
else :
    query = """ SELECT from_stop_I, to_stop_I, d
                        FROM stop_distances
                    """
    params = None
stop_data_df = pd . read_sql_query ( query , self . conn , params = params )
return stop_data_df


signature:
def get_day_start_ut_span ( self ) :
    """Return the first and last day _ start _ ut
 Returns
 first _ day _ start _ ut : int
 last _ day _ start _ ut : int"""

implementation:
cur = self . conn . cursor ( )
first_day_start_ut , last_day_start_ut = cur . execute ( "SELECT min(day_start_ut), max(day_start_ut) FROM days;" ) . fetchone ( )
return first_day_start_ut , last_day_start_ut


signature:
def homogenize_stops_table_with_other_db ( self , source ) :
    """This function takes an external database , looks of common stops and adds the missing stops to both databases .
 In addition the stop _ pair _ I column is added . This id links the stops between these two sources .
 : param source : directory of external database
 : return :"""

implementation:
cur = self . conn . cursor ( )
self . attach_gtfs_database ( source )
query_inner_join = """SELECT t1.*
                              FROM stops t1
                              INNER JOIN other.stops t2
                              ON t1.stop_id=t2.stop_id
                              AND find_distance(t1.lon, t1.lat, t2.lon, t2.lat) <= 50"""
df_inner_join = self . execute_custom_query_pandas ( query_inner_join )
print ( "number of common stops: " , len ( df_inner_join . index ) )
df_not_in_other = self . execute_custom_query_pandas ( "SELECT * FROM stops EXCEPT " + query_inner_join )
print ( "number of stops missing in second feed: " , len ( df_not_in_other . index ) )
df_not_in_self = self . execute_custom_query_pandas ( "SELECT * FROM other.stops EXCEPT " + query_inner_join . replace ( "t1.*" , "t2.*" ) )
print ( "number of stops missing in first feed: " , len ( df_not_in_self . index ) )
try :
    self . execute_custom_query ( """ALTER TABLE stops ADD COLUMN stop_pair_I INT """ )
    self . execute_custom_query ( """ALTER TABLE other.stops ADD COLUMN stop_pair_I INT """ )
except sqlite3 . OperationalError :
    pass
stop_id_stub = "added_stop_"
counter = 0
rows_to_update_self = [ ]
rows_to_update_other = [ ]
rows_to_add_to_self = [ ]
rows_to_add_to_other = [ ]
for items in df_inner_join . itertuples ( index = False ) :
    rows_to_update_self . append ( ( counter , items [ 1 ] ) )
    rows_to_update_other . append ( ( counter , items [ 1 ] ) )
    counter += 1
for items in df_not_in_other . itertuples ( index = False ) :
    rows_to_update_self . append ( ( counter , items [ 1 ] ) )
    rows_to_add_to_other . append ( ( stop_id_stub + str ( counter ) , ) + tuple ( items [ x ] for x in [ 2 , 3 , 4 , 5 , 6 , 8 , 9 ] ) + ( counter , ) )
    counter += 1
for items in df_not_in_self . itertuples ( index = False ) :
    rows_to_update_other . append ( ( counter , items [ 1 ] ) )
    rows_to_add_to_self . append ( ( stop_id_stub + str ( counter ) , ) + tuple ( items [ x ] for x in [ 2 , 3 , 4 , 5 , 6 , 8 , 9 ] ) + ( counter , ) )
    counter += 1
query_add_row = """INSERT INTO stops(
                                    stop_id,
                                    code,
                                    name,
                                    desc,
                                    lat,
                                    lon,
                                    location_type,
                                    wheelchair_boarding,
                                    stop_pair_I) VALUES (%s) """ % ( ", " . join ( [ "?" for x in range ( 9 ) ] ) )
query_update_row = """UPDATE stops SET stop_pair_I=? WHERE stop_id=?"""
print ( "adding rows to databases" )
cur . executemany ( query_add_row , rows_to_add_to_self )
cur . executemany ( query_update_row , rows_to_update_self )
cur . executemany ( query_add_row . replace ( "stops" , "other.stops" ) , rows_to_add_to_other )
cur . executemany ( query_update_row . replace ( "stops" , "other.stops" ) , rows_to_update_other )
self . conn . commit ( )
print ( "finished" )


signature:
def read_data_as_dataframe ( self , travel_impedance_measure , from_stop_I = None , to_stop_I = None , statistic = None ) :
    """Recover pre - computed travel _ impedance between od - pairs from the database .
 Returns
 values : number | Pandas DataFrame"""

implementation:
to_select = [ ]
where_clauses = [ ]
to_select . append ( "from_stop_I" )
to_select . append ( "to_stop_I" )
if from_stop_I is not None :
    where_clauses . append ( "from_stop_I=" + str ( int ( from_stop_I ) ) )
if to_stop_I is not None :
    where_clauses . append ( "to_stop_I=" + str ( int ( to_stop_I ) ) )
where_clause = ""
if len ( where_clauses ) > 0 :
    where_clause = " WHERE " + " AND " . join ( where_clauses )
if not statistic :
    to_select . extend ( [ "min" , "mean" , "median" , "max" ] )
else :
    to_select . append ( statistic )
to_select_clause = "," . join ( to_select )
if not to_select_clause :
    to_select_clause = "*"
sql = "SELECT " + to_select_clause + " FROM " + travel_impedance_measure + where_clause + ";"
df = pd . read_sql ( sql , self . conn )
return df


signature:
def insert_data ( self , travel_impedance_measure_name , data ) :
    """Parameters
 travel _ impedance _ measure _ name : str
 data : list [ dict ]
 Each list element must contain keys :
 " from _ stop _ I " , " to _ stop _ I " , " min " , " max " , " median " and " mean " """

implementation:
f = float
data_tuple = [ ( int ( x [ "from_stop_I" ] ) , int ( x [ "to_stop_I" ] ) , f ( x [ "min" ] ) , f ( x [ "max" ] ) , f ( x [ "median" ] ) , f ( x [ "mean" ] ) ) for x in data ]
insert_stmt = '''INSERT OR REPLACE INTO ''' + travel_impedance_measure_name + ''' (
                              from_stop_I,
                              to_stop_I,
                              min,
                              max,
                              median,
                              mean) VALUES (?, ?, ?, ?, ?, ?) '''
self . conn . executemany ( insert_stmt , data_tuple )
self . conn . commit ( )


signature:
def add_event ( self , event ) :
    """Add an event to the heap / priority queue
 Parameters
 event : Event"""

implementation:
assert event . dep_time_ut <= event . arr_time_ut
heappush ( self . heap , event )


signature:
def add_walk_events_to_heap ( self , transfer_distances , e , start_time_ut , walk_speed , uninfected_stops , max_duration_ut ) :
    """Parameters
 transfer _ distances :
 e : Event
 start _ time _ ut : int
 walk _ speed : float
 uninfected _ stops : list
 max _ duration _ ut : int"""

implementation:
n = len ( transfer_distances )
dists_values = transfer_distances . values
to_stop_I_index = np . nonzero ( transfer_distances . columns == 'to_stop_I' ) [ 0 ] [ 0 ]
d_index = np . nonzero ( transfer_distances . columns == 'd' ) [ 0 ] [ 0 ]
for i in range ( n ) :
    transfer_to_stop_I = dists_values [ i , to_stop_I_index ]
    if transfer_to_stop_I in uninfected_stops :
        d = dists_values [ i , d_index ]
        transfer_arr_time = e . arr_time_ut + int ( d / float ( walk_speed ) )
        if transfer_arr_time > start_time_ut + max_duration_ut :
            continue
        te = Event ( transfer_arr_time , e . arr_time_ut , e . to_stop_I , transfer_to_stop_I , WALK )
        self . add_event ( te )


signature:
def _check_dep_time_is_valid ( self , dep_time ) :
    """A simple checker , that connections are coming in descending order of departure time
 and that no departure time has been " skipped " .
 Parameters
 dep _ time
 Returns
 None"""

implementation:
assert dep_time <= self . _min_dep_time , "Labels should be entered in decreasing order of departure time."
dep_time_index = self . dep_times_to_index [ dep_time ]
if self . _min_dep_time < float ( 'inf' ) :
    min_dep_index = self . dep_times_to_index [ self . _min_dep_time ]
    assert min_dep_index == dep_time_index or ( min_dep_index == dep_time_index - 1 ) , "dep times should be ordered sequentially"
else :
    assert dep_time_index is 0 , "first dep_time index should be zero (ensuring that all connections are properly handled)"
self . _min_dep_time = dep_time


signature:
def update ( self , new_labels , departure_time_backup = None ) :
    """Update the profile with the new labels .
 Each new label should have the same departure _ time .
 Parameters
 new _ labels : list [ LabelTime ]
 Returns
 added : bool
 whether new _ pareto _ tuple was added to the set of pareto - optimal tuples"""

implementation:
if self . _closed :
    raise RuntimeError ( "Profile is closed, no updates can be made" )
try :
    departure_time = next ( iter ( new_labels ) ) . departure_time
except StopIteration :
    departure_time = departure_time_backup
self . _check_dep_time_is_valid ( departure_time )
for new_label in new_labels :
    assert ( new_label . departure_time == departure_time )
dep_time_index = self . dep_times_to_index [ departure_time ]
if dep_time_index > 0 : # Departure time is modified in order to not pass on labels which are not Pareto - optimal when departure time is ignored .
    mod_prev_labels = [ label . get_copy_with_specified_departure_time ( departure_time ) for label in self . _label_bags [ dep_time_index - 1 ] ]
else :
    mod_prev_labels = list ( )
mod_prev_labels += self . _label_bags [ dep_time_index ]
walk_label = self . _get_label_to_target ( departure_time )
if walk_label :
    new_labels = new_labels + [ walk_label ]
new_frontier = merge_pareto_frontiers ( new_labels , mod_prev_labels )
self . _label_bags [ dep_time_index ] = new_frontier
return True


signature:
def evaluate ( self , dep_time , first_leg_can_be_walk = True , connection_arrival_time = None ) :
    """Get the pareto _ optimal set of Labels , given a departure time .
 Parameters
 dep _ time : float , int
 time in unix seconds
 first _ leg _ can _ be _ walk : bool , optional
 whether to allow walking to target to be included into the profile
 ( I . e . whether this function is called when scanning a pseudo - connection :
 " double " walks are not allowed . )
 connection _ arrival _ time : float , int , optional
 used for computing the walking label if dep _ time , i . e . , connection . arrival _ stop _ next _ departure _ time , is infinity )
 connection : connection object
 Returns
 pareto _ optimal _ labels : set
 Set of Labels"""

implementation:
walk_labels = list ( )
# walk label towards target
if first_leg_can_be_walk and self . _walk_to_target_duration != float ( 'inf' ) : # add walk _ label
    if connection_arrival_time is not None :
        walk_labels . append ( self . _get_label_to_target ( connection_arrival_time ) )
    else :
        walk_labels . append ( self . _get_label_to_target ( dep_time ) )
# if dep time is larger than the largest dep time - > only walk labels are possible
if dep_time in self . dep_times_to_index :
    assert ( dep_time != float ( 'inf' ) )
    index = self . dep_times_to_index [ dep_time ]
    labels = self . _label_bags [ index ]
    pareto_optimal_labels = merge_pareto_frontiers ( labels , walk_labels )
else :
    pareto_optimal_labels = walk_labels
if not first_leg_can_be_walk :
    pareto_optimal_labels = [ label for label in pareto_optimal_labels if not label . first_leg_is_walk ]
return pareto_optimal_labels


signature:
def finalize ( self , neighbor_label_bags = None , walk_durations = None , departure_arrival_stop_pairs = None ) :
    """Parameters
 neighbor _ label _ bags : list
 each list element is a list of labels corresponding to a neighboring node
 ( note : only labels with first connection being a departure should be included )
 walk _ durations : list
 departure _ arrival _ stop _ pairs : list of tuples
 Returns
 None"""

implementation:
assert ( not self . _finalized )
if self . _final_pareto_optimal_labels is None :
    self . _compute_real_connection_labels ( )
if neighbor_label_bags is not None :
    assert ( len ( walk_durations ) == len ( neighbor_label_bags ) )
    self . _compute_final_pareto_optimal_labels ( neighbor_label_bags , walk_durations , departure_arrival_stop_pairs )
else :
    self . _final_pareto_optimal_labels = self . _real_connection_labels
self . _finalized = True
self . _closed = True


signature:
def exists_by_source ( self ) :
    """Does this GTFS contain this file ? ( file specified by the class )"""

implementation:
exists_list = [ ]
for source in self . gtfs_sources :
    if isinstance ( source , dict ) : # source can now be either a dict or a zipfile
        if self . fname in source :
            if source [ self . fname ] :
                exists_list . append ( True )
                continue
    # Handle zipfiles specially
    if "zipfile" in source :
        try :
            Z = zipfile . ZipFile ( source [ 'zipfile' ] , mode = 'r' )
            Z . getinfo ( os . path . join ( source [ 'zip_commonprefix' ] , self . fname ) )
            exists_list . append ( True )
            continue
        # File does not exist in the zip archive
        except KeyError :
            print ( self . fname , ' missing in ' , source )
            exists_list . append ( False )
            continue
    # Normal filename
    elif isinstance ( source , string_types ) :
        if os . path . exists ( os . path . join ( source , self . fname ) ) :
            exists_list . append ( True )
            continue
    exists_list . append ( False )
# the " file " was not found in any of the sources , return false
return exists_list


signature:
def create_table ( self , conn ) :
    """Make table definitions"""

implementation:
# Make cursor
cur = conn . cursor ( )
# Drop table if it already exists , to be recreated . This
# could in the future abort if table already exists , and not
# recreate it from scratch .
# cur . execute ( ' ' ' DROP TABLE IF EXISTS % s ' ' ' % self . table )
# conn . commit ( )
if self . tabledef is None :
    return
if not self . tabledef . startswith ( 'CREATE' ) : # " normal " table creation .
    cur . execute ( 'CREATE TABLE IF NOT EXISTS %s %s' % ( self . table , self . tabledef ) )
else : # When tabledef contains the full CREATE statement ( for
# virtual tables ) .
    cur . execute ( self . tabledef )
conn . commit ( )


signature:
def insert_data ( self , conn ) :
    """Load data from GTFS file into database"""

implementation:
cur = conn . cursor ( )
# This is a bit hackish . It is annoying to have to write the
# INSERT statement yourself and keep it up to date with the
# table rows . This gets the first row , figures out the field
# names from that , and then makes an INSERT statement like
# " INSERT INTO table ( col1 , col2 , . . . ) VALUES ( : col1 , : col2,
# . . . ) " . The " : col1 " is sqlite syntax for named value .
csv_reader_generators , prefixes = self . _get_csv_reader_generators ( )
for csv_reader , prefix in zip ( csv_reader_generators , prefixes ) :
    try :
        row = next ( iter ( self . gen_rows ( [ csv_reader ] , [ prefix ] ) ) )
        fields = row . keys ( )
    except StopIteration : # The file has * only * a header and no data .
    # next ( iter ( ) ) yields StopIteration and we can ' t
    # proceed . Since there is nothing to import , just continue the loop
        print ( "Not importing %s into %s for %s" % ( self . fname , self . table , prefix ) )
        continue
    stmt = '''INSERT INTO %s (%s) VALUES (%s)''' % ( self . table , ( ', ' . join ( [ x for x in fields if x [ 0 ] != '_' ] + self . extra_keys ) ) , ( ', ' . join ( [ ":" + x for x in fields if x [ 0 ] != '_' ] + self . extra_values ) ) )
    # This does the actual insertions . Passed the INSERT
    # statement and then an iterator over dictionaries . Each
    # dictionary is inserted .
    if self . print_progress :
        print ( 'Importing %s into %s for %s' % ( self . fname , self . table , prefix ) )
    # the first row was consumed by fetching the fields
    # ( this could be optimized )
    from itertools import chain
    rows = chain ( [ row ] , self . gen_rows ( [ csv_reader ] , [ prefix ] ) )
    cur . executemany ( stmt , rows )
    conn . commit ( )


signature:
def import_ ( self , conn ) :
    """Do the actual import . Copy data and store in connection object .
 This function :
 - Creates the tables
 - Imports data ( using self . gen _ rows )
 - Run any post _ import hooks .
 - Creates any indexs
 - Does * not * run self . make _ views - those must be done
 after all tables are loaded ."""

implementation:
if self . print_progress :
    print ( 'Beginning' , self . __class__ . __name__ )
# what is this mystical self . _ conn ?
self . _conn = conn
self . create_table ( conn )
# This does insertions
if self . mode in ( 'all' , 'import' ) and self . fname and self . exists ( ) and self . table not in ignore_tables :
    self . insert_data ( conn )
# This makes indexes in the DB .
if self . mode in ( 'all' , 'index' ) and hasattr ( self , 'index' ) :
    self . create_index ( conn )
# Any post - processing to be done after the full import .
if self . mode in ( 'all' , 'import' ) and hasattr ( self , 'post_import' ) :
    self . run_post_import ( conn )
# Commit it all
conn . commit ( )


signature:
def copy ( cls , conn , ** where ) :
    """Copy data from one table to another while filtering data at the same time
 Parameters
 conn : sqlite3 DB connection . It must have a second database
 attached as " other " .
 * * where : keyword arguments
 specifying ( start _ ut and end _ ut for filtering , see the copy _ where clause in the subclasses )"""

implementation:
cur = conn . cursor ( )
if where and cls . copy_where :
    copy_where = cls . copy_where . format ( ** where )
    # print ( copy _ where )
else :
    copy_where = ''
cur . execute ( 'INSERT INTO %s ' 'SELECT * FROM source.%s %s' % ( cls . table , cls . table , copy_where ) )


signature:
def get_journey_legs_to_target ( self , target , fastest_path = True , min_boardings = False , all_leg_sections = True , ignore_walk = False , diff_threshold = None , diff_path = None ) :
    """Returns a dataframe of aggregated sections from source nodes to target . The returned sections are either
 transfer point to transfer point or stop to stop . In a before after setting , the results can be filtered based
 on values in a difference db .
 : param target :
 : param fastest _ path :
 : param min _ boardings :
 : param all _ leg _ sections :
 : param ignore _ walk :
 : param diff _ threshold :
 : param diff _ path :
 : return :"""

implementation:
assert not ( fastest_path and min_boardings )
if min_boardings :
    raise NotImplementedError
if all_leg_sections and diff_threshold :
    raise NotImplementedError
added_constraints = ""
add_diff = ""
if fastest_path :
    added_constraints += " AND journeys.pre_journey_wait_fp>=0"
if ignore_walk :
    added_constraints += " AND legs.trip_I >= 0"
if diff_path and diff_threshold :
    self . conn = attach_database ( self . conn , diff_path , name = "diff" )
    add_diff = ", diff.diff_temporal_distance"
    added_constraints += " AND abs(diff_temporal_distance.diff_mean) >= %s " "AND diff_temporal_distance.from_stop_I = journeys.from_stop_I " "AND diff_temporal_distance.to_stop_I = journeys.to_stop_I" % ( diff_threshold , )
if all_leg_sections :
    df = self . _get_journey_legs_to_target_with_all_sections ( target , added_constraints )
else :
    query = """SELECT from_stop_I, to_stop_I, coalesce(type, -1) AS type,
                         count(*) AS n_trips
                         FROM
                         (SELECT legs.* FROM legs, journeys %s
                         WHERE journeys.journey_id = legs.journey_id AND journeys.to_stop_I = %s %s) q1
                         LEFT JOIN (SELECT * FROM other.trips, other.routes WHERE trips.route_I = routes.route_I) q2
                         ON q1.trip_I = q2.trip_I
                         GROUP BY from_stop_I, to_stop_I, type""" % ( add_diff , str ( target ) , added_constraints )
    df = read_sql_query ( query , self . conn )
return df


signature:
def get_upstream_stops_ratio ( self , target , trough_stops , ratio ) :
    """Selects the stops for which the ratio or higher proportion of trips to the target passes trough a set of trough stops
 : param target : target of trips
 : param trough _ stops : stops where the selected trips are passing trough
 : param ratio : threshold for inclusion
 : return :"""

implementation:
if isinstance ( trough_stops , list ) :
    trough_stops = "," . join ( trough_stops )
query = """SELECT stops.* FROM other.stops,
                    (SELECT q2.from_stop_I AS stop_I FROM
                    (SELECT journeys.from_stop_I, count(*) AS n_total FROM journeys
                    WHERE journeys.to_stop_I = {target}
                    GROUP BY from_stop_I) q1,
                    (SELECT journeys.from_stop_I, count(*) AS n_trough FROM journeys, legs
                    WHERE journeys.journey_id=legs.journey_id AND legs.from_stop_I IN ({trough_stops}) AND journeys.to_stop_I = {target}
                    GROUP BY journeys.from_stop_I) q2
                    WHERE q1.from_stop_I = q2.from_stop_I AND n_trough/(n_total*1.0) >= {ratio}) q1
                    WHERE stops.stop_I = q1.stop_I""" . format ( target = target , trough_stops = trough_stops , ratio = ratio )
df = read_sql_query ( query , self . conn )
return df


signature:
def get_spatial_bounds ( gtfs , as_dict = False ) :
    """Parameters
 gtfs
 Returns
 min _ lon : float
 max _ lon : float
 min _ lat : float
 max _ lat : float"""

implementation:
stats = get_stats ( gtfs )
lon_min = stats [ 'lon_min' ]
lon_max = stats [ 'lon_max' ]
lat_min = stats [ 'lat_min' ]
lat_max = stats [ 'lat_max' ]
if as_dict :
    return { 'lon_min' : lon_min , 'lon_max' : lon_max , 'lat_min' : lat_min , 'lat_max' : lat_max }
else :
    return lon_min , lon_max , lat_min , lat_max


signature:
def get_median_lat_lon_of_stops ( gtfs ) :
    """Get median latitude AND longitude of stops
 Parameters
 gtfs : GTFS
 Returns
 median _ lat : float
 median _ lon : float"""

implementation:
stops = gtfs . get_table ( "stops" )
median_lat = numpy . percentile ( stops [ 'lat' ] . values , 50 )
median_lon = numpy . percentile ( stops [ 'lon' ] . values , 50 )
return median_lat , median_lon


signature:
def get_centroid_of_stops ( gtfs ) :
    """Get mean latitude AND longitude of stops
 Parameters
 gtfs : GTFS
 Returns
 mean _ lat : float
 mean _ lon : float"""

implementation:
stops = gtfs . get_table ( "stops" )
mean_lat = numpy . mean ( stops [ 'lat' ] . values )
mean_lon = numpy . mean ( stops [ 'lon' ] . values )
return mean_lat , mean_lon


signature:
def write_stats_as_csv ( gtfs , path_to_csv , re_write = False ) :
    """Writes data from get _ stats to csv file
 Parameters
 gtfs : GTFS
 path _ to _ csv : str
 filepath to the csv file to be generated
 re _ write :
 insted of appending , create a new one ."""

implementation:
stats_dict = get_stats ( gtfs )
# check if file exist
if re_write :
    os . remove ( path_to_csv )
# if not os . path . isfile ( path _ to _ csv ) :
# is _ new = True
# else :
# is _ new = False
is_new = True
mode = 'r' if os . path . exists ( path_to_csv ) else 'w+'
with open ( path_to_csv , mode ) as csvfile :
    for line in csvfile :
        if line :
            is_new = False
        else :
            is_new = True
with open ( path_to_csv , 'a' ) as csvfile :
    if ( sys . version_info > ( 3 , 0 ) ) :
        delimiter = u","
    else :
        delimiter = b","
    statswriter = csv . writer ( csvfile , delimiter = delimiter )
    # write column names if
    if is_new :
        statswriter . writerow ( [ key for key in sorted ( stats_dict . keys ( ) ) ] )
    row_to_write = [ ]
    # write stats row sorted by column name
    for key in sorted ( stats_dict . keys ( ) ) :
        row_to_write . append ( stats_dict [ key ] )
    statswriter . writerow ( row_to_write )


signature:
def get_stats ( gtfs ) :
    """Get basic statistics of the GTFS data .
 Parameters
 gtfs : GTFS
 Returns
 stats : dict
 A dictionary of various statistics .
 Keys should be strings , values should be inputtable to a database ( int , date , str , . . . )
 ( but not a list )"""

implementation:
stats = { }
# Basic table counts
for table in [ 'agencies' , 'routes' , 'stops' , 'stop_times' , 'trips' , 'calendar' , 'shapes' , 'calendar_dates' , 'days' , 'stop_distances' , 'frequencies' , 'feed_info' , 'transfers' ] :
    stats [ "n_" + table ] = gtfs . get_row_count ( table )
# Agency names
agencies = gtfs . get_table ( "agencies" )
stats [ "agencies" ] = "_" . join ( agencies [ 'name' ] . values )
# Stop lat / lon range
stops = gtfs . get_table ( "stops" )
lats = stops [ 'lat' ] . values
lons = stops [ 'lon' ] . values
percentiles = [ 0 , 10 , 50 , 90 , 100 ]
try :
    lat_percentiles = numpy . percentile ( lats , percentiles )
except IndexError :
    lat_percentiles = [ None ] * 5
lat_min , lat_10 , lat_median , lat_90 , lat_max = lat_percentiles
stats [ "lat_min" ] = lat_min
stats [ "lat_10" ] = lat_10
stats [ "lat_median" ] = lat_median
stats [ "lat_90" ] = lat_90
stats [ "lat_max" ] = lat_max
try :
    lon_percentiles = numpy . percentile ( lons , percentiles )
except IndexError :
    lon_percentiles = [ None ] * 5
lon_min , lon_10 , lon_median , lon_90 , lon_max = lon_percentiles
stats [ "lon_min" ] = lon_min
stats [ "lon_10" ] = lon_10
stats [ "lon_median" ] = lon_median
stats [ "lon_90" ] = lon_90
stats [ "lon_max" ] = lon_max
if len ( lats ) > 0 :
    stats [ "height_km" ] = wgs84_distance ( lat_min , lon_median , lat_max , lon_median ) / 1000.
    stats [ "width_km" ] = wgs84_distance ( lon_min , lat_median , lon_max , lat_median ) / 1000.
else :
    stats [ "height_km" ] = None
    stats [ "width_km" ] = None
first_day_start_ut , last_day_start_ut = gtfs . get_day_start_ut_span ( )
stats [ "start_time_ut" ] = first_day_start_ut
if last_day_start_ut is None :
    stats [ "end_time_ut" ] = None
else : # 28 ( instead of 24 ) comes from the GTFS stANDard
    stats [ "end_time_ut" ] = last_day_start_ut + 28 * 3600
stats [ "start_date" ] = gtfs . get_min_date ( )
stats [ "end_date" ] = gtfs . get_max_date ( )
# Maximum activity day
max_activity_date = gtfs . execute_custom_query ( 'SELECT count(*), date ' 'FROM days ' 'GROUP BY date ' 'ORDER BY count(*) DESC, date ' 'LIMIT 1;' ) . fetchone ( )
if max_activity_date :
    stats [ "max_activity_date" ] = max_activity_date [ 1 ]
    max_activity_hour = gtfs . get_cursor ( ) . execute ( 'SELECT count(*), arr_time_hour FROM day_stop_times ' 'WHERE date=? GROUP BY arr_time_hour ' 'ORDER BY count(*) DESC;' , ( stats [ "max_activity_date" ] , ) ) . fetchone ( )
    if max_activity_hour :
        stats [ "max_activity_hour" ] = max_activity_hour [ 1 ]
    else :
        stats [ "max_activity_hour" ] = None
# Fleet size estimate : considering each line separately
if max_activity_date and max_activity_hour :
    fleet_size_estimates = _fleet_size_estimate ( gtfs , stats [ 'max_activity_hour' ] , stats [ 'max_activity_date' ] )
    stats . update ( fleet_size_estimates )
# Compute simple distributions of various columns that have a finite range of values .
# Commented lines refer to values that are not imported yet , ?
stats [ 'routes__type__dist' ] = _distribution ( gtfs , 'routes' , 'type' )
# stats [ ' stop _ times _ _ pickup _ type _ _ dist ' ] = _ distribution ( gtfs , ' stop _ times ' , ' pickup _ type ' )
# stats [ ' stop _ times _ _ drop _ off _ type _ _ dist ' ] = _ distribution ( gtfs , ' stop _ times ' , ' drop _ off _ type ' )
# stats [ ' stop _ times _ _ timepoint _ _ dist ' ] = _ distribution ( gtfs , ' stop _ times ' , ' timepoint ' )
stats [ 'calendar_dates__exception_type__dist' ] = _distribution ( gtfs , 'calendar_dates' , 'exception_type' )
stats [ 'frequencies__exact_times__dist' ] = _distribution ( gtfs , 'frequencies' , 'exact_times' )
stats [ 'transfers__transfer_type__dist' ] = _distribution ( gtfs , 'transfers' , 'transfer_type' )
stats [ 'agencies__lang__dist' ] = _distribution ( gtfs , 'agencies' , 'lang' )
stats [ 'stops__location_type__dist' ] = _distribution ( gtfs , 'stops' , 'location_type' )
# stats [ ' stops _ _ wheelchair _ boarding _ _ dist ' ] = _ distribution ( gtfs , ' stops ' , ' wheelchair _ boarding ' )
# stats [ ' trips _ _ wheelchair _ accessible _ _ dist ' ] = _ distribution ( gtfs , ' trips ' , ' wheelchair _ accessible ' )
# stats [ ' trips _ _ bikes _ allowed _ _ dist ' ] = _ distribution ( gtfs , ' trips ' , ' bikes _ allowed ' )
# stats [ ' ' ] = _ distribution ( gtfs , ' ' , ' ' )
stats = _feed_calendar_span ( gtfs , stats )
return stats


signature:
def _distribution ( gtfs , table , column ) :
    """Count occurrences of values AND return it as a string .
 Example return value : ' 1:5 2:15'"""

implementation:
cur = gtfs . conn . cursor ( )
cur . execute ( 'SELECT {column}, count(*) ' 'FROM {table} GROUP BY {column} ' 'ORDER BY {column}' . format ( column = column , table = table ) )
return ' ' . join ( '%s:%s' % ( t , c ) for t , c in cur )


signature:
def _fleet_size_estimate ( gtfs , hour , date ) :
    """Calculates fleet size estimates by two separate formula :
 1 . Considering all routes separately with no interlining and doing a deficit calculation at every terminal
 2 . By looking at the maximum number of vehicles in simultaneous movement
 Parameters
 gtfs : GTFS
 hour : int
 date : ?
 Returns
 results : dict
 a dict with keys :
 fleet _ size _ route _ based
 fleet _ size _ max _ movement"""

implementation:
results = { }
fleet_size_list = [ ]
cur = gtfs . conn . cursor ( )
rows = cur . execute ( 'SELECT type, max(vehicles) ' 'FROM (' 'SELECT type, direction_id, sum(vehicles) as vehicles ' 'FROM ' '(' 'SELECT trips.route_I, trips.direction_id, routes.route_id, name, type, count(*) as vehicles, cycle_time_min ' 'FROM trips, routes, days, ' '(' 'SELECT first_trip.route_I, first_trip.direction_id, first_trip_start_time, first_trip_end_time, ' 'MIN(start_time_ds) as return_trip_start_time, end_time_ds as return_trip_end_time, ' '(end_time_ds - first_trip_start_time)/60 as cycle_time_min ' 'FROM ' 'trips, ' '(SELECT route_I, direction_id, MIN(start_time_ds) as first_trip_start_time, ' 'end_time_ds as first_trip_end_time ' 'FROM trips, days ' 'WHERE trips.trip_I=days.trip_I AND start_time_ds >= ? * 3600 ' 'AND start_time_ds <= (? + 1) * 3600 AND date = ? ' 'GROUP BY route_I, direction_id) first_trip ' 'WHERE first_trip.route_I = trips.route_I ' 'AND first_trip.direction_id != trips.direction_id ' 'AND start_time_ds >= first_trip_end_time ' 'GROUP BY trips.route_I, trips.direction_id' ') return_trip ' 'WHERE trips.trip_I=days.trip_I AND trips.route_I= routes.route_I ' 'AND date = ? AND trips.route_I = return_trip.route_I ' 'AND trips.direction_id = return_trip.direction_id ' 'AND start_time_ds >= first_trip_start_time ' 'AND start_time_ds < return_trip_end_time ' 'GROUP BY trips.route_I, trips.direction_id ' 'ORDER BY type, name, vehicles desc' ') cycle_times ' 'GROUP BY direction_id, type' ') vehicles_type ' 'GROUP BY type;' , ( hour , hour , date , date ) )
for row in rows :
    fleet_size_list . append ( str ( row [ 0 ] ) + ':' + str ( row [ 1 ] ) )
results [ 'fleet_size_route_based' ] = " " . join ( fleet_size_list )
# Fleet size estimate : maximum number of vehicles in movement
fleet_size_list = [ ]
fleet_size_dict = { }
if hour :
    for minute in range ( hour * 3600 , ( hour + 1 ) * 3600 , 60 ) :
        rows = gtfs . conn . cursor ( ) . execute ( 'SELECT type, count(*) ' 'FROM trips, routes, days ' 'WHERE trips.route_I = routes.route_I ' 'AND trips.trip_I=days.trip_I ' 'AND start_time_ds <= ? ' 'AND end_time_ds > ? + 60 ' 'AND date = ? ' 'GROUP BY type;' , ( minute , minute , date ) )
        for row in rows :
            if fleet_size_dict . get ( row [ 0 ] , 0 ) < row [ 1 ] :
                fleet_size_dict [ row [ 0 ] ] = row [ 1 ]
for key in fleet_size_dict . keys ( ) :
    fleet_size_list . append ( str ( key ) + ':' + str ( fleet_size_dict [ key ] ) )
results [ "fleet_size_max_movement" ] = ' ' . join ( fleet_size_list )
return results


signature:
def _feed_calendar_span ( gtfs , stats ) :
    """Computes the temporal coverage of each source feed
 Parameters
 gtfs : gtfspy . GTFS object
 stats : dict
 where to append the stats
 Returns
 stats : dict"""

implementation:
n_feeds = _n_gtfs_sources ( gtfs ) [ 0 ]
max_start = None
min_end = None
if n_feeds > 1 :
    for i in range ( n_feeds ) :
        feed_key = "feed_" + str ( i ) + "_"
        start_key = feed_key + "calendar_start"
        end_key = feed_key + "calendar_end"
        calendar_span = gtfs . conn . cursor ( ) . execute ( 'SELECT min(date), max(date) FROM trips, days ' 'WHERE trips.trip_I = days.trip_I AND trip_id LIKE ?;' , ( feed_key + '%' , ) ) . fetchone ( )
        stats [ start_key ] = calendar_span [ 0 ]
        stats [ end_key ] = calendar_span [ 1 ]
        if calendar_span [ 0 ] is not None and calendar_span [ 1 ] is not None :
            if not max_start and not min_end :
                max_start = calendar_span [ 0 ]
                min_end = calendar_span [ 1 ]
            else :
                if gtfs . get_day_start_ut ( calendar_span [ 0 ] ) > gtfs . get_day_start_ut ( max_start ) :
                    max_start = calendar_span [ 0 ]
                if gtfs . get_day_start_ut ( calendar_span [ 1 ] ) < gtfs . get_day_start_ut ( min_end ) :
                    min_end = calendar_span [ 1 ]
    stats [ "latest_feed_start_date" ] = max_start
    stats [ "earliest_feed_end_date" ] = min_end
else :
    stats [ "latest_feed_start_date" ] = stats [ "start_date" ]
    stats [ "earliest_feed_end_date" ] = stats [ "end_date" ]
return stats


signature:
def route_frequencies ( gtfs , results_by_mode = False ) :
    """Return the frequency of all types of routes per day .
 Parameters
 gtfs : GTFS
 Returns
 pandas . DataFrame with columns
 route _ I , type , frequency"""

implementation:
day = gtfs . get_suitable_date_for_daily_extract ( )
query = ( " SELECT f.route_I, type, frequency FROM routes as r" " JOIN" " (SELECT route_I, COUNT(route_I) as frequency" " FROM" " (SELECT date, route_I, trip_I" " FROM day_stop_times" " WHERE date = '{day}'" " GROUP by route_I, trip_I)" " GROUP BY route_I) as f" " ON f.route_I = r.route_I" " ORDER BY frequency DESC" . format ( day = day ) )
return pd . DataFrame ( gtfs . execute_custom_query_pandas ( query ) )


signature:
def hourly_frequencies ( gtfs , st , et , route_type ) :
    """Return all the number of vehicles ( i . e . busses , trams , etc ) that pass hourly through a stop in a time frame .
 Parameters
 gtfs : GTFS
 st : int
 start time of the time framein unix time
 et : int
 end time of the time frame in unix time
 route _ type : int
 Returns
 numeric pandas . DataFrame with columns
 stop _ I , lat , lon , frequency"""

implementation:
timeframe = et - st
hours = timeframe / 3600
day = gtfs . get_suitable_date_for_daily_extract ( )
stops = gtfs . get_stops_for_route_type ( route_type ) . T . drop_duplicates ( ) . T
query = ( "SELECT * FROM stops as x" " JOIN" " (SELECT * , COUNT(*)/{h} as frequency" " FROM stop_times, days" " WHERE stop_times.trip_I = days.trip_I" " AND dep_time_ds > {st}" " AND dep_time_ds < {et}" " AND date = '{day}'" " GROUP BY stop_I) as y" " ON y.stop_I = x.stop_I" . format ( h = hours , st = st , et = et , day = day ) )
try :
    trips_frequency = gtfs . execute_custom_query_pandas ( query ) . T . drop_duplicates ( ) . T
    df = pd . merge ( stops [ [ 'stop_I' , 'lat' , 'lon' ] ] , trips_frequency [ [ 'stop_I' , 'frequency' ] ] , on = 'stop_I' , how = 'inner' )
    return df . apply ( pd . to_numeric )
except :
    raise ValueError ( "Maybe too short time frame!" )


signature:
def get_vehicle_hours_by_type ( gtfs , route_type ) :
    """Return the sum of vehicle hours in a particular day by route type ."""

implementation:
day = gtfs . get_suitable_date_for_daily_extract ( )
query = ( " SELECT * , SUM(end_time_ds - start_time_ds)/3600 as vehicle_hours_type" " FROM" " (SELECT * FROM day_trips as q1" " INNER JOIN" " (SELECT route_I, type FROM routes) as q2" " ON q1.route_I = q2.route_I" " WHERE type = {route_type}" " AND date = '{day}')" . format ( day = day , route_type = route_type ) )
df = gtfs . execute_custom_query_pandas ( query )
return df [ 'vehicle_hours_type' ] . item ( )


signature:
def _scan_footpaths ( self , stop_id , walk_departure_time ) :
    """Scan the footpaths originating from stop _ id
 Parameters
 stop _ id : int"""

implementation:
for _ , neighbor , data in self . _walk_network . edges_iter ( nbunch = [ stop_id ] , data = True ) :
    d_walk = data [ "d_walk" ]
    arrival_time = walk_departure_time + d_walk / self . _walk_speed
    self . _update_stop_label ( neighbor , arrival_time )


signature:
def timeit ( method ) :
    """A Python decorator for printing out the execution time for a function .
 Adapted from :
 www . andreas - jung . com / contents / a - python - decorator - for - measuring - the - execution - time - of - methods"""

implementation:
def timed ( * args , ** kw ) :
    time_start = time . time ( )
    result = method ( * args , ** kw )
    time_end = time . time ( )
    print ( 'timeit: %r %2.2f sec (%r, %r) ' % ( method . __name__ , time_end - time_start , str ( args ) [ : 20 ] , kw ) )
    return result
return timed


signature:
def validate_and_get_warnings ( self ) :
    """Validates / checks a given GTFS feed with respect to a number of different issues .
 The set of warnings that are checked for , can be found in the gtfs _ validator . ALL _ WARNINGS
 Returns
 warnings : WarningsContainer"""

implementation:
self . warnings_container . clear ( )
self . _validate_stops_with_same_stop_time ( )
self . _validate_speeds_and_trip_times ( )
self . _validate_stop_spacings ( )
self . _validate_stop_sequence ( )
self . _validate_misplaced_stops ( )
return self . warnings_container


signature:
def clean_password ( self ) :
    """Check that the password is valid ."""

implementation:
value = self . cleaned_data . get ( 'password' )
if value not in self . valid_passwords :
    raise forms . ValidationError ( 'Incorrect password.' )
return value


signature:
def clean ( self ) :
    """When receiving the filled out form , check for valid access ."""

implementation:
cleaned_data = super ( AuthForm , self ) . clean ( )
user = self . get_user ( )
if self . staff_only and ( not user or not user . is_staff ) :
    raise forms . ValidationError ( 'Sorry, only staff are allowed.' )
if self . superusers_only and ( not user or not user . is_superuser ) :
    raise forms . ValidationError ( 'Sorry, only superusers are allowed.' )
return cleaned_data


signature:
def authenticate ( self , token_value ) :
    """Check that the password is valid .
 This allows for revoking of a user ' s preview rights by changing the
 valid passwords ."""

implementation:
try :
    backend_path , user_id = token_value . split ( ':' , 1 )
except ( ValueError , AttributeError ) :
    return False
backend = auth . load_backend ( backend_path )
return bool ( backend . get_user ( user_id ) )


signature:
def get_lockdown_form ( form_path ) :
    """Return a form class for a given string pointing to a lockdown form ."""

implementation:
if not form_path :
    raise ImproperlyConfigured ( 'No LOCKDOWN_FORM specified.' )
form_path_list = form_path . split ( "." )
new_module = "." . join ( form_path_list [ : - 1 ] )
attr = form_path_list [ - 1 ]
try :
    mod = import_module ( new_module )
except ( ImportError , ValueError ) :
    raise ImproperlyConfigured ( 'Module configured in LOCKDOWN_FORM (%s) to' ' contain the form class couldn\'t be ' 'found.' % new_module )
try :
    form = getattr ( mod , attr )
except AttributeError :
    raise ImproperlyConfigured ( 'The module configured in LOCKDOWN_FORM ' ' (%s) doesn\'t define a "%s" form.' % ( new_module , attr ) )
return form


signature:
def process_request ( self , request ) :
    """Check if each request is allowed to access the current resource ."""

implementation:
try :
    session = request . session
except AttributeError :
    raise ImproperlyConfigured ( 'django-lockdown requires the Django ' 'sessions framework' )
# Don ' t lock down if django - lockdown is disabled altogether .
if settings . ENABLED is False :
    return None
# Don ' t lock down if the client REMOTE _ ADDR matched and is part of the
# exception list .
if self . remote_addr_exceptions :
    remote_addr_exceptions = self . remote_addr_exceptions
else :
    remote_addr_exceptions = settings . REMOTE_ADDR_EXCEPTIONS
if remote_addr_exceptions : # If forwarding proxies are used they must be listed as trusted
    trusted_proxies = self . trusted_proxies or settings . TRUSTED_PROXIES
    remote_addr = request . META . get ( 'REMOTE_ADDR' )
    if remote_addr in remote_addr_exceptions :
        return None
    if remote_addr in trusted_proxies : # If REMOTE _ ADDR is a trusted proxy check x - forwarded - for
        x_forwarded_for = request . META . get ( 'HTTP_X_FORWARDED_FOR' )
        if x_forwarded_for :
            remote_addr = x_forwarded_for . split ( ',' ) [ - 1 ] . strip ( )
            if remote_addr in remote_addr_exceptions :
                return None
# Don ' t lock down if the URL matches an exception pattern .
if self . url_exceptions :
    url_exceptions = compile_url_exceptions ( self . url_exceptions )
else :
    url_exceptions = compile_url_exceptions ( settings . URL_EXCEPTIONS )
for pattern in url_exceptions :
    if pattern . search ( request . path ) :
        return None
# Don ' t lock down if the URL resolves to a whitelisted view .
try :
    resolved_path = resolve ( request . path )
except Resolver404 :
    pass
else :
    if resolved_path . func in settings . VIEW_EXCEPTIONS :
        return None
# Don ' t lock down if outside of the lockdown dates .
if self . until_date :
    until_date = self . until_date
else :
    until_date = settings . UNTIL_DATE
if self . after_date :
    after_date = self . after_date
else :
    after_date = settings . AFTER_DATE
if until_date or after_date :
    locked_date = False
    if until_date and datetime . datetime . now ( ) < until_date :
        locked_date = True
    if after_date and datetime . datetime . now ( ) > after_date :
        locked_date = True
    if not locked_date :
        return None
form_data = request . POST if request . method == 'POST' else None
if self . form :
    form_class = self . form
else :
    form_class = get_lockdown_form ( settings . FORM )
form = form_class ( data = form_data , ** self . form_kwargs )
authorized = False
token = session . get ( self . session_key )
if hasattr ( form , 'authenticate' ) :
    if form . authenticate ( token ) :
        authorized = True
elif token is True :
    authorized = True
if authorized and self . logout_key and self . logout_key in request . GET :
    if self . session_key in session :
        del session [ self . session_key ]
    querystring = request . GET . copy ( )
    del querystring [ self . logout_key ]
    return self . redirect ( request )
# Don ' t lock down if the user is already authorized for previewing .
if authorized :
    return None
if form . is_valid ( ) :
    if hasattr ( form , 'generate_token' ) :
        token = form . generate_token ( )
    else :
        token = True
    session [ self . session_key ] = token
    return self . redirect ( request )
page_data = { 'until_date' : until_date , 'after_date' : after_date }
if not hasattr ( form , 'show_form' ) or form . show_form ( ) :
    page_data [ 'form' ] = form
if self . extra_context :
    page_data . update ( self . extra_context )
return render ( request , 'lockdown/form.html' , page_data )


signature:
def redirect ( self , request ) :
    """Handle redirects properly ."""

implementation:
url = request . path
querystring = request . GET . copy ( )
if self . logout_key and self . logout_key in request . GET :
    del querystring [ self . logout_key ]
if querystring :
    url = '%s?%s' % ( url , querystring . urlencode ( ) )
return HttpResponseRedirect ( url )


signature:
def infer ( pattern , base_path = None ) :
    """https : / / github . com / frictionlessdata / datapackage - py # infer"""

implementation:
package = Package ( { } , base_path = base_path )
descriptor = package . infer ( pattern )
return descriptor


signature:
def get ( self , profile_id ) :
    '''Returns the profile with the received ID as a dict
 If a local copy of the profile exists , it ' ll be returned . If not , it ' ll
 be downloaded from the web . The results are cached , so any subsequent
 calls won ' t hit the filesystem or the web .
 Args :
 profile _ id ( str ) : The ID of the profile you want .
 Raises :
 RegistryError : If there was some problem opening the profile file
 or its format was incorrect .'''

implementation:
if profile_id not in self . _profiles :
    try :
        self . _profiles [ profile_id ] = self . _get_profile ( profile_id )
    except ( ValueError , IOError ) as e :
        six . raise_from ( RegistryError ( e ) , e )
return self . _profiles [ profile_id ]


signature:
def _get_profile ( self , profile_id ) :
    '''dict : Return the profile with the received ID as a dict ( None if it
 doesn ' t exist ) .'''

implementation:
profile_metadata = self . _registry . get ( profile_id )
if not profile_metadata :
    return
path = self . _get_absolute_path ( profile_metadata . get ( 'schema_path' ) )
url = profile_metadata . get ( 'schema' )
if path :
    try :
        return self . _load_json_file ( path )
    except IOError as local_exc :
        if not url :
            raise local_exc
        try :
            return self . _load_json_url ( url )
        except IOError :
            msg = ( 'Error loading profile locally at "{path}" ' 'and remotely at "{url}".' ) . format ( path = path , url = url )
            six . raise_from ( IOError ( msg ) , local_exc )
elif url :
    return self . _load_json_url ( url )


signature:
def _get_registry ( self , registry_path_or_url ) :
    '''dict : Return the registry as dict with profiles keyed by id .'''

implementation:
if registry_path_or_url . startswith ( 'http' ) :
    profiles = self . _load_json_url ( registry_path_or_url )
else :
    profiles = self . _load_json_file ( registry_path_or_url )
try :
    registry = { }
    for profile in profiles :
        registry [ profile [ 'id' ] ] = profile
    return registry
except KeyError as e :
    msg = ( 'Registry at "{path}" has no "id" column.' ) . format ( path = registry_path_or_url )
    six . raise_from ( ValueError ( msg ) , e )


signature:
def _get_absolute_path ( self , relative_path ) :
    '''str : Return the received relative _ path joined with the base path
 ( None if there were some error ) .'''

implementation:
try :
    return os . path . join ( self . base_path , relative_path )
except ( AttributeError , TypeError ) :
    pass


signature:
def _load_json_url ( self , url ) :
    '''dict : Return the JSON at the local path or URL as a dict .'''

implementation:
res = requests . get ( url )
res . raise_for_status ( )
return res . json ( )


signature:
def get_descriptor_base_path ( descriptor ) :
    """Get descriptor base path if string or return None ."""

implementation:
# Infer from path / url
if isinstance ( descriptor , six . string_types ) :
    if os . path . exists ( descriptor ) :
        base_path = os . path . dirname ( os . path . abspath ( descriptor ) )
    else : # suppose descriptor is a URL
        base_path = os . path . dirname ( descriptor )
# Current dir by default
else :
    base_path = '.'
return base_path


signature:
def retrieve_descriptor ( descriptor ) :
    """Retrieve descriptor ."""

implementation:
the_descriptor = descriptor
if the_descriptor is None :
    the_descriptor = { }
if isinstance ( the_descriptor , six . string_types ) :
    try :
        if os . path . isfile ( the_descriptor ) :
            with open ( the_descriptor , 'r' ) as f :
                the_descriptor = json . load ( f )
        else :
            req = requests . get ( the_descriptor )
            req . raise_for_status ( )
            # Force UTF8 encoding for ' text / plain ' sources
            req . encoding = 'utf8'
            the_descriptor = req . json ( )
    except ( IOError , requests . exceptions . RequestException ) as error :
        message = 'Unable to load JSON at "%s"' % descriptor
        six . raise_from ( exceptions . DataPackageException ( message ) , error )
    except ValueError as error : # Python2 doesn ' t have json . JSONDecodeError ( use ValueErorr )
        message = 'Unable to parse JSON at "%s". %s' % ( descriptor , error )
        six . raise_from ( exceptions . DataPackageException ( message ) , error )
if hasattr ( the_descriptor , 'read' ) :
    try :
        the_descriptor = json . load ( the_descriptor )
    except ValueError as e :
        six . raise_from ( exceptions . DataPackageException ( str ( e ) ) , e )
if not isinstance ( the_descriptor , dict ) :
    msg = 'Data must be a \'dict\', but was a \'{0}\''
    raise exceptions . DataPackageException ( msg . format ( type ( the_descriptor ) . __name__ ) )
return the_descriptor


signature:
def dereference_package_descriptor ( descriptor , base_path ) :
    """Dereference data package descriptor ( IN - PLACE FOR NOW ) ."""

implementation:
for resource in descriptor . get ( 'resources' , [ ] ) :
    dereference_resource_descriptor ( resource , base_path , descriptor )
return descriptor


signature:
def dereference_resource_descriptor ( descriptor , base_path , base_descriptor = None ) :
    """Dereference resource descriptor ( IN - PLACE FOR NOW ) ."""

implementation:
PROPERTIES = [ 'schema' , 'dialect' ]
if base_descriptor is None :
    base_descriptor = descriptor
for property in PROPERTIES :
    value = descriptor . get ( property )
    # URI - > No
    if not isinstance ( value , six . string_types ) :
        continue
    # URI - > Pointer
    if value . startswith ( '#' ) :
        try :
            pointer = jsonpointer . JsonPointer ( value [ 1 : ] )
            descriptor [ property ] = pointer . resolve ( base_descriptor )
        except Exception as error :
            message = 'Not resolved Pointer URI "%s" for resource.%s' % ( value , property )
            six . raise_from ( exceptions . DataPackageException ( message ) , error )
    # URI - > Remote
    elif value . startswith ( 'http' ) :
        try :
            response = requests . get ( value )
            response . raise_for_status ( )
            descriptor [ property ] = response . json ( )
        except Exception as error :
            message = 'Not resolved Remote URI "%s" for resource.%s' % ( value , property )
            six . raise_from ( exceptions . DataPackageException ( message ) , error )
    # URI - > Local
    else :
        if not is_safe_path ( value ) :
            raise exceptions . DataPackageException ( 'Not safe path in Local URI "%s" ' 'for resource.%s' % ( value , property ) )
        if not base_path :
            raise exceptions . DataPackageException ( 'Local URI "%s" requires base path ' 'for resource.%s' % ( value , property ) )
        fullpath = os . path . join ( base_path , value )
        try :
            with io . open ( fullpath , encoding = 'utf-8' ) as file :
                descriptor [ property ] = json . load ( file )
        except Exception as error :
            message = 'Not resolved Local URI "%s" for resource.%s' % ( value , property )
            six . raise_from ( exceptions . DataPackageException ( message ) , error )
return descriptor


signature:
def expand_package_descriptor ( descriptor ) :
    """Apply defaults to data package descriptor ( IN - PLACE FOR NOW ) ."""

implementation:
descriptor . setdefault ( 'profile' , config . DEFAULT_DATA_PACKAGE_PROFILE )
for resource in descriptor . get ( 'resources' , [ ] ) :
    expand_resource_descriptor ( resource )
return descriptor


signature:
def expand_resource_descriptor ( descriptor ) :
    """Apply defaults to resource descriptor ( IN - PLACE FOR NOW ) ."""

implementation:
descriptor . setdefault ( 'profile' , config . DEFAULT_RESOURCE_PROFILE )
if descriptor [ 'profile' ] == 'tabular-data-resource' : # Schema
    schema = descriptor . get ( 'schema' )
    if schema is not None :
        for field in schema . get ( 'fields' , [ ] ) :
            field . setdefault ( 'type' , config . DEFAULT_FIELD_TYPE )
            field . setdefault ( 'format' , config . DEFAULT_FIELD_FORMAT )
        schema . setdefault ( 'missingValues' , config . DEFAULT_MISSING_VALUES )
    # Dialect
    dialect = descriptor . get ( 'dialect' )
    if dialect is not None :
        for key , value in config . DEFAULT_DIALECT . items ( ) :
            dialect . setdefault ( key , value )
return descriptor


signature:
def is_safe_path ( path ) :
    """Check if path is safe and allowed ."""

implementation:
contains_windows_var = lambda val : re . match ( r'%.+%' , val )
contains_posix_var = lambda val : re . match ( r'\$.+' , val )
unsafeness_conditions = [ os . path . isabs ( path ) , ( '..%s' % os . path . sep ) in path , path . startswith ( '~' ) , os . path . expandvars ( path ) != path , contains_windows_var ( path ) , contains_posix_var ( path ) , ]
return not any ( unsafeness_conditions )


signature:
def _extract_zip_if_possible ( descriptor ) :
    """If descriptor is a path to zip file extract and return ( tempdir , descriptor )"""

implementation:
tempdir = None
result = descriptor
try :
    if isinstance ( descriptor , six . string_types ) :
        res = requests . get ( descriptor )
        res . raise_for_status ( )
        result = res . content
except ( IOError , ValueError , requests . exceptions . RequestException ) :
    pass
try :
    the_zip = result
    if isinstance ( the_zip , bytes ) :
        try :
            os . path . isfile ( the_zip )
        except ( TypeError , ValueError ) : # the _ zip contains the zip file contents
            the_zip = io . BytesIO ( the_zip )
    if zipfile . is_zipfile ( the_zip ) :
        with zipfile . ZipFile ( the_zip , 'r' ) as z :
            _validate_zip ( z )
            descriptor_path = [ f for f in z . namelist ( ) if f . endswith ( 'datapackage.json' ) ] [ 0 ]
            tempdir = tempfile . mkdtemp ( '-datapackage' )
            z . extractall ( tempdir )
            result = os . path . join ( tempdir , descriptor_path )
    else :
        result = descriptor
except ( TypeError , zipfile . BadZipfile ) :
    pass
if hasattr ( descriptor , 'seek' ) : # Rewind descriptor if it ' s a file , as we read it for testing if it ' s
# a zip file
    descriptor . seek ( 0 )
return ( tempdir , result )


signature:
def _validate_zip ( the_zip ) :
    """Validate zipped data package"""

implementation:
datapackage_jsons = [ f for f in the_zip . namelist ( ) if f . endswith ( 'datapackage.json' ) ]
if len ( datapackage_jsons ) != 1 :
    msg = 'DataPackage must have only one "datapackage.json" (had {n})'
    raise exceptions . DataPackageException ( msg . format ( n = len ( datapackage_jsons ) ) )


signature:
def _slugify_foreign_key ( schema ) :
    """Slugify foreign key"""

implementation:
for foreign_key in schema . get ( 'foreignKeys' , [ ] ) :
    foreign_key [ 'reference' ] [ 'resource' ] = _slugify_resource_name ( foreign_key [ 'reference' ] . get ( 'resource' , '' ) )
return schema


signature:
def get_resource ( self , name ) :
    """https : / / github . com / frictionlessdata / datapackage - py # package"""

implementation:
for resource in self . resources :
    if resource . name == name :
        return resource
return None


signature:
def add_resource ( self , descriptor ) :
    """https : / / github . com / frictionlessdata / datapackage - py # package"""

implementation:
self . __current_descriptor . setdefault ( 'resources' , [ ] )
self . __current_descriptor [ 'resources' ] . append ( descriptor )
self . __build ( )
return self . __resources [ - 1 ]


signature:
def remove_resource ( self , name ) :
    """https : / / github . com / frictionlessdata / datapackage - py # package"""

implementation:
resource = self . get_resource ( name )
if resource :
    predicat = lambda resource : resource . get ( 'name' ) != name
    self . __current_descriptor [ 'resources' ] = list ( filter ( predicat , self . __current_descriptor [ 'resources' ] ) )
    self . __build ( )
return resource


signature:
def infer ( self , pattern = False ) :
    """https : / / github . com / frictionlessdata / datapackage - py # package"""

implementation:
# Files
if pattern : # No base path
    if not self . __base_path :
        message = 'Base path is required for pattern infer'
        raise exceptions . DataPackageException ( message )
    # Add resources
    options = { 'recursive' : True } if '**' in pattern else { }
    for path in glob . glob ( os . path . join ( self . __base_path , pattern ) , ** options ) :
        self . add_resource ( { 'path' : os . path . relpath ( path , self . __base_path ) } )
# Resources
for index , resource in enumerate ( self . resources ) :
    descriptor = resource . infer ( )
    self . __current_descriptor [ 'resources' ] [ index ] = descriptor
    self . __build ( )
# Profile
if self . __next_descriptor [ 'profile' ] == config . DEFAULT_DATA_PACKAGE_PROFILE :
    if self . resources and all ( map ( lambda resource : resource . tabular , self . resources ) ) :
        self . __current_descriptor [ 'profile' ] = 'tabular-data-package'
        self . __build ( )
return self . __current_descriptor


signature:
def save ( self , target = None , storage = None , ** options ) :
    """https : / / github . com / frictionlessdata / datapackage - py # package"""

implementation:
# Save package to storage
if storage is not None :
    if not isinstance ( storage , Storage ) :
        storage = Storage . connect ( storage , ** options )
    buckets = [ ]
    schemas = [ ]
    for resource in self . resources :
        if resource . tabular :
            resource . infer ( )
            buckets . append ( _slugify_resource_name ( resource . name ) )
            schemas . append ( resource . schema . descriptor )
    schemas = list ( map ( _slugify_foreign_key , schemas ) )
    storage . create ( buckets , schemas , force = True )
    for bucket in storage . buckets :
        resource = self . resources [ buckets . index ( bucket ) ]
        storage . write ( bucket , resource . iter ( ) )
# Save descriptor to json
elif str ( target ) . endswith ( '.json' ) :
    mode = 'w'
    encoding = 'utf-8'
    if six . PY2 :
        mode = 'wb'
        encoding = None
    helpers . ensure_dir ( target )
    with io . open ( target , mode = mode , encoding = encoding ) as file :
        json . dump ( self . __current_descriptor , file , indent = 4 )
# Save package to zip
else :
    try :
        with zipfile . ZipFile ( target , 'w' ) as z :
            descriptor = json . loads ( json . dumps ( self . __current_descriptor ) )
            for index , resource in enumerate ( self . resources ) :
                if not resource . name :
                    continue
                if not resource . local :
                    continue
                path = os . path . abspath ( resource . source )
                basename = resource . descriptor . get ( 'name' )
                resource_format = resource . descriptor . get ( 'format' )
                if resource_format :
                    basename = '.' . join ( [ basename , resource_format . lower ( ) ] )
                path_inside_dp = os . path . join ( 'data' , basename )
                z . write ( path , path_inside_dp )
                descriptor [ 'resources' ] [ index ] [ 'path' ] = path_inside_dp
            z . writestr ( 'datapackage.json' , json . dumps ( descriptor ) )
    except ( IOError , zipfile . BadZipfile , zipfile . LargeZipFile ) as exception :
        six . raise_from ( exceptions . DataPackageException ( exception ) , exception )
return True


signature:
def attributes ( self ) :
    """tuple : Attributes defined in the schema and the data package ."""

implementation:
# Deprecate
warnings . warn ( 'Property "package.attributes" is deprecated.' , UserWarning )
# Get attributes
attributes = set ( self . to_dict ( ) . keys ( ) )
try :
    attributes . update ( self . profile . properties . keys ( ) )
except AttributeError :
    pass
return tuple ( attributes )


signature:
def required_attributes ( self ) :
    """tuple : The schema ' s required attributed ."""

implementation:
# Deprecate
warnings . warn ( 'Property "package.required_attributes" is deprecated.' , UserWarning )
required = ( )
# Get required
try :
    if self . profile . required is not None :
        required = tuple ( self . profile . required )
except AttributeError :
    pass
return required


signature:
def validate ( self ) :
    """" Validate this Data Package ."""

implementation:
# Deprecate
warnings . warn ( 'Property "package.validate" is deprecated.' , UserWarning )
descriptor = self . to_dict ( )
self . profile . validate ( descriptor )


signature:
def iter_errors ( self ) :
    """" Lazily yields each ValidationError for the received data dict ."""

implementation:
# Deprecate
warnings . warn ( 'Property "package.iter_errors" is deprecated.' , UserWarning )
return self . profile . iter_errors ( self . to_dict ( ) )


signature:
def validate ( self , descriptor ) :
    """https : / / github . com / frictionlessdata / datapackage - py # schema"""

implementation:
# Collect errors
errors = [ ]
for error in self . _validator . iter_errors ( descriptor ) :
    if isinstance ( error , jsonschema . exceptions . ValidationError ) :
        message = str ( error . message )
        if six . PY2 :
            message = message . replace ( 'u\'' , '\'' )
        descriptor_path = '/' . join ( map ( str , error . path ) )
        profile_path = '/' . join ( map ( str , error . schema_path ) )
        error = exceptions . ValidationError ( 'Descriptor validation error: %s ' 'at "%s" in descriptor and ' 'at "%s" in profile' % ( message , descriptor_path , profile_path ) )
    errors . append ( error )
# Raise error
if errors :
    message = 'There are %s validation errors (see exception.errors)' % len ( errors )
    raise exceptions . ValidationError ( message , errors = errors )
return True


signature:
def iter_errors ( self , data ) :
    """Lazily yields each ValidationError for the received data dict ."""

implementation:
# Deprecate
warnings . warn ( 'Property "profile.iter_errors" is deprecated.' , UserWarning )
for error in self . _validator . iter_errors ( data ) :
    yield error


signature:
def tabular ( self ) :
    """https : / / github . com / frictionlessdata / datapackage - py # resource"""

implementation:
if self . __current_descriptor . get ( 'profile' ) == 'tabular-data-resource' :
    return True
if not self . __strict :
    if self . __current_descriptor . get ( 'format' ) in config . TABULAR_FORMATS :
        return True
    if self . __source_inspection . get ( 'tabular' , False ) :
        return True
return False


signature:
def iter ( self , relations = False , ** options ) :
    """https : / / github . com / frictionlessdata / datapackage - py # resource"""

implementation:
# Error for non tabular
if not self . tabular :
    message = 'Methods iter/read are not supported for non tabular data'
    raise exceptions . DataPackageException ( message )
# Get relations
if relations :
    relations = self . __get_relations ( )
return self . __get_table ( ) . iter ( relations = relations , ** options )


signature:
def raw_iter ( self , stream = False ) :
    """https : / / github . com / frictionlessdata / datapackage - py # resource"""

implementation:
# Error for inline
if self . inline :
    message = 'Methods raw_iter/raw_read are not supported for inline data'
    raise exceptions . DataPackageException ( message )
# Get filelike
if self . multipart :
    filelike = _MultipartSource ( self . source , remote = self . remote )
elif self . remote :
    if self . __table_options . get ( 'http_session' ) :
        http_session = self . __table_options [ 'http_session' ]
    else :
        http_session = requests . Session ( )
        http_session . headers = config . HTTP_HEADERS
    res = http_session . get ( self . source , stream = True )
    filelike = res . raw
else :
    filelike = io . open ( self . source , 'rb' )
return filelike


signature:
def raw_read ( self ) :
    """https : / / github . com / frictionlessdata / datapackage - py # resource"""

implementation:
contents = b''
with self . raw_iter ( ) as filelike :
    for chunk in filelike :
        contents += chunk
return contents


signature:
def infer ( self , ** options ) :
    """https : / / github . com / frictionlessdata / datapackage - py # resource"""

implementation:
descriptor = deepcopy ( self . __current_descriptor )
# Blank - > Stop
if self . __source_inspection . get ( 'blank' ) :
    return descriptor
# Name
if not descriptor . get ( 'name' ) :
    descriptor [ 'name' ] = self . __source_inspection [ 'name' ]
# Only for non inline / storage
if not self . inline and not self . __storage : # Format
    if not descriptor . get ( 'format' ) :
        descriptor [ 'format' ] = self . __source_inspection [ 'format' ]
    # Mediatype
    if not descriptor . get ( 'mediatype' ) :
        descriptor [ 'mediatype' ] = 'text/%s' % descriptor [ 'format' ]
    # Encoding
    if not descriptor . get ( 'encoding' ) :
        contents = b''
        with self . raw_iter ( stream = True ) as stream :
            for chunk in stream :
                contents += chunk
                if len ( contents ) > 1000 :
                    break
        encoding = cchardet . detect ( contents ) [ 'encoding' ]
        if encoding is not None :
            encoding = encoding . lower ( )
            descriptor [ 'encoding' ] = 'utf-8' if encoding == 'ascii' else encoding
# Schema
if not descriptor . get ( 'schema' ) :
    if self . tabular :
        descriptor [ 'schema' ] = self . __get_table ( ) . infer ( ** options )
# Profile
if descriptor . get ( 'profile' ) == config . DEFAULT_RESOURCE_PROFILE :
    if self . tabular :
        descriptor [ 'profile' ] = 'tabular-data-resource'
# Save descriptor
self . __current_descriptor = descriptor
self . __build ( )
return descriptor


signature:
def commit ( self , strict = None ) :
    """https : / / github . com / frictionlessdata / datapackage - py # resource"""

implementation:
if strict is not None :
    self . __strict = strict
elif self . __current_descriptor == self . __next_descriptor :
    return False
self . __current_descriptor = deepcopy ( self . __next_descriptor )
self . __table = None
self . __build ( )
return True


signature:
def save ( self , target , storage = None , ** options ) :
    """https : / / github . com / frictionlessdata / datapackage - py # resource"""

implementation:
# Save resource to storage
if storage is not None :
    if self . tabular :
        self . infer ( )
        storage . create ( target , self . schema . descriptor , force = True )
        storage . write ( target , self . iter ( ) )
# Save descriptor to json
else :
    mode = 'w'
    encoding = 'utf-8'
    if six . PY2 :
        mode = 'wb'
        encoding = None
    helpers . ensure_dir ( target )
    with io . open ( target , mode = mode , encoding = encoding ) as file :
        json . dump ( self . __current_descriptor , file , indent = 4 )


signature:
def push_datapackage ( descriptor , backend , ** backend_options ) :
    """Push Data Package to storage .
 All parameters should be used as keyword arguments .
 Args :
 descriptor ( str ) : path to descriptor
 backend ( str ) : backend name like ` sql ` or ` bigquery `
 backend _ options ( dict ) : backend options mentioned in backend docs"""

implementation:
# Deprecated
warnings . warn ( 'Functions "push/pull_datapackage" are deprecated. ' 'Please use "Package" class' , UserWarning )
# Init maps
tables = [ ]
schemas = [ ]
datamap = { }
mapping = { }
# Init model
model = Package ( descriptor )
# Get storage
plugin = import_module ( 'jsontableschema.plugins.%s' % backend )
storage = plugin . Storage ( ** backend_options )
# Collect tables / schemas / data
for resource in model . resources :
    if not resource . tabular :
        continue
    name = resource . descriptor . get ( 'name' , None )
    table = _convert_path ( resource . descriptor [ 'path' ] , name )
    schema = resource . descriptor [ 'schema' ]
    data = resource . table . iter ( keyed = True )
    # TODO : review
    def values ( schema , data ) :
        for item in data :
            row = [ ]
            for field in schema [ 'fields' ] :
                row . append ( item . get ( field [ 'name' ] , None ) )
            yield tuple ( row )
    tables . append ( table )
    schemas . append ( schema )
    datamap [ table ] = values ( schema , data )
    if name is not None :
        mapping [ name ] = table
schemas = _convert_schemas ( mapping , schemas )
# Create tables
for table in tables :
    if table in storage . buckets :
        storage . delete ( table )
storage . create ( tables , schemas )
# Write data to tables
for table in storage . buckets :
    if table in datamap :
        storage . write ( table , datamap [ table ] )
return storage


signature:
def pull_datapackage ( descriptor , name , backend , ** backend_options ) :
    """Pull Data Package from storage .
 All parameters should be used as keyword arguments .
 Args :
 descriptor ( str ) : path where to store descriptor
 name ( str ) : name of the pulled datapackage
 backend ( str ) : backend name like ` sql ` or ` bigquery `
 backend _ options ( dict ) : backend options mentioned in backend docs"""

implementation:
# Deprecated
warnings . warn ( 'Functions "push/pull_datapackage" are deprecated. ' 'Please use "Package" class' , UserWarning )
# Save datapackage name
datapackage_name = name
# Get storage
plugin = import_module ( 'jsontableschema.plugins.%s' % backend )
storage = plugin . Storage ( ** backend_options )
# Iterate over tables
resources = [ ]
for table in storage . buckets : # Prepare
    schema = storage . describe ( table )
    base = os . path . dirname ( descriptor )
    path , name = _restore_path ( table )
    fullpath = os . path . join ( base , path )
    # Write data
    helpers . ensure_dir ( fullpath )
    with io . open ( fullpath , 'wb' ) as file :
        model = Schema ( deepcopy ( schema ) )
        data = storage . iter ( table )
        writer = csv . writer ( file , encoding = 'utf-8' )
        writer . writerow ( model . headers )
        for row in data :
            writer . writerow ( row )
    # Add resource
    resource = { 'schema' : schema , 'path' : path }
    if name is not None :
        resource [ 'name' ] = name
    resources . append ( resource )
# Write descriptor
mode = 'w'
encoding = 'utf-8'
if six . PY2 :
    mode = 'wb'
    encoding = None
resources = _restore_resources ( resources )
helpers . ensure_dir ( descriptor )
with io . open ( descriptor , mode = mode , encoding = encoding ) as file :
    descriptor = { 'name' : datapackage_name , 'resources' : resources , }
    json . dump ( descriptor , file , indent = 4 )
return storage


signature:
def _convert_path ( path , name ) :
    """Convert resource ' s path and name to storage ' s table name .
 Args :
 path ( str ) : resource path
 name ( str ) : resource name
 Returns :
 str : table name"""

implementation:
table = os . path . splitext ( path ) [ 0 ]
table = table . replace ( os . path . sep , '__' )
if name is not None :
    table = '___' . join ( [ table , name ] )
table = re . sub ( '[^0-9a-zA-Z_]+' , '_' , table )
table = table . lower ( )
return table


signature:
def _restore_path ( table ) :
    """Restore resource ' s path and name from storage ' s table .
 Args :
 table ( str ) : table name
 Returns :
 ( str , str ) : resource path and name"""

implementation:
name = None
splited = table . split ( '___' )
path = splited [ 0 ]
if len ( splited ) == 2 :
    name = splited [ 1 ]
path = path . replace ( '__' , os . path . sep )
path += '.csv'
return path , name


signature:
def _convert_schemas ( mapping , schemas ) :
    """Convert schemas to be compatible with storage schemas .
 Foreign keys related operations .
 Args :
 mapping ( dict ) : mapping between resource name and table name
 schemas ( list ) : schemas
 Raises :
 ValueError : if there is no resource
 for some foreign key in given mapping
 Returns :
 list : converted schemas"""

implementation:
schemas = deepcopy ( schemas )
for schema in schemas :
    for fk in schema . get ( 'foreignKeys' , [ ] ) :
        resource = fk [ 'reference' ] [ 'resource' ]
        if resource != 'self' :
            if resource not in mapping :
                message = 'Not resource "%s" for foreign key "%s"'
                message = message % ( resource , fk )
                raise ValueError ( message )
            fk [ 'reference' ] [ 'resource' ] = mapping [ resource ]
return schemas


signature:
def _restore_resources ( resources ) :
    """Restore schemas from being compatible with storage schemas .
 Foreign keys related operations .
 Args :
 list : resources from storage
 Returns :
 list : restored resources"""

implementation:
resources = deepcopy ( resources )
for resource in resources :
    schema = resource [ 'schema' ]
    for fk in schema . get ( 'foreignKeys' , [ ] ) :
        _ , name = _restore_path ( fk [ 'reference' ] [ 'resource' ] )
        fk [ 'reference' ] [ 'resource' ] = name
return resources


signature:
def _buffer_incomplete_responses ( raw_output , buf ) :
    """It is possible for some of gdb ' s output to be read before it completely finished its response .
 In that case , a partial mi response was read , which cannot be parsed into structured data .
 We want to ALWAYS parse complete mi records . To do this , we store a buffer of gdb ' s
 output if the output did not end in a newline .
 Args :
 raw _ output : Contents of the gdb mi output
 buf ( str ) : Buffered gdb response from the past . This is incomplete and needs to be prepended to
 gdb ' s next output .
 Returns :
 ( raw _ output , buf )"""

implementation:
if raw_output :
    if buf : # concatenate buffer and new output
        raw_output = b"" . join ( [ buf , raw_output ] )
        buf = None
    if b"\n" not in raw_output : # newline was not found , so assume output is incomplete and store in buffer
        buf = raw_output
        raw_output = None
    elif not raw_output . endswith ( b"\n" ) : # raw output doesn ' t end in a newline , so store everything after the last newline ( if anything )
    # in the buffer , and parse everything before it
        remainder_offset = raw_output . rindex ( b"\n" ) + 1
        buf = raw_output [ remainder_offset : ]
        raw_output = raw_output [ : remainder_offset ]
return ( raw_output , buf )


signature:
def _make_non_blocking ( file_obj ) :
    """make file object non - blocking
 Windows doesn ' t have the fcntl module , but someone on
 stack overflow supplied this code as an answer , and it works
 http : / / stackoverflow . com / a / 34504971/2893090"""

implementation:
if USING_WINDOWS :
    LPDWORD = POINTER ( DWORD )
    PIPE_NOWAIT = wintypes . DWORD ( 0x00000001 )
    SetNamedPipeHandleState = windll . kernel32 . SetNamedPipeHandleState
    SetNamedPipeHandleState . argtypes = [ HANDLE , LPDWORD , LPDWORD , LPDWORD ]
    SetNamedPipeHandleState . restype = BOOL
    h = msvcrt . get_osfhandle ( file_obj . fileno ( ) )
    res = windll . kernel32 . SetNamedPipeHandleState ( h , byref ( PIPE_NOWAIT ) , None , None )
    if res == 0 :
        raise ValueError ( WinError ( ) )
else : # Set the file status flag ( F _ SETFL ) on the pipes to be non - blocking
# so we can attempt to read from a pipe with no new data without locking
# the program up
    fcntl . fcntl ( file_obj , fcntl . F_SETFL , os . O_NONBLOCK )


signature:
def spawn_new_gdb_subprocess ( self ) :
    """Spawn a new gdb subprocess with the arguments supplied to the object
 during initialization . If gdb subprocess already exists , terminate it before
 spanwing a new one .
 Return int : gdb process id"""

implementation:
if self . gdb_process :
    self . logger . debug ( "Killing current gdb subprocess (pid %d)" % self . gdb_process . pid )
    self . exit ( )
self . logger . debug ( 'Launching gdb: "%s"' % " " . join ( self . cmd ) )
# Use pipes to the standard streams
self . gdb_process = subprocess . Popen ( self . cmd , shell = False , stdout = subprocess . PIPE , stdin = subprocess . PIPE , stderr = subprocess . PIPE , bufsize = 0 , )
_make_non_blocking ( self . gdb_process . stdout )
_make_non_blocking ( self . gdb_process . stderr )
# save file numbers for use later
self . stdout_fileno = self . gdb_process . stdout . fileno ( )
self . stderr_fileno = self . gdb_process . stderr . fileno ( )
self . stdin_fileno = self . gdb_process . stdin . fileno ( )
self . read_list = [ self . stdout_fileno , self . stderr_fileno ]
self . write_list = [ self . stdin_fileno ]
# string buffers for unifinished gdb output
self . _incomplete_output = { "stdout" : None , "stderr" : None }
return self . gdb_process . pid


signature:
def verify_valid_gdb_subprocess ( self ) :
    """Verify there is a process object , and that it is still running .
 Raise NoGdbProcessError if either of the above are not true ."""

implementation:
if not self . gdb_process :
    raise NoGdbProcessError ( "gdb process is not attached" )
elif self . gdb_process . poll ( ) is not None :
    raise NoGdbProcessError ( "gdb process has already finished with return code: %s" % str ( self . gdb_process . poll ( ) ) )


signature:
def write ( self , mi_cmd_to_write , timeout_sec = DEFAULT_GDB_TIMEOUT_SEC , raise_error_on_timeout = True , read_response = True , ) :
    """Write to gdb process . Block while parsing responses from gdb for a maximum of timeout _ sec .
 Args :
 mi _ cmd _ to _ write ( str or list ) : String to write to gdb . If list , it is joined by newlines .
 timeout _ sec ( float ) : Maximum number of seconds to wait for response before exiting . Must be > = 0.
 raise _ error _ on _ timeout ( bool ) : If read _ response is True , raise error if no response is received
 read _ response ( bool ) : Block and read response . If there is a separate thread running ,
 this can be false , and the reading thread read the output .
 Returns :
 List of parsed gdb responses if read _ response is True , otherwise [ ]
 Raises :
 NoGdbProcessError if there is no gdb subprocess running
 TypeError if mi _ cmd _ to _ write is not valid"""

implementation:
self . verify_valid_gdb_subprocess ( )
if timeout_sec < 0 :
    self . logger . warning ( "timeout_sec was negative, replacing with 0" )
    timeout_sec = 0
# Ensure proper type of the mi command
if type ( mi_cmd_to_write ) in [ str , unicode ] :
    pass
elif type ( mi_cmd_to_write ) == list :
    mi_cmd_to_write = "\n" . join ( mi_cmd_to_write )
else :
    raise TypeError ( "The gdb mi command must a be str or list. Got " + str ( type ( mi_cmd_to_write ) ) )
self . logger . debug ( "writing: %s" , mi_cmd_to_write )
if not mi_cmd_to_write . endswith ( "\n" ) :
    mi_cmd_to_write_nl = mi_cmd_to_write + "\n"
else :
    mi_cmd_to_write_nl = mi_cmd_to_write
if USING_WINDOWS : # select not implemented in windows for pipes
# assume it ' s always ready
    outputready = [ self . stdin_fileno ]
else :
    _ , outputready , _ = select . select ( [ ] , self . write_list , [ ] , timeout_sec )
for fileno in outputready :
    if fileno == self . stdin_fileno : # ready to write
        self . gdb_process . stdin . write ( mi_cmd_to_write_nl . encode ( ) )
        # don ' t forget to flush for Python3 , otherwise gdb won ' t realize there is data
        # to evaluate , and we won ' t get a response
        self . gdb_process . stdin . flush ( )
    else :
        self . logger . error ( "got unexpected fileno %d" % fileno )
if read_response is True :
    return self . get_gdb_response ( timeout_sec = timeout_sec , raise_error_on_timeout = raise_error_on_timeout )
else :
    return [ ]