signature:
def plot_txn_time_hist ( transactions , bin_minutes = 5 , tz = 'America/New_York' , ax = None , ** kwargs ) :
    """Plots a histogram of transaction times , binning the times into
 buckets of a given duration .
 Parameters
 transactions : pd . DataFrame
 Prices and amounts of executed trades . One row per trade .
 - See full explanation in tears . create _ full _ tear _ sheet .
 bin _ minutes : float , optional
 Sizes of the bins in minutes , defaults to 5 minutes .
 tz : str , optional
 Time zone to plot against . Note that if the specified
 zone does not apply daylight savings , the distribution
 may be partially offset .
 ax : matplotlib . Axes , optional
 Axes upon which to plot .
 * * kwargs , optional
 Passed to plotting function .
 Returns
 ax : matplotlib . Axes
 The axes that were plotted on ."""

implementation:
if ax is None :
    ax = plt . gca ( )
txn_time = transactions . copy ( )
txn_time . index = txn_time . index . tz_convert ( pytz . timezone ( tz ) )
txn_time . index = txn_time . index . map ( lambda x : x . hour * 60 + x . minute )
txn_time [ 'trade_value' ] = ( txn_time . amount * txn_time . price ) . abs ( )
txn_time = txn_time . groupby ( level = 0 ) . sum ( ) . reindex ( index = range ( 570 , 961 ) )
txn_time . index = ( txn_time . index / bin_minutes ) . astype ( int ) * bin_minutes
txn_time = txn_time . groupby ( level = 0 ) . sum ( )
txn_time [ 'time_str' ] = txn_time . index . map ( lambda x : str ( datetime . time ( int ( x / 60 ) , x % 60 ) ) [ : - 3 ] )
trade_value_sum = txn_time . trade_value . sum ( )
txn_time . trade_value = txn_time . trade_value . fillna ( 0 ) / trade_value_sum
ax . bar ( txn_time . index , txn_time . trade_value , width = bin_minutes , ** kwargs )
ax . set_xlim ( 570 , 960 )
ax . set_xticks ( txn_time . index [ : : int ( 30 / bin_minutes ) ] )
ax . set_xticklabels ( txn_time . time_str [ : : int ( 30 / bin_minutes ) ] )
ax . set_title ( 'Transaction time distribution' )
ax . set_ylabel ( 'Proportion' )
ax . set_xlabel ( '' )
return ax


signature:
def show_worst_drawdown_periods ( returns , top = 5 ) :
    """Prints information about the worst drawdown periods .
 Prints peak dates , valley dates , recovery dates , and net
 drawdowns .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 top : int , optional
 Amount of top drawdowns periods to plot ( default 5 ) ."""

implementation:
drawdown_df = timeseries . gen_drawdown_table ( returns , top = top )
utils . print_table ( drawdown_df . sort_values ( 'Net drawdown in %' , ascending = False ) , name = 'Worst drawdown periods' , float_format = '{0:.2f}' . format , )


signature:
def plot_monthly_returns_timeseries ( returns , ax = None , ** kwargs ) :
    """Plots monthly returns as a timeseries .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 ax : matplotlib . Axes , optional
 Axes upon which to plot .
 * * kwargs , optional
 Passed to seaborn plotting function .
 Returns
 ax : matplotlib . Axes
 The axes that were plotted on ."""

implementation:
def cumulate_returns ( x ) :
    return ep . cum_returns ( x ) [ - 1 ]
if ax is None :
    ax = plt . gca ( )
monthly_rets = returns . resample ( 'M' ) . apply ( lambda x : cumulate_returns ( x ) )
monthly_rets = monthly_rets . to_period ( )
sns . barplot ( x = monthly_rets . index , y = monthly_rets . values , color = 'steelblue' )
locs , labels = plt . xticks ( )
plt . setp ( labels , rotation = 90 )
# only show x - labels on year boundary
xticks_coord = [ ]
xticks_label = [ ]
count = 0
for i in monthly_rets . index :
    if i . month == 1 :
        xticks_label . append ( i )
        xticks_coord . append ( count )
        # plot yearly boundary line
        ax . axvline ( count , color = 'gray' , ls = '--' , alpha = 0.3 )
    count += 1
ax . axhline ( 0.0 , color = 'darkgray' , ls = '-' )
ax . set_xticks ( xticks_coord )
ax . set_xticklabels ( xticks_label )
return ax


signature:
def plot_round_trip_lifetimes ( round_trips , disp_amount = 16 , lsize = 18 , ax = None ) :
    """Plots timespans and directions of a sample of round trip trades .
 Parameters
 round _ trips : pd . DataFrame
 DataFrame with one row per round trip trade .
 - See full explanation in round _ trips . extract _ round _ trips
 ax : matplotlib . Axes , optional
 Axes upon which to plot .
 Returns
 ax : matplotlib . Axes
 The axes that were plotted on ."""

implementation:
if ax is None :
    ax = plt . subplot ( )
symbols_sample = round_trips . symbol . unique ( )
np . random . seed ( 1 )
sample = np . random . choice ( round_trips . symbol . unique ( ) , replace = False , size = min ( disp_amount , len ( symbols_sample ) ) )
sample_round_trips = round_trips [ round_trips . symbol . isin ( sample ) ]
symbol_idx = pd . Series ( np . arange ( len ( sample ) ) , index = sample )
for symbol , sym_round_trips in sample_round_trips . groupby ( 'symbol' ) :
    for _ , row in sym_round_trips . iterrows ( ) :
        c = 'b' if row . long else 'r'
        y_ix = symbol_idx [ symbol ] + 0.05
        ax . plot ( [ row [ 'open_dt' ] , row [ 'close_dt' ] ] , [ y_ix , y_ix ] , color = c , linewidth = lsize , solid_capstyle = 'butt' )
ax . set_yticks ( range ( disp_amount ) )
ax . set_yticklabels ( [ utils . format_asset ( s ) for s in sample ] )
ax . set_ylim ( ( - 0.5 , min ( len ( sample ) , disp_amount ) - 0.5 ) )
blue = patches . Rectangle ( [ 0 , 0 ] , 1 , 1 , color = 'b' , label = 'Long' )
red = patches . Rectangle ( [ 0 , 0 ] , 1 , 1 , color = 'r' , label = 'Short' )
leg = ax . legend ( handles = [ blue , red ] , loc = 'lower left' , frameon = True , framealpha = 0.5 )
leg . get_frame ( ) . set_edgecolor ( 'black' )
ax . grid ( False )
return ax


signature:
def show_profit_attribution ( round_trips ) :
    """Prints the share of total PnL contributed by each
 traded name .
 Parameters
 round _ trips : pd . DataFrame
 DataFrame with one row per round trip trade .
 - See full explanation in round _ trips . extract _ round _ trips
 ax : matplotlib . Axes , optional
 Axes upon which to plot .
 Returns
 ax : matplotlib . Axes
 The axes that were plotted on ."""

implementation:
total_pnl = round_trips [ 'pnl' ] . sum ( )
pnl_attribution = round_trips . groupby ( 'symbol' ) [ 'pnl' ] . sum ( ) / total_pnl
pnl_attribution . name = ''
pnl_attribution . index = pnl_attribution . index . map ( utils . format_asset )
utils . print_table ( pnl_attribution . sort_values ( inplace = False , ascending = False , ) , name = 'Profitability (PnL / PnL total) per name' , float_format = '{:.2%}' . format , )


signature:
def plot_prob_profit_trade ( round_trips , ax = None ) :
    """Plots a probability distribution for the event of making
 a profitable trade .
 Parameters
 round _ trips : pd . DataFrame
 DataFrame with one row per round trip trade .
 - See full explanation in round _ trips . extract _ round _ trips
 ax : matplotlib . Axes , optional
 Axes upon which to plot .
 Returns
 ax : matplotlib . Axes
 The axes that were plotted on ."""

implementation:
x = np . linspace ( 0 , 1. , 500 )
round_trips [ 'profitable' ] = round_trips . pnl > 0
dist = sp . stats . beta ( round_trips . profitable . sum ( ) , ( ~ round_trips . profitable ) . sum ( ) )
y = dist . pdf ( x )
lower_perc = dist . ppf ( .025 )
upper_perc = dist . ppf ( .975 )
lower_plot = dist . ppf ( .001 )
upper_plot = dist . ppf ( .999 )
if ax is None :
    ax = plt . subplot ( )
ax . plot ( x , y )
ax . axvline ( lower_perc , color = '0.5' )
ax . axvline ( upper_perc , color = '0.5' )
ax . set_xlabel ( 'Probability of making a profitable decision' )
ax . set_ylabel ( 'Belief' )
ax . set_xlim ( lower_plot , upper_plot )
ax . set_ylim ( ( 0 , y . max ( ) + 1. ) )
return ax


signature:
def plot_cones ( name , bounds , oos_returns , num_samples = 1000 , ax = None , cone_std = ( 1. , 1.5 , 2. ) , random_seed = None , num_strikes = 3 ) :
    """Plots the upper and lower bounds of an n standard deviation
 cone of forecasted cumulative returns . Redraws a new cone when
 cumulative returns fall outside of last cone drawn .
 Parameters
 name : str
 Account name to be used as figure title .
 bounds : pandas . core . frame . DataFrame
 Contains upper and lower cone boundaries . Column names are
 strings corresponding to the number of standard devations
 above ( positive ) or below ( negative ) the projected mean
 cumulative returns .
 oos _ returns : pandas . core . frame . DataFrame
 Non - cumulative out - of - sample returns .
 num _ samples : int
 Number of samples to draw from the in - sample daily returns .
 Each sample will be an array with length num _ days .
 A higher number of samples will generate a more accurate
 bootstrap cone .
 ax : matplotlib . Axes , optional
 Axes upon which to plot .
 cone _ std : list of int / float
 Number of standard devations to use in the boundaries of
 the cone . If multiple values are passed , cone bounds will
 be generated for each value .
 random _ seed : int
 Seed for the pseudorandom number generator used by the pandas
 sample method .
 num _ strikes : int
 Upper limit for number of cones drawn . Can be anything from 0 to 3.
 Returns
 Returns are either an ax or fig option , but not both . If a
 matplotlib . Axes instance is passed in as ax , then it will be modified
 and returned . This allows for users to plot interactively in jupyter
 notebook . When no ax object is passed in , a matplotlib . figure instance
 is generated and returned . This figure can then be used to save
 the plot as an image without viewing it .
 ax : matplotlib . Axes
 The axes that were plotted on .
 fig : matplotlib . figure
 The figure instance which contains all the plot elements ."""

implementation:
if ax is None :
    fig = figure . Figure ( figsize = ( 10 , 8 ) )
    FigureCanvasAgg ( fig )
    axes = fig . add_subplot ( 111 )
else :
    axes = ax
returns = ep . cum_returns ( oos_returns , starting_value = 1. )
bounds_tmp = bounds . copy ( )
returns_tmp = returns . copy ( )
cone_start = returns . index [ 0 ]
colors = [ "green" , "orange" , "orangered" , "darkred" ]
for c in range ( num_strikes + 1 ) :
    if c > 0 :
        tmp = returns . loc [ cone_start : ]
        bounds_tmp = bounds_tmp . iloc [ 0 : len ( tmp ) ]
        bounds_tmp = bounds_tmp . set_index ( tmp . index )
        crossing = ( tmp < bounds_tmp [ float ( - 2. ) ] . iloc [ : len ( tmp ) ] )
        if crossing . sum ( ) <= 0 :
            break
        cone_start = crossing . loc [ crossing ] . index [ 0 ]
        returns_tmp = returns . loc [ cone_start : ]
        bounds_tmp = ( bounds - ( 1 - returns . loc [ cone_start ] ) )
    for std in cone_std :
        x = returns_tmp . index
        y1 = bounds_tmp [ float ( std ) ] . iloc [ : len ( returns_tmp ) ]
        y2 = bounds_tmp [ float ( - std ) ] . iloc [ : len ( returns_tmp ) ]
        axes . fill_between ( x , y1 , y2 , color = colors [ c ] , alpha = 0.5 )
# Plot returns line graph
label = 'Cumulative returns = {:.2f}%' . format ( ( returns . iloc [ - 1 ] - 1 ) * 100 )
axes . plot ( returns . index , returns . values , color = 'black' , lw = 3. , label = label )
if name is not None :
    axes . set_title ( name )
axes . axhline ( 1 , color = 'black' , alpha = 0.2 )
axes . legend ( frameon = True , framealpha = 0.5 )
if ax is None :
    return fig
else :
    return axes


signature:
def var_cov_var_normal ( P , c , mu = 0 , sigma = 1 ) :
    """Variance - covariance calculation of daily Value - at - Risk in a
 portfolio .
 Parameters
 P : float
 Portfolio value .
 c : float
 Confidence level .
 mu : float , optional
 Mean .
 Returns
 float
 Variance - covariance ."""

implementation:
alpha = sp . stats . norm . ppf ( 1 - c , mu , sigma )
return P - P * ( alpha + 1 )


signature:
def sortino_ratio ( returns , required_return = 0 , period = DAILY ) :
    """Determines the Sortino ratio of a strategy .
 Parameters
 returns : pd . Series or pd . DataFrame
 Daily returns of the strategy , noncumulative .
 - See full explanation in : func : ` ~ pyfolio . timeseries . cum _ returns ` .
 required _ return : float / series
 minimum acceptable return
 period : str , optional
 Defines the periodicity of the ' returns ' data for purposes of
 annualizing . Can be ' monthly ' , ' weekly ' , or ' daily ' .
 - Defaults to ' daily ' .
 Returns
 depends on input type
 series = = > float
 DataFrame = = > np . array
 Annualized Sortino ratio ."""

implementation: return ep . sortino_ratio ( returns , required_return = required_return )


signature:
def downside_risk ( returns , required_return = 0 , period = DAILY ) :
    """Determines the downside deviation below a threshold
 Parameters
 returns : pd . Series or pd . DataFrame
 Daily returns of the strategy , noncumulative .
 - See full explanation in : func : ` ~ pyfolio . timeseries . cum _ returns ` .
 required _ return : float / series
 minimum acceptable return
 period : str , optional
 Defines the periodicity of the ' returns ' data for purposes of
 annualizing . Can be ' monthly ' , ' weekly ' , or ' daily ' .
 - Defaults to ' daily ' .
 Returns
 depends on input type
 series = = > float
 DataFrame = = > np . array
 Annualized downside deviation"""

implementation: return ep . downside_risk ( returns , required_return = required_return , period = period )


signature:
def sharpe_ratio ( returns , risk_free = 0 , period = DAILY ) :
    """Determines the Sharpe ratio of a strategy .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in : func : ` ~ pyfolio . timeseries . cum _ returns ` .
 risk _ free : int , float
 Constant risk - free return throughout the period .
 period : str , optional
 Defines the periodicity of the ' returns ' data for purposes of
 annualizing . Can be ' monthly ' , ' weekly ' , or ' daily ' .
 - Defaults to ' daily ' .
 Returns
 float
 Sharpe ratio .
 np . nan
 If insufficient length of returns or if if adjusted returns are 0.
 Note
 See https : / / en . wikipedia . org / wiki / Sharpe _ ratio for more details ."""

implementation: return ep . sharpe_ratio ( returns , risk_free = risk_free , period = period )


signature:
def rolling_beta ( returns , factor_returns , rolling_window = APPROX_BDAYS_PER_MONTH * 6 ) :
    """Determines the rolling beta of a strategy .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 factor _ returns : pd . Series or pd . DataFrame
 Daily noncumulative returns of the benchmark factor to which betas are
 computed . Usually a benchmark such as market returns .
 - If DataFrame is passed , computes rolling beta for each column .
 - This is in the same style as returns .
 rolling _ window : int , optional
 The size of the rolling window , in days , over which to compute
 beta ( default 6 months ) .
 Returns
 pd . Series
 Rolling beta .
 Note
 See https : / / en . wikipedia . org / wiki / Beta _ ( finance ) for more details ."""

implementation:
if factor_returns . ndim > 1 : # Apply column - wise
    return factor_returns . apply ( partial ( rolling_beta , returns ) , rolling_window = rolling_window )
else :
    out = pd . Series ( index = returns . index )
    for beg , end in zip ( returns . index [ 0 : - rolling_window ] , returns . index [ rolling_window : ] ) :
        out . loc [ end ] = ep . beta ( returns . loc [ beg : end ] , factor_returns . loc [ beg : end ] )
    return out


signature:
def rolling_regression ( returns , factor_returns , rolling_window = APPROX_BDAYS_PER_MONTH * 6 , nan_threshold = 0.1 ) :
    """Computes rolling factor betas using a multivariate linear regression
 ( separate linear regressions is problematic because the factors may be
 confounded ) .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 factor _ returns : pd . DataFrame
 Daily noncumulative returns of the benchmark factor to which betas are
 computed . Usually a benchmark such as market returns .
 - Computes rolling beta for each column .
 - This is in the same style as returns .
 rolling _ window : int , optional
 The days window over which to compute the beta . Defaults to 6 months .
 nan _ threshold : float , optional
 If there are more than this fraction of NaNs , the rolling regression
 for the given date will be skipped .
 Returns
 pandas . DataFrame
 DataFrame containing rolling beta coefficients to SMB , HML and UMD"""

implementation:
# We need to drop NaNs to regress
ret_no_na = returns . dropna ( )
columns = [ 'alpha' ] + factor_returns . columns . tolist ( )
rolling_risk = pd . DataFrame ( columns = columns , index = ret_no_na . index )
rolling_risk . index . name = 'dt'
for beg , end in zip ( ret_no_na . index [ : - rolling_window ] , ret_no_na . index [ rolling_window : ] ) :
    returns_period = ret_no_na [ beg : end ]
    factor_returns_period = factor_returns . loc [ returns_period . index ]
    if np . all ( factor_returns_period . isnull ( ) . mean ( ) ) < nan_threshold :
        factor_returns_period_dnan = factor_returns_period . dropna ( )
        reg = linear_model . LinearRegression ( fit_intercept = True ) . fit ( factor_returns_period_dnan , returns_period . loc [ factor_returns_period_dnan . index ] )
        rolling_risk . loc [ end , factor_returns . columns ] = reg . coef_
        rolling_risk . loc [ end , 'alpha' ] = reg . intercept_
return rolling_risk


signature:
def gross_lev ( positions ) :
    """Calculates the gross leverage of a strategy .
 Parameters
 positions : pd . DataFrame
 Daily net position values .
 - See full explanation in tears . create _ full _ tear _ sheet .
 Returns
 pd . Series
 Gross leverage ."""

implementation:
exposure = positions . drop ( 'cash' , axis = 1 ) . abs ( ) . sum ( axis = 1 )
return exposure / positions . sum ( axis = 1 )


signature:
def value_at_risk ( returns , period = None , sigma = 2.0 ) :
    """Get value at risk ( VaR ) .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 period : str , optional
 Period over which to calculate VaR . Set to ' weekly ' ,
 ' monthly ' , or ' yearly ' , otherwise defaults to period of
 returns ( typically daily ) .
 sigma : float , optional
 Standard deviations of VaR , default 2."""

implementation:
if period is not None :
    returns_agg = ep . aggregate_returns ( returns , period )
else :
    returns_agg = returns . copy ( )
value_at_risk = returns_agg . mean ( ) - sigma * returns_agg . std ( )
return value_at_risk


signature:
def perf_stats ( returns , factor_returns = None , positions = None , transactions = None , turnover_denom = 'AGB' ) :
    """Calculates various performance metrics of a strategy , for use in
 plotting . show _ perf _ stats .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 factor _ returns : pd . Series , optional
 Daily noncumulative returns of the benchmark factor to which betas are
 computed . Usually a benchmark such as market returns .
 - This is in the same style as returns .
 - If None , do not compute alpha , beta , and information ratio .
 positions : pd . DataFrame
 Daily net position values .
 - See full explanation in tears . create _ full _ tear _ sheet .
 transactions : pd . DataFrame
 Prices and amounts of executed trades . One row per trade .
 - See full explanation in tears . create _ full _ tear _ sheet .
 turnover _ denom : str
 Either AGB or portfolio _ value , default AGB .
 - See full explanation in txn . get _ turnover .
 Returns
 pd . Series
 Performance metrics ."""

implementation:
stats = pd . Series ( )
for stat_func in SIMPLE_STAT_FUNCS :
    stats [ STAT_FUNC_NAMES [ stat_func . __name__ ] ] = stat_func ( returns )
if positions is not None :
    stats [ 'Gross leverage' ] = gross_lev ( positions ) . mean ( )
    if transactions is not None :
        stats [ 'Daily turnover' ] = get_turnover ( positions , transactions , turnover_denom ) . mean ( )
if factor_returns is not None :
    for stat_func in FACTOR_STAT_FUNCS :
        res = stat_func ( returns , factor_returns )
        stats [ STAT_FUNC_NAMES [ stat_func . __name__ ] ] = res
return stats


signature:
def perf_stats_bootstrap ( returns , factor_returns = None , return_stats = True , ** kwargs ) :
    """Calculates various bootstrapped performance metrics of a strategy .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 factor _ returns : pd . Series , optional
 Daily noncumulative returns of the benchmark factor to which betas are
 computed . Usually a benchmark such as market returns .
 - This is in the same style as returns .
 - If None , do not compute alpha , beta , and information ratio .
 return _ stats : boolean ( optional )
 If True , returns a DataFrame of mean , median , 5 and 95 percentiles
 for each perf metric .
 If False , returns a DataFrame with the bootstrap samples for
 each perf metric .
 Returns
 pd . DataFrame
 if return _ stats is True :
 - Distributional statistics of bootstrapped sampling
 distribution of performance metrics .
 if return _ stats is False :
 - Bootstrap samples for each performance metric ."""

implementation:
bootstrap_values = OrderedDict ( )
for stat_func in SIMPLE_STAT_FUNCS :
    stat_name = STAT_FUNC_NAMES [ stat_func . __name__ ]
    bootstrap_values [ stat_name ] = calc_bootstrap ( stat_func , returns )
if factor_returns is not None :
    for stat_func in FACTOR_STAT_FUNCS :
        stat_name = STAT_FUNC_NAMES [ stat_func . __name__ ]
        bootstrap_values [ stat_name ] = calc_bootstrap ( stat_func , returns , factor_returns = factor_returns )
bootstrap_values = pd . DataFrame ( bootstrap_values )
if return_stats :
    stats = bootstrap_values . apply ( calc_distribution_stats )
    return stats . T [ [ 'mean' , 'median' , '5%' , '95%' ] ]
else :
    return bootstrap_values


signature:
def calc_bootstrap ( func , returns , * args , ** kwargs ) :
    """Performs a bootstrap analysis on a user - defined function returning
 a summary statistic .
 Parameters
 func : function
 Function that either takes a single array ( commonly returns )
 or two arrays ( commonly returns and factor returns ) and
 returns a single value ( commonly a summary
 statistic ) . Additional args and kwargs are passed as well .
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 factor _ returns : pd . Series , optional
 Daily noncumulative returns of the benchmark factor to which betas are
 computed . Usually a benchmark such as market returns .
 - This is in the same style as returns .
 n _ samples : int , optional
 Number of bootstrap samples to draw . Default is 1000.
 Increasing this will lead to more stable / accurate estimates .
 Returns
 numpy . ndarray
 Bootstrapped sampling distribution of passed in func ."""

implementation:
n_samples = kwargs . pop ( 'n_samples' , 1000 )
out = np . empty ( n_samples )
factor_returns = kwargs . pop ( 'factor_returns' , None )
for i in range ( n_samples ) :
    idx = np . random . randint ( len ( returns ) , size = len ( returns ) )
    returns_i = returns . iloc [ idx ] . reset_index ( drop = True )
    if factor_returns is not None :
        factor_returns_i = factor_returns . iloc [ idx ] . reset_index ( drop = True )
        out [ i ] = func ( returns_i , factor_returns_i , * args , ** kwargs )
    else :
        out [ i ] = func ( returns_i , * args , ** kwargs )
return out


signature:
def calc_distribution_stats ( x ) :
    """Calculate various summary statistics of data .
 Parameters
 x : numpy . ndarray or pandas . Series
 Array to compute summary statistics for .
 Returns
 pandas . Series
 Series containing mean , median , std , as well as 5 , 25 , 75 and
 95 percentiles of passed in values ."""

implementation: return pd . Series ( { 'mean' : np . mean ( x ) , 'median' : np . median ( x ) , 'std' : np . std ( x ) , '5%' : np . percentile ( x , 5 ) , '25%' : np . percentile ( x , 25 ) , '75%' : np . percentile ( x , 75 ) , '95%' : np . percentile ( x , 95 ) , 'IQR' : np . subtract . reduce ( np . percentile ( x , [ 75 , 25 ] ) ) , } )


signature:
def get_max_drawdown_underwater ( underwater ) :
    """Determines peak , valley , and recovery dates given an ' underwater '
 DataFrame .
 An underwater DataFrame is a DataFrame that has precomputed
 rolling drawdown .
 Parameters
 underwater : pd . Series
 Underwater returns ( rolling drawdown ) of a strategy .
 Returns
 peak : datetime
 The maximum drawdown ' s peak .
 valley : datetime
 The maximum drawdown ' s valley .
 recovery : datetime
 The maximum drawdown ' s recovery ."""

implementation:
valley = np . argmin ( underwater )
# end of the period
# Find first 0
peak = underwater [ : valley ] [ underwater [ : valley ] == 0 ] . index [ - 1 ]
# Find last 0
try :
    recovery = underwater [ valley : ] [ underwater [ valley : ] == 0 ] . index [ 0 ]
except IndexError :
    recovery = np . nan
    # drawdown not recovered
return peak , valley , recovery


signature:
def get_max_drawdown ( returns ) :
    """Determines the maximum drawdown of a strategy .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in : func : ` ~ pyfolio . timeseries . cum _ returns ` .
 Returns
 float
 Maximum drawdown .
 Note
 See https : / / en . wikipedia . org / wiki / Drawdown _ ( economics ) for more details ."""

implementation:
returns = returns . copy ( )
df_cum = cum_returns ( returns , 1.0 )
running_max = np . maximum . accumulate ( df_cum )
underwater = df_cum / running_max - 1
return get_max_drawdown_underwater ( underwater )


signature:
def get_top_drawdowns ( returns , top = 10 ) :
    """Finds top drawdowns , sorted by drawdown amount .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 top : int , optional
 The amount of top drawdowns to find ( default 10 ) .
 Returns
 drawdowns : list
 List of drawdown peaks , valleys , and recoveries . See get _ max _ drawdown ."""

implementation:
returns = returns . copy ( )
df_cum = ep . cum_returns ( returns , 1.0 )
running_max = np . maximum . accumulate ( df_cum )
underwater = df_cum / running_max - 1
drawdowns = [ ]
for t in range ( top ) :
    peak , valley , recovery = get_max_drawdown_underwater ( underwater )
    # Slice out draw - down period
    if not pd . isnull ( recovery ) :
        underwater . drop ( underwater [ peak : recovery ] . index [ 1 : - 1 ] , inplace = True )
    else : # drawdown has not ended yet
        underwater = underwater . loc [ : peak ]
    drawdowns . append ( ( peak , valley , recovery ) )
    if ( len ( returns ) == 0 ) or ( len ( underwater ) == 0 ) :
        break
return drawdowns


signature:
def gen_drawdown_table ( returns , top = 10 ) :
    """Places top drawdowns in a table .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 top : int , optional
 The amount of top drawdowns to find ( default 10 ) .
 Returns
 df _ drawdowns : pd . DataFrame
 Information about top drawdowns ."""

implementation:
df_cum = ep . cum_returns ( returns , 1.0 )
drawdown_periods = get_top_drawdowns ( returns , top = top )
df_drawdowns = pd . DataFrame ( index = list ( range ( top ) ) , columns = [ 'Net drawdown in %' , 'Peak date' , 'Valley date' , 'Recovery date' , 'Duration' ] )
for i , ( peak , valley , recovery ) in enumerate ( drawdown_periods ) :
    if pd . isnull ( recovery ) :
        df_drawdowns . loc [ i , 'Duration' ] = np . nan
    else :
        df_drawdowns . loc [ i , 'Duration' ] = len ( pd . date_range ( peak , recovery , freq = 'B' ) )
    df_drawdowns . loc [ i , 'Peak date' ] = ( peak . to_pydatetime ( ) . strftime ( '%Y-%m-%d' ) )
    df_drawdowns . loc [ i , 'Valley date' ] = ( valley . to_pydatetime ( ) . strftime ( '%Y-%m-%d' ) )
    if isinstance ( recovery , float ) :
        df_drawdowns . loc [ i , 'Recovery date' ] = recovery
    else :
        df_drawdowns . loc [ i , 'Recovery date' ] = ( recovery . to_pydatetime ( ) . strftime ( '%Y-%m-%d' ) )
    df_drawdowns . loc [ i , 'Net drawdown in %' ] = ( ( df_cum . loc [ peak ] - df_cum . loc [ valley ] ) / df_cum . loc [ peak ] ) * 100
df_drawdowns [ 'Peak date' ] = pd . to_datetime ( df_drawdowns [ 'Peak date' ] )
df_drawdowns [ 'Valley date' ] = pd . to_datetime ( df_drawdowns [ 'Valley date' ] )
df_drawdowns [ 'Recovery date' ] = pd . to_datetime ( df_drawdowns [ 'Recovery date' ] )
return df_drawdowns


signature:
def rolling_volatility ( returns , rolling_vol_window ) :
    """Determines the rolling volatility of a strategy .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 rolling _ vol _ window : int
 Length of rolling window , in days , over which to compute .
 Returns
 pd . Series
 Rolling volatility ."""

implementation: return returns . rolling ( rolling_vol_window ) . std ( ) * np . sqrt ( APPROX_BDAYS_PER_YEAR )


signature:
def rolling_sharpe ( returns , rolling_sharpe_window ) :
    """Determines the rolling Sharpe ratio of a strategy .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 rolling _ sharpe _ window : int
 Length of rolling window , in days , over which to compute .
 Returns
 pd . Series
 Rolling Sharpe ratio .
 Note
 See https : / / en . wikipedia . org / wiki / Sharpe _ ratio for more details ."""

implementation: return returns . rolling ( rolling_sharpe_window ) . mean ( ) / returns . rolling ( rolling_sharpe_window ) . std ( ) * np . sqrt ( APPROX_BDAYS_PER_YEAR )


signature:
def simulate_paths ( is_returns , num_days , starting_value = 1 , num_samples = 1000 , random_seed = None ) :
    """Gnerate alternate paths using available values from in - sample returns .
 Parameters
 is _ returns : pandas . core . frame . DataFrame
 Non - cumulative in - sample returns .
 num _ days : int
 Number of days to project the probability cone forward .
 starting _ value : int or float
 Starting value of the out of sample period .
 num _ samples : int
 Number of samples to draw from the in - sample daily returns .
 Each sample will be an array with length num _ days .
 A higher number of samples will generate a more accurate
 bootstrap cone .
 random _ seed : int
 Seed for the pseudorandom number generator used by the pandas
 sample method .
 Returns
 samples : numpy . ndarray"""

implementation:
samples = np . empty ( ( num_samples , num_days ) )
seed = np . random . RandomState ( seed = random_seed )
for i in range ( num_samples ) :
    samples [ i , : ] = is_returns . sample ( num_days , replace = True , random_state = seed )
return samples


signature:
def summarize_paths ( samples , cone_std = ( 1. , 1.5 , 2. ) , starting_value = 1. ) :
    """Gnerate the upper and lower bounds of an n standard deviation
 cone of forecasted cumulative returns .
 Parameters
 samples : numpy . ndarray
 Alternative paths , or series of possible outcomes .
 cone _ std : list of int / float
 Number of standard devations to use in the boundaries of
 the cone . If multiple values are passed , cone bounds will
 be generated for each value .
 Returns
 samples : pandas . core . frame . DataFrame"""

implementation:
cum_samples = ep . cum_returns ( samples . T , starting_value = starting_value ) . T
cum_mean = cum_samples . mean ( axis = 0 )
cum_std = cum_samples . std ( axis = 0 )
if isinstance ( cone_std , ( float , int ) ) :
    cone_std = [ cone_std ]
cone_bounds = pd . DataFrame ( columns = pd . Float64Index ( [ ] ) )
for num_std in cone_std :
    cone_bounds . loc [ : , float ( num_std ) ] = cum_mean + cum_std * num_std
    cone_bounds . loc [ : , float ( - num_std ) ] = cum_mean - cum_std * num_std
return cone_bounds


signature:
def forecast_cone_bootstrap ( is_returns , num_days , cone_std = ( 1. , 1.5 , 2. ) , starting_value = 1 , num_samples = 1000 , random_seed = None ) :
    """Determines the upper and lower bounds of an n standard deviation
 cone of forecasted cumulative returns . Future cumulative mean and
 standard devation are computed by repeatedly sampling from the
 in - sample daily returns ( i . e . bootstrap ) . This cone is non - parametric ,
 meaning it does not assume that returns are normally distributed .
 Parameters
 is _ returns : pd . Series
 In - sample daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 num _ days : int
 Number of days to project the probability cone forward .
 cone _ std : int , float , or list of int / float
 Number of standard devations to use in the boundaries of
 the cone . If multiple values are passed , cone bounds will
 be generated for each value .
 starting _ value : int or float
 Starting value of the out of sample period .
 num _ samples : int
 Number of samples to draw from the in - sample daily returns .
 Each sample will be an array with length num _ days .
 A higher number of samples will generate a more accurate
 bootstrap cone .
 random _ seed : int
 Seed for the pseudorandom number generator used by the pandas
 sample method .
 Returns
 pd . DataFrame
 Contains upper and lower cone boundaries . Column names are
 strings corresponding to the number of standard devations
 above ( positive ) or below ( negative ) the projected mean
 cumulative returns ."""

implementation:
samples = simulate_paths ( is_returns = is_returns , num_days = num_days , starting_value = starting_value , num_samples = num_samples , random_seed = random_seed )
cone_bounds = summarize_paths ( samples = samples , cone_std = cone_std , starting_value = starting_value )
return cone_bounds


signature:
def extract_interesting_date_ranges ( returns ) :
    """Extracts returns based on interesting events . See
 gen _ date _ range _ interesting .
 Parameters
 returns : pd . Series
 Daily returns of the strategy , noncumulative .
 - See full explanation in tears . create _ full _ tear _ sheet .
 Returns
 ranges : OrderedDict
 Date ranges , with returns , of all valid events ."""

implementation:
returns_dupe = returns . copy ( )
returns_dupe . index = returns_dupe . index . map ( pd . Timestamp )
ranges = OrderedDict ( )
for name , ( start , end ) in PERIODS . items ( ) :
    try :
        period = returns_dupe . loc [ start : end ]
        if len ( period ) == 0 :
            continue
        ranges [ name ] = period
    except BaseException :
        continue
return ranges


signature:
def model_returns_t_alpha_beta ( data , bmark , samples = 2000 , progressbar = True ) :
    """Run Bayesian alpha - beta - model with T distributed returns .
 This model estimates intercept ( alpha ) and slope ( beta ) of two
 return sets . Usually , these will be algorithm returns and
 benchmark returns ( e . g . S & P500 ) . The data is assumed to be T
 distributed and thus is robust to outliers and takes tail events
 into account . If a pandas . DataFrame is passed as a benchmark , then
 multiple linear regression is used to estimate alpha and beta .
 Parameters
 returns : pandas . Series
 Series of simple returns of an algorithm or stock .
 bmark : pandas . DataFrame
 DataFrame of benchmark returns ( e . g . , S & P500 ) or risk factors ( e . g . ,
 Fama - French SMB , HML , and UMD ) .
 If bmark has more recent returns than returns _ train , these dates
 will be treated as missing values and predictions will be
 generated for them taking market correlations into account .
 samples : int ( optional )
 Number of posterior samples to draw .
 Returns
 model : pymc . Model object
 PyMC3 model containing all random variables .
 trace : pymc3 . sampling . BaseTrace object
 A PyMC3 trace object that contains samples for each parameter
 of the posterior ."""

implementation:
data_bmark = pd . concat ( [ data , bmark ] , axis = 1 ) . dropna ( )
with pm . Model ( ) as model :
    sigma = pm . HalfCauchy ( 'sigma' , beta = 1 )
    nu = pm . Exponential ( 'nu_minus_two' , 1. / 10. )
    # alpha and beta
    X = data_bmark . iloc [ : , 1 ]
    y = data_bmark . iloc [ : , 0 ]
    alpha_reg = pm . Normal ( 'alpha' , mu = 0 , sd = .1 )
    beta_reg = pm . Normal ( 'beta' , mu = 0 , sd = 1 )
    mu_reg = alpha_reg + beta_reg * X
    pm . StudentT ( 'returns' , nu = nu + 2 , mu = mu_reg , sd = sigma , observed = y )
    trace = pm . sample ( samples , progressbar = progressbar )
return model , trace


signature:
def model_returns_normal ( data , samples = 500 , progressbar = True ) :
    """Run Bayesian model assuming returns are normally distributed .
 Parameters
 returns : pandas . Series
 Series of simple returns of an algorithm or stock .
 samples : int ( optional )
 Number of posterior samples to draw .
 Returns
 model : pymc . Model object
 PyMC3 model containing all random variables .
 trace : pymc3 . sampling . BaseTrace object
 A PyMC3 trace object that contains samples for each parameter
 of the posterior ."""

implementation:
with pm . Model ( ) as model :
    mu = pm . Normal ( 'mean returns' , mu = 0 , sd = .01 , testval = data . mean ( ) )
    sigma = pm . HalfCauchy ( 'volatility' , beta = 1 , testval = data . std ( ) )
    returns = pm . Normal ( 'returns' , mu = mu , sd = sigma , observed = data )
    pm . Deterministic ( 'annual volatility' , returns . distribution . variance ** .5 * np . sqrt ( 252 ) )
    pm . Deterministic ( 'sharpe' , returns . distribution . mean / returns . distribution . variance ** .5 * np . sqrt ( 252 ) )
    trace = pm . sample ( samples , progressbar = progressbar )
return model , trace


signature:
def model_best ( y1 , y2 , samples = 1000 , progressbar = True ) :
    """Bayesian Estimation Supersedes the T - Test
 This model runs a Bayesian hypothesis comparing if y1 and y2 come
 from the same distribution . Returns are assumed to be T - distributed .
 In addition , computes annual volatility and Sharpe of in and
 out - of - sample periods .
 This model replicates the example used in :
 Kruschke , John . ( 2012 ) Bayesian estimation supersedes the t
 test . Journal of Experimental Psychology : General .
 Parameters
 y1 : array - like
 Array of returns ( e . g . in - sample )
 y2 : array - like
 Array of returns ( e . g . out - of - sample )
 samples : int , optional
 Number of posterior samples to draw .
 Returns
 model : pymc . Model object
 PyMC3 model containing all random variables .
 trace : pymc3 . sampling . BaseTrace object
 A PyMC3 trace object that contains samples for each parameter
 of the posterior .
 See Also
 plot _ stoch _ vol : plotting of tochastic volatility model"""

implementation:
y = np . concatenate ( ( y1 , y2 ) )
mu_m = np . mean ( y )
mu_p = 0.000001 * 1 / np . std ( y ) ** 2
sigma_low = np . std ( y ) / 1000
sigma_high = np . std ( y ) * 1000
with pm . Model ( ) as model :
    group1_mean = pm . Normal ( 'group1_mean' , mu = mu_m , tau = mu_p , testval = y1 . mean ( ) )
    group2_mean = pm . Normal ( 'group2_mean' , mu = mu_m , tau = mu_p , testval = y2 . mean ( ) )
    group1_std = pm . Uniform ( 'group1_std' , lower = sigma_low , upper = sigma_high , testval = y1 . std ( ) )
    group2_std = pm . Uniform ( 'group2_std' , lower = sigma_low , upper = sigma_high , testval = y2 . std ( ) )
    nu = pm . Exponential ( 'nu_minus_two' , 1 / 29. , testval = 4. ) + 2.
    returns_group1 = pm . StudentT ( 'group1' , nu = nu , mu = group1_mean , lam = group1_std ** - 2 , observed = y1 )
    returns_group2 = pm . StudentT ( 'group2' , nu = nu , mu = group2_mean , lam = group2_std ** - 2 , observed = y2 )
    diff_of_means = pm . Deterministic ( 'difference of means' , group2_mean - group1_mean )
    pm . Deterministic ( 'difference of stds' , group2_std - group1_std )
    pm . Deterministic ( 'effect size' , diff_of_means / pm . math . sqrt ( ( group1_std ** 2 + group2_std ** 2 ) / 2 ) )
    pm . Deterministic ( 'group1_annual_volatility' , returns_group1 . distribution . variance ** .5 * np . sqrt ( 252 ) )
    pm . Deterministic ( 'group2_annual_volatility' , returns_group2 . distribution . variance ** .5 * np . sqrt ( 252 ) )
    pm . Deterministic ( 'group1_sharpe' , returns_group1 . distribution . mean / returns_group1 . distribution . variance ** .5 * np . sqrt ( 252 ) )
    pm . Deterministic ( 'group2_sharpe' , returns_group2 . distribution . mean / returns_group2 . distribution . variance ** .5 * np . sqrt ( 252 ) )
    trace = pm . sample ( samples , progressbar = progressbar )
return model , trace


signature:
def plot_best ( trace = None , data_train = None , data_test = None , samples = 1000 , burn = 200 , axs = None ) :
    """Plot BEST significance analysis .
 Parameters
 trace : pymc3 . sampling . BaseTrace , optional
 trace object as returned by model _ best ( )
 If not passed , will run model _ best ( ) , for which
 data _ train and data _ test are required .
 data _ train : pandas . Series , optional
 Returns of in - sample period .
 Required if trace = None .
 data _ test : pandas . Series , optional
 Returns of out - of - sample period .
 Required if trace = None .
 samples : int , optional
 Posterior samples to draw .
 burn : int
 Posterior sampels to discard as burn - in .
 axs : array of matplotlib . axes objects , optional
 Plot into passed axes objects . Needs 6 axes .
 Returns
 None
 See Also
 model _ best : Estimation of BEST model ."""

implementation:
if trace is None :
    if ( data_train is not None ) or ( data_test is not None ) :
        raise ValueError ( 'Either pass trace or data_train and data_test' )
    trace = model_best ( data_train , data_test , samples = samples )
trace = trace [ burn : ]
if axs is None :
    fig , axs = plt . subplots ( ncols = 2 , nrows = 3 , figsize = ( 16 , 4 ) )
def distplot_w_perc ( trace , ax ) :
    sns . distplot ( trace , ax = ax )
    ax . axvline ( stats . scoreatpercentile ( trace , 2.5 ) , color = '0.5' , label = '2.5 and 97.5 percentiles' )
    ax . axvline ( stats . scoreatpercentile ( trace , 97.5 ) , color = '0.5' )
sns . distplot ( trace [ 'group1_mean' ] , ax = axs [ 0 ] , label = 'Backtest' )
sns . distplot ( trace [ 'group2_mean' ] , ax = axs [ 0 ] , label = 'Forward' )
axs [ 0 ] . legend ( loc = 0 , frameon = True , framealpha = 0.5 )
axs [ 1 ] . legend ( loc = 0 , frameon = True , framealpha = 0.5 )
distplot_w_perc ( trace [ 'difference of means' ] , axs [ 1 ] )
axs [ 0 ] . set ( xlabel = 'Mean' , ylabel = 'Belief' , yticklabels = [ ] )
axs [ 1 ] . set ( xlabel = 'Difference of means' , yticklabels = [ ] )
sns . distplot ( trace [ 'group1_annual_volatility' ] , ax = axs [ 2 ] , label = 'Backtest' )
sns . distplot ( trace [ 'group2_annual_volatility' ] , ax = axs [ 2 ] , label = 'Forward' )
distplot_w_perc ( trace [ 'group2_annual_volatility' ] - trace [ 'group1_annual_volatility' ] , axs [ 3 ] )
axs [ 2 ] . set ( xlabel = 'Annual volatility' , ylabel = 'Belief' , yticklabels = [ ] )
axs [ 2 ] . legend ( loc = 0 , frameon = True , framealpha = 0.5 )
axs [ 3 ] . set ( xlabel = 'Difference of volatility' , yticklabels = [ ] )
sns . distplot ( trace [ 'group1_sharpe' ] , ax = axs [ 4 ] , label = 'Backtest' )
sns . distplot ( trace [ 'group2_sharpe' ] , ax = axs [ 4 ] , label = 'Forward' )
distplot_w_perc ( trace [ 'group2_sharpe' ] - trace [ 'group1_sharpe' ] , axs [ 5 ] )
axs [ 4 ] . set ( xlabel = 'Sharpe' , ylabel = 'Belief' , yticklabels = [ ] )
axs [ 4 ] . legend ( loc = 0 , frameon = True , framealpha = 0.5 )
axs [ 5 ] . set ( xlabel = 'Difference of Sharpes' , yticklabels = [ ] )
sns . distplot ( trace [ 'effect size' ] , ax = axs [ 6 ] )
axs [ 6 ] . axvline ( stats . scoreatpercentile ( trace [ 'effect size' ] , 2.5 ) , color = '0.5' )
axs [ 6 ] . axvline ( stats . scoreatpercentile ( trace [ 'effect size' ] , 97.5 ) , color = '0.5' )
axs [ 6 ] . set ( xlabel = 'Difference of means normalized by volatility' , ylabel = 'Belief' , yticklabels = [ ] )


signature:
def model_stoch_vol ( data , samples = 2000 , progressbar = True ) :
    """Run stochastic volatility model .
 This model estimates the volatility of a returns series over time .
 Returns are assumed to be T - distributed . lambda ( width of
 T - distributed ) is assumed to follow a random - walk .
 Parameters
 data : pandas . Series
 Return series to model .
 samples : int , optional
 Posterior samples to draw .
 Returns
 model : pymc . Model object
 PyMC3 model containing all random variables .
 trace : pymc3 . sampling . BaseTrace object
 A PyMC3 trace object that contains samples for each parameter
 of the posterior .
 See Also
 plot _ stoch _ vol : plotting of tochastic volatility model"""

implementation:
from pymc3 . distributions . timeseries import GaussianRandomWalk
with pm . Model ( ) as model :
    nu = pm . Exponential ( 'nu' , 1. / 10 , testval = 5. )
    sigma = pm . Exponential ( 'sigma' , 1. / .02 , testval = .1 )
    s = GaussianRandomWalk ( 's' , sigma ** - 2 , shape = len ( data ) )
    volatility_process = pm . Deterministic ( 'volatility_process' , pm . math . exp ( - 2 * s ) )
    pm . StudentT ( 'r' , nu , lam = volatility_process , observed = data )
    trace = pm . sample ( samples , progressbar = progressbar )
return model , trace


signature:
def plot_stoch_vol ( data , trace = None , ax = None ) :
    """Generate plot for stochastic volatility model .
 Parameters
 data : pandas . Series
 Returns to model .
 trace : pymc3 . sampling . BaseTrace object , optional
 trace as returned by model _ stoch _ vol
 If not passed , sample from model .
 ax : matplotlib . axes object , optional
 Plot into axes object
 Returns
 ax object
 See Also
 model _ stoch _ vol : run stochastic volatility model"""

implementation:
if trace is None :
    trace = model_stoch_vol ( data )
if ax is None :
    fig , ax = plt . subplots ( figsize = ( 15 , 8 ) )
data . abs ( ) . plot ( ax = ax )
ax . plot ( data . index , np . exp ( trace [ 's' , : : 30 ] . T ) , 'r' , alpha = .03 )
ax . set ( title = 'Stochastic volatility' , xlabel = 'Time' , ylabel = 'Volatility' )
ax . legend ( [ 'Abs returns' , 'Stochastic volatility process' ] , frameon = True , framealpha = 0.5 )
return ax


signature:
def compute_bayes_cone ( preds , starting_value = 1. ) :
    """Compute 5 , 25 , 75 and 95 percentiles of cumulative returns , used
 for the Bayesian cone .
 Parameters
 preds : numpy . array
 Multiple ( simulated ) cumulative returns .
 starting _ value : int ( optional )
 Have cumulative returns start around this value .
 Default = 1.
 Returns
 dict of percentiles over time
 Dictionary mapping percentiles ( 5 , 25 , 75 , 95 ) to a
 timeseries ."""

implementation:
def scoreatpercentile ( cum_preds , p ) :
    return [ stats . scoreatpercentile ( c , p ) for c in cum_preds . T ]
cum_preds = np . cumprod ( preds + 1 , 1 ) * starting_value
perc = { p : scoreatpercentile ( cum_preds , p ) for p in ( 5 , 25 , 75 , 95 ) }
return perc


signature:
def compute_consistency_score ( returns_test , preds ) :
    """Compute Bayesian consistency score .
 Parameters
 returns _ test : pd . Series
 Observed cumulative returns .
 preds : numpy . array
 Multiple ( simulated ) cumulative returns .
 Returns
 Consistency score
 Score from 100 ( returns _ test perfectly on the median line of the
 Bayesian cone spanned by preds ) to 0 ( returns _ test completely
 outside of Bayesian cone . )"""

implementation:
returns_test_cum = cum_returns ( returns_test , starting_value = 1. )
cum_preds = np . cumprod ( preds + 1 , 1 )
q = [ sp . stats . percentileofscore ( cum_preds [ : , i ] , returns_test_cum . iloc [ i ] , kind = 'weak' ) for i in range ( len ( returns_test_cum ) ) ]
# normalize to be from 100 ( perfect median line ) to 0 ( completely outside
# of cone )
return 100 - np . abs ( 50 - np . mean ( q ) ) / .5


signature:
def run_model ( model , returns_train , returns_test = None , bmark = None , samples = 500 , ppc = False , progressbar = True ) :
    """Run one of the Bayesian models .
 Parameters
 model : { ' alpha _ beta ' , ' t ' , ' normal ' , ' best ' }
 Which model to run
 returns _ train : pd . Series
 Timeseries of simple returns
 returns _ test : pd . Series ( optional )
 Out - of - sample returns . Datetimes in returns _ test will be added to
 returns _ train as missing values and predictions will be generated
 for them .
 bmark : pd . Series or pd . DataFrame ( optional )
 Only used for alpha _ beta to estimate regression coefficients .
 If bmark has more recent returns than returns _ train , these dates
 will be treated as missing values and predictions will be
 generated for them taking market correlations into account .
 samples : int ( optional )
 Number of posterior samples to draw .
 ppc : boolean ( optional )
 Whether to run a posterior predictive check . Will generate
 samples of length returns _ test . Returns a second argument
 that contains the PPC of shape samples x len ( returns _ test ) .
 Returns
 trace : pymc3 . sampling . BaseTrace object
 A PyMC3 trace object that contains samples for each parameter
 of the posterior .
 ppc : numpy . array ( if ppc = = True )
 PPC of shape samples x len ( returns _ test ) ."""

implementation:
if model == 'alpha_beta' :
    model , trace = model_returns_t_alpha_beta ( returns_train , bmark , samples , progressbar = progressbar )
elif model == 't' :
    model , trace = model_returns_t ( returns_train , samples , progressbar = progressbar )
elif model == 'normal' :
    model , trace = model_returns_normal ( returns_train , samples , progressbar = progressbar )
elif model == 'best' :
    model , trace = model_best ( returns_train , returns_test , samples = samples , progressbar = progressbar )
else :
    raise NotImplementedError ( 'Model {} not found.' 'Use alpha_beta, t, normal, or best.' . format ( model ) )
if ppc :
    ppc_samples = pm . sample_ppc ( trace , samples = samples , model = model , size = len ( returns_test ) , progressbar = progressbar )
    return trace , ppc_samples [ 'returns' ]
return trace


signature:
def plot_bayes_cone ( returns_train , returns_test , ppc , plot_train_len = 50 , ax = None ) :
    """Generate cumulative returns plot with Bayesian cone .
 Parameters
 returns _ train : pd . Series
 Timeseries of simple returns
 returns _ test : pd . Series
 Out - of - sample returns . Datetimes in returns _ test will be added to
 returns _ train as missing values and predictions will be generated
 for them .
 ppc : np . array
 Posterior predictive samples of shape samples x
 len ( returns _ test ) .
 plot _ train _ len : int ( optional )
 How many data points to plot of returns _ train . Useful to zoom in on
 the prediction if there is a long backtest period .
 ax : matplotlib . Axis ( optional )
 Axes upon which to plot .
 Returns
 score : float
 Consistency score ( see compute _ consistency _ score )
 trace : pymc3 . sampling . BaseTrace
 A PyMC3 trace object that contains samples for each parameter
 of the posterior ."""

implementation:
score = compute_consistency_score ( returns_test , ppc )
ax = _plot_bayes_cone ( returns_train , returns_test , ppc , plot_train_len = plot_train_len , ax = ax )
ax . text ( 0.40 , 0.90 , 'Consistency score: %.1f' % score , verticalalignment = 'bottom' , horizontalalignment = 'right' , transform = ax . transAxes , )
ax . set_ylabel ( 'Cumulative returns' )
return score


signature:
def atrous_conv1d ( prev_layer , n_filter = 32 , filter_size = 2 , stride = 1 , dilation = 1 , act = None , padding = 'SAME' , data_format = 'NWC' , W_init = tf . truncated_normal_initializer ( stddev = 0.02 ) , b_init = tf . constant_initializer ( value = 0.0 ) , W_init_args = None , b_init_args = None , name = 'atrous_1d' , ) :
    """Simplified version of : class : ` AtrousConv1dLayer ` .
 Parameters
 prev _ layer : : class : ` Layer `
 Previous layer .
 n _ filter : int
 The number of filters .
 filter _ size : int
 The filter size .
 stride : tuple of int
 The strides : ( height , width ) .
 dilation : int
 The filter dilation size .
 act : activation function
 The activation function of this layer .
 padding : str
 The padding algorithm type : " SAME " or " VALID " .
 data _ format : str
 Default is ' NWC ' as it is a 1D CNN .
 W _ init : initializer
 The initializer for the weight matrix .
 b _ init : initializer or None
 The initializer for the bias vector . If None , skip biases .
 W _ init _ args : dictionary
 The arguments for the weight matrix initializer .
 b _ init _ args : dictionary
 The arguments for the bias vector initializer .
 name : str
 A unique layer name .
 Returns
 : class : ` Layer `
 A : class : ` AtrousConv1dLayer ` object"""

implementation: return Conv1dLayer ( prev_layer = prev_layer , act = act , shape = ( filter_size , int ( prev_layer . outputs . get_shape ( ) [ - 1 ] ) , n_filter ) , stride = stride , padding = padding , dilation_rate = dilation , data_format = data_format , W_init = W_init , b_init = b_init , W_init_args = W_init_args , b_init_args = b_init_args , name = name , )


signature:
def _GetNextLogCountPerToken ( token ) :
    """Wrapper for _ log _ counter _ per _ token .
 Args :
 token : The token for which to look up the count .
 Returns :
 The number of times this function has been called with
 * token * as an argument ( starting at 0)"""

implementation:
global _log_counter_per_token
# pylint : disable = global - variable - not - assigned
_log_counter_per_token [ token ] = 1 + _log_counter_per_token . get ( token , - 1 )
return _log_counter_per_token [ token ]


signature:
def log_every_n ( level , msg , n , * args ) :
    """Log ' msg % args ' at level ' level ' once per ' n ' times .
 Logs the 1st call , ( N + 1 ) st call , ( 2N + 1 ) st call , etc .
 Not threadsafe .
 Args :
 level : The level at which to log .
 msg : The message to be logged .
 n : The number of times this should be called before it is logged .
 * args : The args to be substituted into the msg ."""

implementation:
count = _GetNextLogCountPerToken ( _GetFileAndLine ( ) )
log_if ( level , msg , not ( count % n ) , * args )


signature:
def log_if ( level , msg , condition , * args ) :
    """Log ' msg % args ' at level ' level ' only if condition is fulfilled ."""

implementation:
if condition :
    vlog ( level , msg , * args )


signature:
def _GetFileAndLine ( ) :
    """Returns ( filename , linenumber ) for the stack frame ."""

implementation:
# Use sys . _ getframe ( ) . This avoids creating a traceback object .
# pylint : disable = protected - access
f = _sys . _getframe ( )
# pylint : enable = protected - access
our_file = f . f_code . co_filename
f = f . f_back
while f :
    code = f . f_code
    if code . co_filename != our_file :
        return ( code . co_filename , f . f_lineno )
    f = f . f_back
return ( '<unknown>' , 0 )


signature:
def google2_log_prefix ( level , timestamp = None , file_and_line = None ) :
    """Assemble a logline prefix using the google2 format ."""

implementation:
# pylint : disable = global - variable - not - assigned
global _level_names
# pylint : enable = global - variable - not - assigned
# Record current time
now = timestamp or _time . time ( )
now_tuple = _time . localtime ( now )
now_microsecond = int ( 1e6 * ( now % 1.0 ) )
( filename , line ) = file_and_line or _GetFileAndLine ( )
basename = _os . path . basename ( filename )
# Severity string
severity = 'I'
if level in _level_names :
    severity = _level_names [ level ] [ 0 ]
s = '%c%02d%02d %02d: %02d: %02d.%06d %5d %s: %d] ' % ( severity , now_tuple [ 1 ] , # month
now_tuple [ 2 ] , # day
now_tuple [ 3 ] , # hour
now_tuple [ 4 ] , # min
now_tuple [ 5 ] , # sec
now_microsecond , _get_thread_id ( ) , basename , line )
return s


signature:
def load_mpii_pose_dataset ( path = 'data' , is_16_pos_only = False ) :
    """Load MPII Human Pose Dataset .
 Parameters
 path : str
 The path that the data is downloaded to .
 is _ 16 _ pos _ only : boolean
 If True , only return the peoples contain 16 pose keypoints . ( Usually be used for single person pose estimation )
 Returns
 img _ train _ list : list of str
 The image directories of training data .
 ann _ train _ list : list of dict
 The annotations of training data .
 img _ test _ list : list of str
 The image directories of testing data .
 ann _ test _ list : list of dict
 The annotations of testing data .
 Examples
 > > > import pprint
 > > > import tensorlayer as tl
 > > > img _ train _ list , ann _ train _ list , img _ test _ list , ann _ test _ list = tl . files . load _ mpii _ pose _ dataset ( )
 > > > image = tl . vis . read _ image ( img _ train _ list [ 0 ] )
 > > > tl . vis . draw _ mpii _ pose _ to _ image ( image , ann _ train _ list [ 0 ] , ' image . png ' )
 > > > pprint . pprint ( ann _ train _ list [ 0 ] )
 References
 - ` MPII Human Pose Dataset . CVPR 14 < http : / / human - pose . mpi - inf . mpg . de > ` _ _
 - ` MPII Human Pose Models . CVPR 16 < http : / / pose . mpi - inf . mpg . de > ` _ _
 - ` MPII Human Shape , Poselet Conditioned Pictorial Structures and etc < http : / / pose . mpi - inf . mpg . de / # related > ` _ _
 - ` MPII Keyponts and ID < http : / / human - pose . mpi - inf . mpg . de / # download > ` _ _"""

implementation:
path = os . path . join ( path , 'mpii_human_pose' )
logging . info ( "Load or Download MPII Human Pose > {}" . format ( path ) )
# annotation
url = "http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/"
tar_filename = "mpii_human_pose_v1_u12_2.zip"
extracted_filename = "mpii_human_pose_v1_u12_2"
if folder_exists ( os . path . join ( path , extracted_filename ) ) is False :
    logging . info ( "[MPII] (annotation) {} is nonexistent in {}" . format ( extracted_filename , path ) )
    maybe_download_and_extract ( tar_filename , path , url , extract = True )
    del_file ( os . path . join ( path , tar_filename ) )
# images
url = "http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/"
tar_filename = "mpii_human_pose_v1.tar.gz"
extracted_filename2 = "images"
if folder_exists ( os . path . join ( path , extracted_filename2 ) ) is False :
    logging . info ( "[MPII] (images) {} is nonexistent in {}" . format ( extracted_filename , path ) )
    maybe_download_and_extract ( tar_filename , path , url , extract = True )
    del_file ( os . path . join ( path , tar_filename ) )
# parse annotation , format see http : / / human - pose . mpi - inf . mpg . de / # download
import scipy . io as sio
logging . info ( "reading annotations from mat file ..." )
# mat = sio . loadmat ( os . path . join ( path , extracted _ filename , " mpii _ human _ pose _ v1 _ u12_1 . mat " ) )
# def fix _ wrong _ joints ( joint ) : # https : / / github . com / mitmul / deeppose / blob / master / datasets / mpii _ dataset . py
# if ' 12 ' in joint and ' 13 ' in joint and ' 2 ' in joint and ' 3 ' in joint :
# if ( ( joint [ ' 12 ' ] [ 0 ] < joint [ ' 13 ' ] [ 0 ] ) and
# ( joint [ ' 3 ' ] [ 0 ] < joint [ ' 2 ' ] [ 0 ] ) ) :
# joint [ ' 2 ' ] , joint [ ' 3 ' ] = joint [ ' 3 ' ] , joint [ ' 2 ' ]
# if ( ( joint [ ' 12 ' ] [ 0 ] > joint [ ' 13 ' ] [ 0 ] ) and
# ( joint [ ' 3 ' ] [ 0 ] > joint [ ' 2 ' ] [ 0 ] ) ) :
# joint [ ' 2 ' ] , joint [ ' 3 ' ] = joint [ ' 3 ' ] , joint [ ' 2 ' ]
# return joint
ann_train_list = [ ]
ann_test_list = [ ]
img_train_list = [ ]
img_test_list = [ ]
def save_joints ( ) : # joint _ data _ fn = os . path . join ( path , ' data . json ' )
# fp = open ( joint _ data _ fn , ' w ' )
    mat = sio . loadmat ( os . path . join ( path , extracted_filename , "mpii_human_pose_v1_u12_1.mat" ) )
    for _ , ( anno , train_flag ) in enumerate ( # all images
    zip ( mat [ 'RELEASE' ] [ 'annolist' ] [ 0 , 0 ] [ 0 ] , mat [ 'RELEASE' ] [ 'img_train' ] [ 0 , 0 ] [ 0 ] ) ) :
        img_fn = anno [ 'image' ] [ 'name' ] [ 0 , 0 ] [ 0 ]
        train_flag = int ( train_flag )
        # print ( i , img _ fn , train _ flag ) # DEBUG print all images
        if train_flag :
            img_train_list . append ( img_fn )
            ann_train_list . append ( [ ] )
        else :
            img_test_list . append ( img_fn )
            ann_test_list . append ( [ ] )
        head_rect = [ ]
        if 'x1' in str ( anno [ 'annorect' ] . dtype ) :
            head_rect = zip ( [ x1 [ 0 , 0 ] for x1 in anno [ 'annorect' ] [ 'x1' ] [ 0 ] ] , [ y1 [ 0 , 0 ] for y1 in anno [ 'annorect' ] [ 'y1' ] [ 0 ] ] , [ x2 [ 0 , 0 ] for x2 in anno [ 'annorect' ] [ 'x2' ] [ 0 ] ] , [ y2 [ 0 , 0 ] for y2 in anno [ 'annorect' ] [ 'y2' ] [ 0 ] ] )
        else :
            head_rect = [ ]
        # TODO
        if 'annopoints' in str ( anno [ 'annorect' ] . dtype ) :
            annopoints = anno [ 'annorect' ] [ 'annopoints' ] [ 0 ]
            head_x1s = anno [ 'annorect' ] [ 'x1' ] [ 0 ]
            head_y1s = anno [ 'annorect' ] [ 'y1' ] [ 0 ]
            head_x2s = anno [ 'annorect' ] [ 'x2' ] [ 0 ]
            head_y2s = anno [ 'annorect' ] [ 'y2' ] [ 0 ]
            for annopoint , head_x1 , head_y1 , head_x2 , head_y2 in zip ( annopoints , head_x1s , head_y1s , head_x2s , head_y2s ) : # if annopoint ! = [ ] :
            # if len ( annopoint ) ! = 0:
                if annopoint . size :
                    head_rect = [ float ( head_x1 [ 0 , 0 ] ) , float ( head_y1 [ 0 , 0 ] ) , float ( head_x2 [ 0 , 0 ] ) , float ( head_y2 [ 0 , 0 ] ) ]
                    # joint coordinates
                    annopoint = annopoint [ 'point' ] [ 0 , 0 ]
                    j_id = [ str ( j_i [ 0 , 0 ] ) for j_i in annopoint [ 'id' ] [ 0 ] ]
                    x = [ x [ 0 , 0 ] for x in annopoint [ 'x' ] [ 0 ] ]
                    y = [ y [ 0 , 0 ] for y in annopoint [ 'y' ] [ 0 ] ]
                    joint_pos = { }
                    for _j_id , ( _x , _y ) in zip ( j_id , zip ( x , y ) ) :
                        joint_pos [ int ( _j_id ) ] = [ float ( _x ) , float ( _y ) ]
                    # joint _ pos = fix _ wrong _ joints ( joint _ pos )
                    # visibility list
                    if 'is_visible' in str ( annopoint . dtype ) :
                        vis = [ v [ 0 ] if v . size > 0 else [ 0 ] for v in annopoint [ 'is_visible' ] [ 0 ] ]
                        vis = dict ( [ ( k , int ( v [ 0 ] ) ) if len ( v ) > 0 else v for k , v in zip ( j_id , vis ) ] )
                    else :
                        vis = None
                    # if len ( joint _ pos ) = = 16:
                    if ( ( is_16_pos_only == True ) and ( len ( joint_pos ) == 16 ) ) or ( is_16_pos_only == False ) : # only use image with 16 key points / or use all
                        data = { 'filename' : img_fn , 'train' : train_flag , 'head_rect' : head_rect , 'is_visible' : vis , 'joint_pos' : joint_pos }
                        # print ( json . dumps ( data ) , file = fp ) # py3
                        if train_flag :
                            ann_train_list [ - 1 ] . append ( data )
                        else :
                            ann_test_list [ - 1 ] . append ( data )
# def write _ line ( datum , fp ) :
# joints = sorted ( [ [ int ( k ) , v ] for k , v in datum [ ' joint _ pos ' ] . items ( ) ] )
# joints = np . array ( [ j for i , j in joints ] ) . flatten ( )
# out = [ datum [ ' filename ' ] ]
# out . extend ( joints )
# out = [ str ( o ) for o in out ]
# out = ' , ' . join ( out )
# print ( out , file = fp )
# def split _ train _ test ( ) :
# # fp _ test = open ( ' data / mpii / test _ joints . csv ' , ' w ' )
# fp _ test = open ( os . path . join ( path , ' test _ joints . csv ' ) , ' w ' )
# # fp _ train = open ( ' data / mpii / train _ joints . csv ' , ' w ' )
# fp _ train = open ( os . path . join ( path , ' train _ joints . csv ' ) , ' w ' )
# # all _ data = open ( ' data / mpii / data . json ' ) . readlines ( )
# all _ data = open ( os . path . join ( path , ' data . json ' ) ) . readlines ( )
# N = len ( all _ data )
# N _ test = int ( N * 0.1)
# N _ train = N - N _ test
# print ( ' N : { } ' . format ( N ) )
# print ( ' N _ train : { } ' . format ( N _ train ) )
# print ( ' N _ test : { } ' . format ( N _ test ) )
# np . random . seed ( 1701)
# perm = np . random . permutation ( N )
# test _ indices = perm [ : N _ test ]
# train _ indices = perm [ N _ test : ]
# print ( ' train _ indices : { } ' . format ( len ( train _ indices ) ) )
# print ( ' test _ indices : { } ' . format ( len ( test _ indices ) ) )
# for i in train _ indices :
# datum = json . loads ( all _ data [ i ] . strip ( ) )
# write _ line ( datum , fp _ train )
# for i in test _ indices :
# datum = json . loads ( all _ data [ i ] . strip ( ) )
# write _ line ( datum , fp _ test )
save_joints ( )
# split _ train _ test ( ) #
# # read images dir
logging . info ( "reading images list ..." )
img_dir = os . path . join ( path , extracted_filename2 )
_img_list = load_file_list ( path = os . path . join ( path , extracted_filename2 ) , regx = '\\.jpg' , printable = False )
# ann _ list = json . load ( open ( os . path . join ( path , ' data . json ' ) ) )
for i , im in enumerate ( img_train_list ) :
    if im not in _img_list :
        print ( 'missing training image {} in {} (remove from img(ann)_train_list)' . format ( im , img_dir ) )
        # img _ train _ list . remove ( im )
        del img_train_list [ i ]
        del ann_train_list [ i ]
for i , im in enumerate ( img_test_list ) :
    if im not in _img_list :
        print ( 'missing testing image {} in {} (remove from img(ann)_test_list)' . format ( im , img_dir ) )
        # img _ test _ list . remove ( im )
        del img_train_list [ i ]
        del ann_train_list [ i ]
# # check annotation and images
n_train_images = len ( img_train_list )
n_test_images = len ( img_test_list )
n_images = n_train_images + n_test_images
logging . info ( "n_images: {} n_train_images: {} n_test_images: {}" . format ( n_images , n_train_images , n_test_images ) )
n_train_ann = len ( ann_train_list )
n_test_ann = len ( ann_test_list )
n_ann = n_train_ann + n_test_ann
logging . info ( "n_ann: {} n_train_ann: {} n_test_ann: {}" . format ( n_ann , n_train_ann , n_test_ann ) )
n_train_people = len ( sum ( ann_train_list , [ ] ) )
n_test_people = len ( sum ( ann_test_list , [ ] ) )
n_people = n_train_people + n_test_people
logging . info ( "n_people: {} n_train_people: {} n_test_people: {}" . format ( n_people , n_train_people , n_test_people ) )
# add path to all image file name
for i , value in enumerate ( img_train_list ) :
    img_train_list [ i ] = os . path . join ( img_dir , value )
for i , value in enumerate ( img_test_list ) :
    img_test_list [ i ] = os . path . join ( img_dir , value )
return img_train_list , ann_train_list , img_test_list , ann_test_list


signature:
def transformer ( U , theta , out_size , name = 'SpatialTransformer2dAffine' ) :
    """Spatial Transformer Layer for ` 2D Affine Transformation < https : / / en . wikipedia . org / wiki / Affine _ transformation > ` _ _
 , see : class : ` SpatialTransformer2dAffineLayer ` class .
 Parameters
 U : list of float
 The output of a convolutional net should have the
 shape [ num _ batch , height , width , num _ channels ] .
 theta : float
 The output of the localisation network should be [ num _ batch , 6 ] , value range should be [ 0 , 1 ] ( via tanh ) .
 out _ size : tuple of int
 The size of the output of the network ( height , width )
 name : str
 Optional function name
 Returns
 Tensor
 The transformed tensor .
 References
 - ` Spatial Transformer Networks < https : / / arxiv . org / abs / 1506.02025 > ` _ _
 - ` TensorFlow / Models < https : / / github . com / tensorflow / models / tree / master / transformer > ` _ _
 Notes
 To initialize the network to the identity transform init .
 > > > import tensorflow as tf
 > > > # ` ` theta ` ` to
 > > > identity = np . array ( [ [ 1 . , 0 . , 0 . ] , [ 0 . , 1 . , 0 . ] ] )
 > > > identity = identity . flatten ( )
 > > > theta = tf . Variable ( initial _ value = identity )"""

implementation:
def _repeat ( x , n_repeats ) :
    with tf . variable_scope ( '_repeat' ) :
        rep = tf . transpose ( tf . expand_dims ( tf . ones ( shape = tf . stack ( [ n_repeats , ] ) ) , 1 ) , [ 1 , 0 ] )
        rep = tf . cast ( rep , 'int32' )
        x = tf . matmul ( tf . reshape ( x , ( - 1 , 1 ) ) , rep )
        return tf . reshape ( x , [ - 1 ] )
def _interpolate ( im , x , y , out_size ) :
    with tf . variable_scope ( '_interpolate' ) : # constants
        num_batch = tf . shape ( im ) [ 0 ]
        height = tf . shape ( im ) [ 1 ]
        width = tf . shape ( im ) [ 2 ]
        channels = tf . shape ( im ) [ 3 ]
        x = tf . cast ( x , 'float32' )
        y = tf . cast ( y , 'float32' )
        height_f = tf . cast ( height , 'float32' )
        width_f = tf . cast ( width , 'float32' )
        out_height = out_size [ 0 ]
        out_width = out_size [ 1 ]
        zero = tf . zeros ( [ ] , dtype = 'int32' )
        max_y = tf . cast ( tf . shape ( im ) [ 1 ] - 1 , 'int32' )
        max_x = tf . cast ( tf . shape ( im ) [ 2 ] - 1 , 'int32' )
        # scale indices from [ - 1 , 1 ] to [ 0 , width / height ]
        x = ( x + 1.0 ) * ( width_f ) / 2.0
        y = ( y + 1.0 ) * ( height_f ) / 2.0
        # do sampling
        x0 = tf . cast ( tf . floor ( x ) , 'int32' )
        x1 = x0 + 1
        y0 = tf . cast ( tf . floor ( y ) , 'int32' )
        y1 = y0 + 1
        x0 = tf . clip_by_value ( x0 , zero , max_x )
        x1 = tf . clip_by_value ( x1 , zero , max_x )
        y0 = tf . clip_by_value ( y0 , zero , max_y )
        y1 = tf . clip_by_value ( y1 , zero , max_y )
        dim2 = width
        dim1 = width * height
        base = _repeat ( tf . range ( num_batch ) * dim1 , out_height * out_width )
        base_y0 = base + y0 * dim2
        base_y1 = base + y1 * dim2
        idx_a = base_y0 + x0
        idx_b = base_y1 + x0
        idx_c = base_y0 + x1
        idx_d = base_y1 + x1
        # use indices to lookup pixels in the flat image and restore
        # channels dim
        im_flat = tf . reshape ( im , tf . stack ( [ - 1 , channels ] ) )
        im_flat = tf . cast ( im_flat , 'float32' )
        Ia = tf . gather ( im_flat , idx_a )
        Ib = tf . gather ( im_flat , idx_b )
        Ic = tf . gather ( im_flat , idx_c )
        Id = tf . gather ( im_flat , idx_d )
        # and finally calculate interpolated values
        x0_f = tf . cast ( x0 , 'float32' )
        x1_f = tf . cast ( x1 , 'float32' )
        y0_f = tf . cast ( y0 , 'float32' )
        y1_f = tf . cast ( y1 , 'float32' )
        wa = tf . expand_dims ( ( ( x1_f - x ) * ( y1_f - y ) ) , 1 )
        wb = tf . expand_dims ( ( ( x1_f - x ) * ( y - y0_f ) ) , 1 )
        wc = tf . expand_dims ( ( ( x - x0_f ) * ( y1_f - y ) ) , 1 )
        wd = tf . expand_dims ( ( ( x - x0_f ) * ( y - y0_f ) ) , 1 )
        output = tf . add_n ( [ wa * Ia , wb * Ib , wc * Ic , wd * Id ] )
        return output
def _meshgrid ( height , width ) :
    with tf . variable_scope ( '_meshgrid' ) : # This should be equivalent to :
    # x _ t , y _ t = np . meshgrid ( np . linspace ( - 1 , 1 , width ) ,
    # np . linspace ( - 1 , 1 , height ) )
    # ones = np . ones ( np . prod ( x _ t . shape ) )
    # grid = np . vstack ( [ x _ t . flatten ( ) , y _ t . flatten ( ) , ones ] )
        x_t = tf . matmul ( tf . ones ( shape = tf . stack ( [ height , 1 ] ) ) , tf . transpose ( tf . expand_dims ( tf . linspace ( - 1.0 , 1.0 , width ) , 1 ) , [ 1 , 0 ] ) )
        y_t = tf . matmul ( tf . expand_dims ( tf . linspace ( - 1.0 , 1.0 , height ) , 1 ) , tf . ones ( shape = tf . stack ( [ 1 , width ] ) ) )
        x_t_flat = tf . reshape ( x_t , ( 1 , - 1 ) )
        y_t_flat = tf . reshape ( y_t , ( 1 , - 1 ) )
        ones = tf . ones_like ( x_t_flat )
        grid = tf . concat ( axis = 0 , values = [ x_t_flat , y_t_flat , ones ] )
        return grid
def _transform ( theta , input_dim , out_size ) :
    with tf . variable_scope ( '_transform' ) :
        num_batch = tf . shape ( input_dim ) [ 0 ]
        num_channels = tf . shape ( input_dim ) [ 3 ]
        theta = tf . reshape ( theta , ( - 1 , 2 , 3 ) )
        theta = tf . cast ( theta , 'float32' )
        # grid of ( x _ t , y _ t , 1 ) , eq ( 1 ) in ref [ 1]
        out_height = out_size [ 0 ]
        out_width = out_size [ 1 ]
        grid = _meshgrid ( out_height , out_width )
        grid = tf . expand_dims ( grid , 0 )
        grid = tf . reshape ( grid , [ - 1 ] )
        grid = tf . tile ( grid , tf . stack ( [ num_batch ] ) )
        grid = tf . reshape ( grid , tf . stack ( [ num_batch , 3 , - 1 ] ) )
        # Transform A x ( x _ t , y _ t , 1 ) ^ T - > ( x _ s , y _ s )
        T_g = tf . matmul ( theta , grid )
        x_s = tf . slice ( T_g , [ 0 , 0 , 0 ] , [ - 1 , 1 , - 1 ] )
        y_s = tf . slice ( T_g , [ 0 , 1 , 0 ] , [ - 1 , 1 , - 1 ] )
        x_s_flat = tf . reshape ( x_s , [ - 1 ] )
        y_s_flat = tf . reshape ( y_s , [ - 1 ] )
        input_transformed = _interpolate ( input_dim , x_s_flat , y_s_flat , out_size )
        output = tf . reshape ( input_transformed , tf . stack ( [ num_batch , out_height , out_width , num_channels ] ) )
        return output
with tf . variable_scope ( name ) :
    output = _transform ( theta , U , out_size )
    return output


signature:
def batch_transformer ( U , thetas , out_size , name = 'BatchSpatialTransformer2dAffine' ) :
    """Batch Spatial Transformer function for ` 2D Affine Transformation < https : / / en . wikipedia . org / wiki / Affine _ transformation > ` _ _ .
 Parameters
 U : list of float
 tensor of inputs [ batch , height , width , num _ channels ]
 thetas : list of float
 a set of transformations for each input [ batch , num _ transforms , 6]
 out _ size : list of int
 the size of the output [ out _ height , out _ width ]
 name : str
 optional function name
 Returns
 float
 Tensor of size [ batch * num _ transforms , out _ height , out _ width , num _ channels ]"""

implementation:
with tf . variable_scope ( name ) :
    num_batch , num_transforms = map ( int , thetas . get_shape ( ) . as_list ( ) [ : 2 ] )
    indices = [ [ i ] * num_transforms for i in xrange ( num_batch ) ]
    input_repeated = tf . gather ( U , tf . reshape ( indices , [ - 1 ] ) )
    return transformer ( input_repeated , thetas , out_size )


signature:
def create_task_spec_def ( ) :
    """Returns the a : class : ` TaskSpecDef ` based on the environment variables for distributed training .
 References
 - ` ML - engine trainer considerations < https : / / cloud . google . com / ml - engine / docs / trainer - considerations # use _ tf _ config > ` _ _
 - ` TensorPort Distributed Computing < https : / / www . tensorport . com / documentation / code - details / > ` _ _"""

implementation:
if 'TF_CONFIG' in os . environ : # TF _ CONFIG is used in ML - engine
    env = json . loads ( os . environ . get ( 'TF_CONFIG' , '{}' ) )
    task_data = env . get ( 'task' , None ) or { 'type' : 'master' , 'index' : 0 }
    cluster_data = env . get ( 'cluster' , None ) or { 'ps' : None , 'worker' : None , 'master' : None }
    return TaskSpecDef ( task_type = task_data [ 'type' ] , index = task_data [ 'index' ] , trial = task_data [ 'trial' ] if 'trial' in task_data else None , ps_hosts = cluster_data [ 'ps' ] , worker_hosts = cluster_data [ 'worker' ] , master = cluster_data [ 'master' ] if 'master' in cluster_data else None )
elif 'JOB_NAME' in os . environ : # JOB _ NAME , TASK _ INDEX , PS _ HOSTS , WORKER _ HOSTS and MASTER _ HOST are used in TensorPort
    return TaskSpecDef ( task_type = os . environ [ 'JOB_NAME' ] , index = os . environ [ 'TASK_INDEX' ] , ps_hosts = os . environ . get ( 'PS_HOSTS' , None ) , worker_hosts = os . environ . get ( 'WORKER_HOSTS' , None ) , master = os . environ . get ( 'MASTER_HOST' , None ) )
else :
    raise Exception ( 'You need to setup TF_CONFIG or JOB_NAME to define the task.' )


signature:
def create_distributed_session ( task_spec = None , checkpoint_dir = None , scaffold = None , hooks = None , chief_only_hooks = None , save_checkpoint_secs = 600 , save_summaries_steps = object ( ) , save_summaries_secs = object ( ) , config = None , stop_grace_period_secs = 120 , log_step_count_steps = 100 ) :
    """Creates a distributed session .
 It calls ` MonitoredTrainingSession ` to create a : class : ` MonitoredSession ` for distributed training .
 Parameters
 task _ spec : : class : ` TaskSpecDef ` .
 The task spec definition from create _ task _ spec _ def ( )
 checkpoint _ dir : str .
 Optional path to a directory where to restore variables .
 scaffold : ` ` Scaffold ` `
 A ` Scaffold ` used for gathering or building supportive ops .
 If not specified , a default one is created . It ' s used to finalize the graph .
 hooks : list of ` ` SessionRunHook ` ` objects .
 Optional
 chief _ only _ hooks : list of ` ` SessionRunHook ` ` objects .
 Activate these hooks if ` is _ chief = = True ` , ignore otherwise .
 save _ checkpoint _ secs : int
 The frequency , in seconds , that a checkpoint is saved
 using a default checkpoint saver . If ` save _ checkpoint _ secs ` is set to
 ` None ` , then the default checkpoint saver isn ' t used .
 save _ summaries _ steps : int
 The frequency , in number of global steps , that the
 summaries are written to disk using a default summary saver . If both
 ` save _ summaries _ steps ` and ` save _ summaries _ secs ` are set to ` None ` , then
 the default summary saver isn ' t used . Default 100.
 save _ summaries _ secs : int
 The frequency , in secs , that the summaries are written
 to disk using a default summary saver . If both ` save _ summaries _ steps ` and
 ` save _ summaries _ secs ` are set to ` None ` , then the default summary saver
 isn ' t used . Default not enabled .
 config : ` ` tf . ConfigProto ` `
 an instance of ` tf . ConfigProto ` proto used to configure the session .
 It ' s the ` config ` argument of constructor of ` tf . Session ` .
 stop _ grace _ period _ secs : int
 Number of seconds given to threads to stop after
 ` close ( ) ` has been called .
 log _ step _ count _ steps : int
 The frequency , in number of global steps , that the
 global step / sec is logged .
 Examples
 A simple example for distributed training where all the workers use the same dataset :
 > > > task _ spec = TaskSpec ( )
 > > > with tf . device ( task _ spec . device _ fn ( ) ) :
 > > > tensors = create _ graph ( )
 > > > with tl . DistributedSession ( task _ spec = task _ spec ,
 . . . checkpoint _ dir = ' / tmp / ckpt ' ) as session :
 > > > while not session . should _ stop ( ) :
 > > > session . run ( tensors )
 An example where the dataset is shared among the workers
 ( see https : / / www . tensorflow . org / programmers _ guide / datasets ) :
 > > > task _ spec = TaskSpec ( )
 > > > # dataset is a : class : ` tf . data . Dataset ` with the raw data
 > > > dataset = create _ dataset ( )
 > > > if task _ spec is not None :
 > > > dataset = dataset . shard ( task _ spec . num _ workers , task _ spec . shard _ index )
 > > > # shuffle or apply a map function to the new sharded dataset , for example :
 > > > dataset = dataset . shuffle ( buffer _ size = 10000)
 > > > dataset = dataset . batch ( batch _ size )
 > > > dataset = dataset . repeat ( num _ epochs )
 > > > # create the iterator for the dataset and the input tensor
 > > > iterator = dataset . make _ one _ shot _ iterator ( )
 > > > next _ element = iterator . get _ next ( )
 > > > with tf . device ( task _ spec . device _ fn ( ) ) :
 > > > # next _ element is the input for the graph
 > > > tensors = create _ graph ( next _ element )
 > > > with tl . DistributedSession ( task _ spec = task _ spec ,
 . . . checkpoint _ dir = ' / tmp / ckpt ' ) as session :
 > > > while not session . should _ stop ( ) :
 > > > session . run ( tensors )
 References
 - ` MonitoredTrainingSession < https : / / www . tensorflow . org / api _ docs / python / tf / train / MonitoredTrainingSession > ` _ _"""

implementation:
target = task_spec . target ( ) if task_spec is not None else None
is_chief = task_spec . is_master ( ) if task_spec is not None else True
return tf . train . MonitoredTrainingSession ( master = target , is_chief = is_chief , checkpoint_dir = checkpoint_dir , scaffold = scaffold , save_checkpoint_secs = save_checkpoint_secs , save_summaries_steps = save_summaries_steps , save_summaries_secs = save_summaries_secs , log_step_count_steps = log_step_count_steps , stop_grace_period_secs = stop_grace_period_secs , config = config , hooks = hooks , chief_only_hooks = chief_only_hooks )


signature:
def validation_metrics ( self ) :
    """A helper function to compute validation related metrics"""

implementation:
if ( self . _validation_iterator is None ) or ( self . _validation_metrics is None ) :
    raise AttributeError ( 'Validation is not setup.' )
n = 0.0
metric_sums = [ 0.0 ] * len ( self . _validation_metrics )
self . _sess . run ( self . _validation_iterator . initializer )
while True :
    try :
        metrics = self . _sess . run ( self . _validation_metrics )
        for i , m in enumerate ( metrics ) :
            metric_sums [ i ] += m
        n += 1.0
    except tf . errors . OutOfRangeError :
        break
for i , m in enumerate ( metric_sums ) :
    metric_sums [ i ] = metric_sums [ i ] / n
return zip ( self . _validation_metrics , metric_sums )


signature:
def train_and_validate_to_end ( self , validate_step_size = 50 ) :
    """A helper function that shows how to train and validate a model at the same time .
 Parameters
 validate _ step _ size : int
 Validate the training network every N steps ."""

implementation:
while not self . _sess . should_stop ( ) :
    self . train_on_batch ( )
    # Run a training step synchronously .
    if self . global_step % validate_step_size == 0 : # logging . info ( " Average loss for validation dataset : % s " % self . get _ validation _ metrics ( ) )
        log_str = 'step: %d, ' % self . global_step
        for n , m in self . validation_metrics :
            log_str += '%s: %f, ' % ( n . name , m )
        logging . info ( log_str )


signature:
def _load_mnist_dataset ( shape , path , name = 'mnist' , url = 'http://yann.lecun.com/exdb/mnist/' ) :
    """A generic function to load mnist - like dataset .
 Parameters :
 shape : tuple
 The shape of digit images .
 path : str
 The path that the data is downloaded to .
 name : str
 The dataset name you want to use ( the default is ' mnist ' ) .
 url : str
 The url of dataset ( the default is ' http : / / yann . lecun . com / exdb / mnist / ' ) ."""

implementation:
path = os . path . join ( path , name )
# Define functions for loading mnist - like data ' s images and labels .
# For convenience , they also download the requested files if needed .
def load_mnist_images ( path , filename ) :
    filepath = maybe_download_and_extract ( filename , path , url )
    logging . info ( filepath )
    # Read the inputs in Yann LeCun ' s binary format .
    with gzip . open ( filepath , 'rb' ) as f :
        data = np . frombuffer ( f . read ( ) , np . uint8 , offset = 16 )
    # The inputs are vectors now , we reshape them to monochrome 2D images ,
    # following the shape convention : ( examples , channels , rows , columns )
    data = data . reshape ( shape )
    # The inputs come as bytes , we convert them to float32 in range [ 0,1 ] .
    # ( Actually to range [ 0 , 255/256 ] , for compatibility to the version
    # provided at http : / / deeplearning . net / data / mnist / mnist . pkl . gz . )
    return data / np . float32 ( 256 )
def load_mnist_labels ( path , filename ) :
    filepath = maybe_download_and_extract ( filename , path , url )
    # Read the labels in Yann LeCun ' s binary format .
    with gzip . open ( filepath , 'rb' ) as f :
        data = np . frombuffer ( f . read ( ) , np . uint8 , offset = 8 )
    # The labels are vectors of integers now , that ' s exactly what we want .
    return data
# Download and read the training and test set images and labels .
logging . info ( "Load or Download {0} > {1}" . format ( name . upper ( ) , path ) )
X_train = load_mnist_images ( path , 'train-images-idx3-ubyte.gz' )
y_train = load_mnist_labels ( path , 'train-labels-idx1-ubyte.gz' )
X_test = load_mnist_images ( path , 't10k-images-idx3-ubyte.gz' )
y_test = load_mnist_labels ( path , 't10k-labels-idx1-ubyte.gz' )
# We reserve the last 10000 training examples for validation .
X_train , X_val = X_train [ : - 10000 ] , X_train [ - 10000 : ]
y_train , y_val = y_train [ : - 10000 ] , y_train [ - 10000 : ]
# We just return all the arrays in order , as expected in main ( ) .
# ( It doesn ' t matter how we do this as long as we can read them again . )
X_train = np . asarray ( X_train , dtype = np . float32 )
y_train = np . asarray ( y_train , dtype = np . int32 )
X_val = np . asarray ( X_val , dtype = np . float32 )
y_val = np . asarray ( y_val , dtype = np . int32 )
X_test = np . asarray ( X_test , dtype = np . float32 )
y_test = np . asarray ( y_test , dtype = np . int32 )
return X_train , y_train , X_val , y_val , X_test , y_test


signature:
def load_cifar10_dataset ( shape = ( - 1 , 32 , 32 , 3 ) , path = 'data' , plotable = False ) :
    """Load CIFAR - 10 dataset .
 It consists of 60000 32x32 colour images in 10 classes , with
 6000 images per class . There are 50000 training images and 10000 test images .
 The dataset is divided into five training batches and one test batch , each with
 10000 images . The test batch contains exactly 1000 randomly - selected images from
 each class . The training batches contain the remaining images in random order ,
 but some training batches may contain more images from one class than another .
 Between them , the training batches contain exactly 5000 images from each class .
 Parameters
 shape : tupe
 The shape of digit images e . g . ( - 1 , 3 , 32 , 32 ) and ( - 1 , 32 , 32 , 3 ) .
 path : str
 The path that the data is downloaded to , defaults is ` ` data / cifar10 / ` ` .
 plotable : boolean
 Whether to plot some image examples , False as default .
 Examples
 > > > X _ train , y _ train , X _ test , y _ test = tl . files . load _ cifar10 _ dataset ( shape = ( - 1 , 32 , 32 , 3 ) )
 References
 - ` CIFAR website < https : / / www . cs . toronto . edu / ~ kriz / cifar . html > ` _ _
 - ` Data download link < https : / / www . cs . toronto . edu / ~ kriz / cifar - 10 - python . tar . gz > ` _ _
 - ` < https : / / teratail . com / questions / 28932 > ` _ _"""

implementation:
path = os . path . join ( path , 'cifar10' )
logging . info ( "Load or Download cifar10 > {}" . format ( path ) )
# Helper function to unpickle the data
def unpickle ( file ) :
    fp = open ( file , 'rb' )
    if sys . version_info . major == 2 :
        data = pickle . load ( fp )
    elif sys . version_info . major == 3 :
        data = pickle . load ( fp , encoding = 'latin-1' )
    fp . close ( )
    return data
filename = 'cifar-10-python.tar.gz'
url = 'https://www.cs.toronto.edu/~kriz/'
# Download and uncompress file
maybe_download_and_extract ( filename , path , url , extract = True )
# Unpickle file and fill in data
X_train = None
y_train = [ ]
for i in range ( 1 , 6 ) :
    data_dic = unpickle ( os . path . join ( path , 'cifar-10-batches-py/' , "data_batch_{}" . format ( i ) ) )
    if i == 1 :
        X_train = data_dic [ 'data' ]
    else :
        X_train = np . vstack ( ( X_train , data_dic [ 'data' ] ) )
    y_train += data_dic [ 'labels' ]
test_data_dic = unpickle ( os . path . join ( path , 'cifar-10-batches-py/' , "test_batch" ) )
X_test = test_data_dic [ 'data' ]
y_test = np . array ( test_data_dic [ 'labels' ] )
if shape == ( - 1 , 3 , 32 , 32 ) :
    X_test = X_test . reshape ( shape )
    X_train = X_train . reshape ( shape )
elif shape == ( - 1 , 32 , 32 , 3 ) :
    X_test = X_test . reshape ( shape , order = 'F' )
    X_train = X_train . reshape ( shape , order = 'F' )
    X_test = np . transpose ( X_test , ( 0 , 2 , 1 , 3 ) )
    X_train = np . transpose ( X_train , ( 0 , 2 , 1 , 3 ) )
else :
    X_test = X_test . reshape ( shape )
    X_train = X_train . reshape ( shape )
y_train = np . array ( y_train )
if plotable :
    logging . info ( '\nCIFAR-10' )
    fig = plt . figure ( 1 )
    logging . info ( 'Shape of a training image: X_train[0] %s' % X_train [ 0 ] . shape )
    plt . ion ( )
    # interactive mode
    count = 1
    for _ in range ( 10 ) : # each row
        for _ in range ( 10 ) : # each column
            _ = fig . add_subplot ( 10 , 10 , count )
            if shape == ( - 1 , 3 , 32 , 32 ) : # plt . imshow ( X _ train [ count - 1 ] , interpolation = ' nearest ' )
                plt . imshow ( np . transpose ( X_train [ count - 1 ] , ( 1 , 2 , 0 ) ) , interpolation = 'nearest' )
                # plt . imshow ( np . transpose ( X _ train [ count - 1 ] , ( 2 , 1 , 0 ) ) , interpolation = ' nearest ' )
            elif shape == ( - 1 , 32 , 32 , 3 ) :
                plt . imshow ( X_train [ count - 1 ] , interpolation = 'nearest' )
                # plt . imshow ( np . transpose ( X _ train [ count - 1 ] , ( 1 , 0 , 2 ) ) , interpolation = ' nearest ' )
            else :
                raise Exception ( "Do not support the given 'shape' to plot the image examples" )
            plt . gca ( ) . xaxis . set_major_locator ( plt . NullLocator ( ) )
            # 不显示刻度 ( tick )
            plt . gca ( ) . yaxis . set_major_locator ( plt . NullLocator ( ) )
            count = count + 1
    plt . draw ( )
    # interactive mode
    plt . pause ( 3 )
    # interactive mode
    logging . info ( "X_train: %s" % X_train . shape )
    logging . info ( "y_train: %s" % y_train . shape )
    logging . info ( "X_test:  %s" % X_test . shape )
    logging . info ( "y_test:  %s" % y_test . shape )
X_train = np . asarray ( X_train , dtype = np . float32 )
X_test = np . asarray ( X_test , dtype = np . float32 )
y_train = np . asarray ( y_train , dtype = np . int32 )
y_test = np . asarray ( y_test , dtype = np . int32 )
return X_train , y_train , X_test , y_test


signature:
def load_cropped_svhn ( path = 'data' , include_extra = True ) :
    """Load Cropped SVHN .
 The Cropped Street View House Numbers ( SVHN ) Dataset contains 32x32x3 RGB images .
 Digit ' 1 ' has label 1 , ' 9 ' has label 9 and ' 0 ' has label 0 ( the original dataset uses 10 to represent ' 0 ' ) , see ` ufldl website < http : / / ufldl . stanford . edu / housenumbers / > ` _ _ .
 Parameters
 path : str
 The path that the data is downloaded to .
 include _ extra : boolean
 If True ( default ) , add extra images to the training set .
 Returns
 X _ train , y _ train , X _ test , y _ test : tuple
 Return splitted training / test set respectively .
 Examples
 > > > X _ train , y _ train , X _ test , y _ test = tl . files . load _ cropped _ svhn ( include _ extra = False )
 > > > tl . vis . save _ images ( X _ train [ 0:100 ] , [ 10 , 10 ] , ' svhn . png ' )"""

implementation:
start_time = time . time ( )
path = os . path . join ( path , 'cropped_svhn' )
logging . info ( "Load or Download Cropped SVHN > {} | include extra images: {}" . format ( path , include_extra ) )
url = "http://ufldl.stanford.edu/housenumbers/"
np_file = os . path . join ( path , "train_32x32.npz" )
if file_exists ( np_file ) is False :
    filename = "train_32x32.mat"
    filepath = maybe_download_and_extract ( filename , path , url )
    mat = sio . loadmat ( filepath )
    X_train = mat [ 'X' ] / 255.0
    # to [ 0 , 1]
    X_train = np . transpose ( X_train , ( 3 , 0 , 1 , 2 ) )
    y_train = np . squeeze ( mat [ 'y' ] , axis = 1 )
    y_train [ y_train == 10 ] = 0
    # replace 10 to 0
    np . savez ( np_file , X = X_train , y = y_train )
    del_file ( filepath )
else :
    v = np . load ( np_file )
    X_train = v [ 'X' ]
    y_train = v [ 'y' ]
logging . info ( "  n_train: {}" . format ( len ( y_train ) ) )
np_file = os . path . join ( path , "test_32x32.npz" )
if file_exists ( np_file ) is False :
    filename = "test_32x32.mat"
    filepath = maybe_download_and_extract ( filename , path , url )
    mat = sio . loadmat ( filepath )
    X_test = mat [ 'X' ] / 255.0
    X_test = np . transpose ( X_test , ( 3 , 0 , 1 , 2 ) )
    y_test = np . squeeze ( mat [ 'y' ] , axis = 1 )
    y_test [ y_test == 10 ] = 0
    np . savez ( np_file , X = X_test , y = y_test )
    del_file ( filepath )
else :
    v = np . load ( np_file )
    X_test = v [ 'X' ]
    y_test = v [ 'y' ]
logging . info ( "  n_test: {}" . format ( len ( y_test ) ) )
if include_extra :
    logging . info ( "  getting extra 531131 images, please wait ..." )
    np_file = os . path . join ( path , "extra_32x32.npz" )
    if file_exists ( np_file ) is False :
        logging . info ( "  the first time to load extra images will take long time to convert the file format ..." )
        filename = "extra_32x32.mat"
        filepath = maybe_download_and_extract ( filename , path , url )
        mat = sio . loadmat ( filepath )
        X_extra = mat [ 'X' ] / 255.0
        X_extra = np . transpose ( X_extra , ( 3 , 0 , 1 , 2 ) )
        y_extra = np . squeeze ( mat [ 'y' ] , axis = 1 )
        y_extra [ y_extra == 10 ] = 0
        np . savez ( np_file , X = X_extra , y = y_extra )
        del_file ( filepath )
    else :
        v = np . load ( np_file )
        X_extra = v [ 'X' ]
        y_extra = v [ 'y' ]
    # print ( X _ train . shape , X _ extra . shape )
    logging . info ( "  adding n_extra {} to n_train {}" . format ( len ( y_extra ) , len ( y_train ) ) )
    t = time . time ( )
    X_train = np . concatenate ( ( X_train , X_extra ) , 0 )
    y_train = np . concatenate ( ( y_train , y_extra ) , 0 )
    # X _ train = np . append ( X _ train , X _ extra , axis = 0)
    # y _ train = np . append ( y _ train , y _ extra , axis = 0)
    logging . info ( "  added n_extra {} to n_train {} took {}s" . format ( len ( y_extra ) , len ( y_train ) , time . time ( ) - t ) )
else :
    logging . info ( "  no extra images are included" )
logging . info ( "  image size: %s n_train: %d n_test: %d" % ( str ( X_train . shape [ 1 : 4 ] ) , len ( y_train ) , len ( y_test ) ) )
logging . info ( "  took: {}s" . format ( int ( time . time ( ) - start_time ) ) )
return X_train , y_train , X_test , y_test


signature:
def load_ptb_dataset ( path = 'data' ) :
    """Load Penn TreeBank ( PTB ) dataset .
 It is used in many LANGUAGE MODELING papers ,
 including " Empirical Evaluation and Combination of Advanced Language
 Modeling Techniques " , " Recurrent Neural Network Regularization " .
 It consists of 929k training words , 73k validation words , and 82k test
 words . It has 10k words in its vocabulary .
 Parameters
 path : str
 The path that the data is downloaded to , defaults is ` ` data / ptb / ` ` .
 Returns
 train _ data , valid _ data , test _ data : list of int
 The training , validating and testing data in integer format .
 vocab _ size : int
 The vocabulary size .
 Examples
 > > > train _ data , valid _ data , test _ data , vocab _ size = tl . files . load _ ptb _ dataset ( )
 References
 - ` ` tensorflow . models . rnn . ptb import reader ` `
 - ` Manual download < http : / / www . fit . vutbr . cz / ~ imikolov / rnnlm / simple - examples . tgz > ` _ _
 Notes
 - If you want to get the raw data , see the source code ."""

implementation:
path = os . path . join ( path , 'ptb' )
logging . info ( "Load or Download Penn TreeBank (PTB) dataset > {}" . format ( path ) )
# Maybe dowload and uncompress tar , or load exsisting files
filename = 'simple-examples.tgz'
url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'
maybe_download_and_extract ( filename , path , url , extract = True )
data_path = os . path . join ( path , 'simple-examples' , 'data' )
train_path = os . path . join ( data_path , "ptb.train.txt" )
valid_path = os . path . join ( data_path , "ptb.valid.txt" )
test_path = os . path . join ( data_path , "ptb.test.txt" )
word_to_id = nlp . build_vocab ( nlp . read_words ( train_path ) )
train_data = nlp . words_to_word_ids ( nlp . read_words ( train_path ) , word_to_id )
valid_data = nlp . words_to_word_ids ( nlp . read_words ( valid_path ) , word_to_id )
test_data = nlp . words_to_word_ids ( nlp . read_words ( test_path ) , word_to_id )
vocab_size = len ( word_to_id )
# logging . info ( nlp . read _ words ( train _ path ) ) # . . . ' according ' , ' to ' , ' mr . ' , ' < unk > ' , ' < eos > ' ]
# logging . info ( train _ data ) # . . . 214 , 5 , 23 , 1 , 2]
# logging . info ( word _ to _ id ) # . . . ' beyond ' : 1295 , ' anti - nuclear ' : 9599 , ' trouble ' : 1520 , ' < eos > ' : 2 . . . }
# logging . info ( vocabulary ) # 10000
# exit ( )
return train_data , valid_data , test_data , vocab_size


signature:
def load_matt_mahoney_text8_dataset ( path = 'data' ) :
    """Load Matt Mahoney ' s dataset .
 Download a text file from Matt Mahoney ' s website
 if not present , and make sure it ' s the right size .
 Extract the first file enclosed in a zip file as a list of words .
 This dataset can be used for Word Embedding .
 Parameters
 path : str
 The path that the data is downloaded to , defaults is ` ` data / mm _ test8 / ` ` .
 Returns
 list of str
 The raw text data e . g . [ . . . . ' their ' , ' families ' , ' who ' , ' were ' , ' expelled ' , ' from ' , ' jerusalem ' , . . . ]
 Examples
 > > > words = tl . files . load _ matt _ mahoney _ text8 _ dataset ( )
 > > > print ( ' Data size ' , len ( words ) )"""

implementation:
path = os . path . join ( path , 'mm_test8' )
logging . info ( "Load or Download matt_mahoney_text8 Dataset> {}" . format ( path ) )
filename = 'text8.zip'
url = 'http://mattmahoney.net/dc/'
maybe_download_and_extract ( filename , path , url , expected_bytes = 31344016 )
with zipfile . ZipFile ( os . path . join ( path , filename ) ) as f :
    word_list = f . read ( f . namelist ( ) [ 0 ] ) . split ( )
    for idx , _ in enumerate ( word_list ) :
        word_list [ idx ] = word_list [ idx ] . decode ( )
return word_list