signature:
def learn ( env , network , seed = None , lr = 5e-4 , total_timesteps = 100000 , buffer_size = 50000 , exploration_fraction = 0.1 , exploration_final_eps = 0.02 , train_freq = 1 , batch_size = 32 , print_freq = 100 , checkpoint_freq = 10000 , checkpoint_path = None , learning_starts = 1000 , gamma = 1.0 , target_network_update_freq = 500 , prioritized_replay = False , prioritized_replay_alpha = 0.6 , prioritized_replay_beta0 = 0.4 , prioritized_replay_beta_iters = None , prioritized_replay_eps = 1e-6 , param_noise = False , callback = None , load_path = None , ** network_kwargs ) :
    """Train a deepq model .
 Parameters
 env : gym . Env
 environment to train on
 network : string or a function
 neural network to use as a q function approximator . If string , has to be one of the names of registered models in baselines . common . models
 ( mlp , cnn , conv _ only ) . If a function , should take an observation tensor and return a latent variable tensor , which
 will be mapped to the Q function heads ( see build _ q _ func in baselines . deepq . models for details on that )
 seed : int or None
 prng seed . The runs with the same seed " should " give the same results . If None , no seeding is used .
 lr : float
 learning rate for adam optimizer
 total _ timesteps : int
 number of env steps to optimizer for
 buffer _ size : int
 size of the replay buffer
 exploration _ fraction : float
 fraction of entire training period over which the exploration rate is annealed
 exploration _ final _ eps : float
 final value of random action probability
 train _ freq : int
 update the model every ` train _ freq ` steps .
 set to None to disable printing
 batch _ size : int
 size of a batched sampled from replay buffer for training
 print _ freq : int
 how often to print out training progress
 set to None to disable printing
 checkpoint _ freq : int
 how often to save the model . This is so that the best version is restored
 at the end of the training . If you do not wish to restore the best version at
 the end of the training set this variable to None .
 learning _ starts : int
 how many steps of the model to collect transitions for before learning starts
 gamma : float
 discount factor
 target _ network _ update _ freq : int
 update the target network every ` target _ network _ update _ freq ` steps .
 prioritized _ replay : True
 if True prioritized replay buffer will be used .
 prioritized _ replay _ alpha : float
 alpha parameter for prioritized replay buffer
 prioritized _ replay _ beta0 : float
 initial value of beta for prioritized replay buffer
 prioritized _ replay _ beta _ iters : int
 number of iterations over which beta will be annealed from initial value
 to 1.0 . If set to None equals to total _ timesteps .
 prioritized _ replay _ eps : float
 epsilon to add to the TD errors when updating priorities .
 param _ noise : bool
 whether or not to use parameter space noise ( https : / / arxiv . org / abs / 1706.01905)
 callback : ( locals , globals ) - > None
 function called at every steps with state of the algorithm .
 If callback returns true training stops .
 load _ path : str
 path to load the model from . ( default : None )
 * * network _ kwargs
 additional keyword arguments to pass to the network builder .
 Returns
 act : ActWrapper
 Wrapper over act function . Adds ability to save it and load it .
 See header of baselines / deepq / categorical . py for details on the act function ."""

implementation:
# Create all the functions necessary to train the model
sess = get_session ( )
set_global_seeds ( seed )
q_func = build_q_func ( network , ** network_kwargs )
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make _ obs _ ph
observation_space = env . observation_space
def make_obs_ph ( name ) :
    return ObservationInput ( observation_space , name = name )
act , train , update_target , debug = deepq . build_train ( make_obs_ph = make_obs_ph , q_func = q_func , num_actions = env . action_space . n , optimizer = tf . train . AdamOptimizer ( learning_rate = lr ) , gamma = gamma , grad_norm_clipping = 10 , param_noise = param_noise )
act_params = { 'make_obs_ph' : make_obs_ph , 'q_func' : q_func , 'num_actions' : env . action_space . n , }
act = ActWrapper ( act , act_params )
# Create the replay buffer
if prioritized_replay :
    replay_buffer = PrioritizedReplayBuffer ( buffer_size , alpha = prioritized_replay_alpha )
    if prioritized_replay_beta_iters is None :
        prioritized_replay_beta_iters = total_timesteps
    beta_schedule = LinearSchedule ( prioritized_replay_beta_iters , initial_p = prioritized_replay_beta0 , final_p = 1.0 )
else :
    replay_buffer = ReplayBuffer ( buffer_size )
    beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule ( schedule_timesteps = int ( exploration_fraction * total_timesteps ) , initial_p = 1.0 , final_p = exploration_final_eps )
# Initialize the parameters and copy them to the target network .
U . initialize ( )
update_target ( )
episode_rewards = [ 0.0 ]
saved_mean_reward = None
obs = env . reset ( )
reset = True
with tempfile . TemporaryDirectory ( ) as td :
    td = checkpoint_path or td
    model_file = os . path . join ( td , "model" )
    model_saved = False
    if tf . train . latest_checkpoint ( td ) is not None :
        load_variables ( model_file )
        logger . log ( 'Loaded model from {}' . format ( model_file ) )
        model_saved = True
    elif load_path is not None :
        load_variables ( load_path )
        logger . log ( 'Loaded model from {}' . format ( load_path ) )
    for t in range ( total_timesteps ) :
        if callback is not None :
            if callback ( locals ( ) , globals ( ) ) :
                break
        # Take action and update exploration to the newest value
        kwargs = { }
        if not param_noise :
            update_eps = exploration . value ( t )
            update_param_noise_threshold = 0.
        else :
            update_eps = 0.
            # Compute the threshold such that the KL divergence between perturbed and non - perturbed
            # policy is comparable to eps - greedy exploration with eps = exploration . value ( t ) .
            # See Appendix C . 1 in Parameter Space Noise for Exploration , Plappert et al . , 2017
            # for detailed explanation .
            update_param_noise_threshold = - np . log ( 1. - exploration . value ( t ) + exploration . value ( t ) / float ( env . action_space . n ) )
            kwargs [ 'reset' ] = reset
            kwargs [ 'update_param_noise_threshold' ] = update_param_noise_threshold
            kwargs [ 'update_param_noise_scale' ] = True
        action = act ( np . array ( obs ) [ None ] , update_eps = update_eps , ** kwargs ) [ 0 ]
        env_action = action
        reset = False
        new_obs , rew , done , _ = env . step ( env_action )
        # Store transition in the replay buffer .
        replay_buffer . add ( obs , action , rew , new_obs , float ( done ) )
        obs = new_obs
        episode_rewards [ - 1 ] += rew
        if done :
            obs = env . reset ( )
            episode_rewards . append ( 0.0 )
            reset = True
        if t > learning_starts and t % train_freq == 0 : # Minimize the error in Bellman ' s equation on a batch sampled from replay buffer .
            if prioritized_replay :
                experience = replay_buffer . sample ( batch_size , beta = beta_schedule . value ( t ) )
                ( obses_t , actions , rewards , obses_tp1 , dones , weights , batch_idxes ) = experience
            else :
                obses_t , actions , rewards , obses_tp1 , dones = replay_buffer . sample ( batch_size )
                weights , batch_idxes = np . ones_like ( rewards ) , None
            td_errors = train ( obses_t , actions , rewards , obses_tp1 , dones , weights )
            if prioritized_replay :
                new_priorities = np . abs ( td_errors ) + prioritized_replay_eps
                replay_buffer . update_priorities ( batch_idxes , new_priorities )
        if t > learning_starts and t % target_network_update_freq == 0 : # Update target network periodically .
            update_target ( )
        mean_100ep_reward = round ( np . mean ( episode_rewards [ - 101 : - 1 ] ) , 1 )
        num_episodes = len ( episode_rewards )
        if done and print_freq is not None and len ( episode_rewards ) % print_freq == 0 :
            logger . record_tabular ( "steps" , t )
            logger . record_tabular ( "episodes" , num_episodes )
            logger . record_tabular ( "mean 100 episode reward" , mean_100ep_reward )
            logger . record_tabular ( "% time spent exploring" , int ( 100 * exploration . value ( t ) ) )
            logger . dump_tabular ( )
        if ( checkpoint_freq is not None and t > learning_starts and num_episodes > 100 and t % checkpoint_freq == 0 ) :
            if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward :
                if print_freq is not None :
                    logger . log ( "Saving model due to mean reward increase: {} -> {}" . format ( saved_mean_reward , mean_100ep_reward ) )
                save_variables ( model_file )
                model_saved = True
                saved_mean_reward = mean_100ep_reward
    if model_saved :
        if print_freq is not None :
            logger . log ( "Restored model with mean reward: {}" . format ( saved_mean_reward ) )
        load_variables ( model_file )
return act


signature:
def save_act ( self , path = None ) :
    """Save model to a pickle located at ` path `"""

implementation:
if path is None :
    path = os . path . join ( logger . get_dir ( ) , "model.pkl" )
with tempfile . TemporaryDirectory ( ) as td :
    save_variables ( os . path . join ( td , "model" ) )
    arc_name = os . path . join ( td , "packed.zip" )
    with zipfile . ZipFile ( arc_name , 'w' ) as zipf :
        for root , dirs , files in os . walk ( td ) :
            for fname in files :
                file_path = os . path . join ( root , fname )
                if file_path != arc_name :
                    zipf . write ( file_path , os . path . relpath ( file_path , td ) )
    with open ( arc_name , "rb" ) as f :
        model_data = f . read ( )
with open ( path , "wb" ) as f :
    cloudpickle . dump ( ( model_data , self . _act_params ) , f )


signature:
def nature_cnn ( unscaled_images , ** conv_kwargs ) :
    """CNN from Nature paper ."""

implementation:
scaled_images = tf . cast ( unscaled_images , tf . float32 ) / 255.
activ = tf . nn . relu
h = activ ( conv ( scaled_images , 'c1' , nf = 32 , rf = 8 , stride = 4 , init_scale = np . sqrt ( 2 ) , ** conv_kwargs ) )
h2 = activ ( conv ( h , 'c2' , nf = 64 , rf = 4 , stride = 2 , init_scale = np . sqrt ( 2 ) , ** conv_kwargs ) )
h3 = activ ( conv ( h2 , 'c3' , nf = 64 , rf = 3 , stride = 1 , init_scale = np . sqrt ( 2 ) , ** conv_kwargs ) )
h3 = conv_to_fc ( h3 )
return activ ( fc ( h3 , 'fc1' , nh = 512 , init_scale = np . sqrt ( 2 ) ) )


signature:
def mlp ( num_layers = 2 , num_hidden = 64 , activation = tf . tanh , layer_norm = False ) :
    """Stack of fully - connected layers to be used in a policy / q - function approximator
 Parameters :
 num _ layers : int number of fully - connected layers ( default : 2)
 num _ hidden : int size of fully - connected layers ( default : 64)
 activation : activation function ( default : tf . tanh )
 Returns :
 function that builds fully connected network with a given input tensor / placeholder"""

implementation:
def network_fn ( X ) :
    h = tf . layers . flatten ( X )
    for i in range ( num_layers ) :
        h = fc ( h , 'mlp_fc{}' . format ( i ) , nh = num_hidden , init_scale = np . sqrt ( 2 ) )
        if layer_norm :
            h = tf . contrib . layers . layer_norm ( h , center = True , scale = True )
        h = activation ( h )
    return h
return network_fn


signature:
def lstm ( nlstm = 128 , layer_norm = False ) :
    """Builds LSTM ( Long - Short Term Memory ) network to be used in a policy .
 Note that the resulting function returns not only the output of the LSTM
 ( i . e . hidden state of lstm for each step in the sequence ) , but also a dictionary
 with auxiliary tensors to be set as policy attributes .
 Specifically ,
 S is a placeholder to feed current state ( LSTM state has to be managed outside policy )
 M is a placeholder for the mask ( used to mask out observations after the end of the episode , but can be used for other purposes too )
 initial _ state is a numpy array containing initial lstm state ( usually zeros )
 state is the output LSTM state ( to be fed into S at the next call )
 An example of usage of lstm - based policy can be found here : common / tests / test _ doc _ examples . py / test _ lstm _ example
 Parameters :
 nlstm : int LSTM hidden state size
 layer _ norm : bool if True , layer - normalized version of LSTM is used
 Returns :
 function that builds LSTM with a given input tensor / placeholder"""

implementation:
def network_fn ( X , nenv = 1 ) :
    nbatch = X . shape [ 0 ]
    nsteps = nbatch // nenv
    h = tf . layers . flatten ( X )
    M = tf . placeholder ( tf . float32 , [ nbatch ] )
    # mask ( done t - 1)
    S = tf . placeholder ( tf . float32 , [ nenv , 2 * nlstm ] )
    # states
    xs = batch_to_seq ( h , nenv , nsteps )
    ms = batch_to_seq ( M , nenv , nsteps )
    if layer_norm :
        h5 , snew = utils . lnlstm ( xs , ms , S , scope = 'lnlstm' , nh = nlstm )
    else :
        h5 , snew = utils . lstm ( xs , ms , S , scope = 'lstm' , nh = nlstm )
    h = seq_to_batch ( h5 )
    initial_state = np . zeros ( S . shape . as_list ( ) , dtype = float )
    return h , { 'S' : S , 'M' : M , 'state' : snew , 'initial_state' : initial_state }
return network_fn


signature:
def conv_only ( convs = [ ( 32 , 8 , 4 ) , ( 64 , 4 , 2 ) , ( 64 , 3 , 1 ) ] , ** conv_kwargs ) :
    '''convolutions - only net
 Parameters :
 conv : list of triples ( filter _ number , filter _ size , stride ) specifying parameters for each layer .
 Returns :
 function that takes tensorflow tensor as input and returns the output of the last convolutional layer'''

implementation:
def network_fn ( X ) :
    out = tf . cast ( X , tf . float32 ) / 255.
    with tf . variable_scope ( "convnet" ) :
        for num_outputs , kernel_size , stride in convs :
            out = layers . convolution2d ( out , num_outputs = num_outputs , kernel_size = kernel_size , stride = stride , activation_fn = tf . nn . relu , ** conv_kwargs )
    return out
return network_fn


signature:
def get_network_builder ( name ) :
    """If you want to register your own network outside models . py , you just need :
 Usage Example :
 from baselines . common . models import register
 @ register ( " your _ network _ name " )
 def your _ network _ define ( * * net _ kwargs ) :
 return network _ fn"""

implementation:
if callable ( name ) :
    return name
elif name in mapping :
    return mapping [ name ]
else :
    raise ValueError ( 'Unknown network type: {}' . format ( name ) )


signature:
def mlp ( hiddens = [ ] , layer_norm = False ) :
    """This model takes as input an observation and returns values of all actions .
 Parameters
 hiddens : [ int ]
 list of sizes of hidden layers
 layer _ norm : bool
 if true applies layer normalization for every layer
 as described in https : / / arxiv . org / abs / 1607.06450
 Returns
 q _ func : function
 q _ function for DQN algorithm ."""

implementation: return lambda * args , ** kwargs : _mlp ( hiddens , layer_norm = layer_norm , * args , ** kwargs )


signature:
def cnn_to_mlp ( convs , hiddens , dueling = False , layer_norm = False ) :
    """This model takes as input an observation and returns values of all actions .
 Parameters
 convs : [ ( int , int , int ) ]
 list of convolutional layers in form of
 ( num _ outputs , kernel _ size , stride )
 hiddens : [ int ]
 list of sizes of hidden layers
 dueling : bool
 if true double the output MLP to compute a baseline
 for action scores
 layer _ norm : bool
 if true applies layer normalization for every layer
 as described in https : / / arxiv . org / abs / 1607.06450
 Returns
 q _ func : function
 q _ function for DQN algorithm ."""

implementation: return lambda * args , ** kwargs : _cnn_to_mlp ( convs , hiddens , dueling , layer_norm = layer_norm , * args , ** kwargs )


signature:
def make_vec_env ( env_id , env_type , num_env , seed , wrapper_kwargs = None , start_index = 0 , reward_scale = 1.0 , flatten_dict_observations = True , gamestate = None ) :
    """Create a wrapped , monitored SubprocVecEnv for Atari and MuJoCo ."""

implementation:
wrapper_kwargs = wrapper_kwargs or { }
mpi_rank = MPI . COMM_WORLD . Get_rank ( ) if MPI else 0
seed = seed + 10000 * mpi_rank if seed is not None else None
logger_dir = logger . get_dir ( )
def make_thunk ( rank ) :
    return lambda : make_env ( env_id = env_id , env_type = env_type , mpi_rank = mpi_rank , subrank = rank , seed = seed , reward_scale = reward_scale , gamestate = gamestate , flatten_dict_observations = flatten_dict_observations , wrapper_kwargs = wrapper_kwargs , logger_dir = logger_dir )
set_global_seeds ( seed )
if num_env > 1 :
    return SubprocVecEnv ( [ make_thunk ( i + start_index ) for i in range ( num_env ) ] )
else :
    return DummyVecEnv ( [ make_thunk ( start_index ) ] )


signature:
def make_mujoco_env ( env_id , seed , reward_scale = 1.0 ) :
    """Create a wrapped , monitored gym . Env for MuJoCo ."""

implementation:
rank = MPI . COMM_WORLD . Get_rank ( )
myseed = seed + 1000 * rank if seed is not None else None
set_global_seeds ( myseed )
env = gym . make ( env_id )
logger_path = None if logger . get_dir ( ) is None else os . path . join ( logger . get_dir ( ) , str ( rank ) )
env = Monitor ( env , logger_path , allow_early_resets = True )
env . seed ( seed )
if reward_scale != 1.0 :
    from baselines . common . retro_wrappers import RewardScaler
    env = RewardScaler ( env , reward_scale )
return env


signature:
def make_robotics_env ( env_id , seed , rank = 0 ) :
    """Create a wrapped , monitored gym . Env for MuJoCo ."""

implementation:
set_global_seeds ( seed )
env = gym . make ( env_id )
env = FlattenDictWrapper ( env , [ 'observation' , 'desired_goal' ] )
env = Monitor ( env , logger . get_dir ( ) and os . path . join ( logger . get_dir ( ) , str ( rank ) ) , info_keywords = ( 'is_success' , ) )
env . seed ( seed )
return env


signature:
def common_arg_parser ( ) :
    """Create an argparse . ArgumentParser for run _ mujoco . py ."""

implementation:
parser = arg_parser ( )
parser . add_argument ( '--env' , help = 'environment ID' , type = str , default = 'Reacher-v2' )
parser . add_argument ( '--env_type' , help = 'type of environment, used when the environment type cannot be automatically determined' , type = str )
parser . add_argument ( '--seed' , help = 'RNG seed' , type = int , default = None )
parser . add_argument ( '--alg' , help = 'Algorithm' , type = str , default = 'ppo2' )
parser . add_argument ( '--num_timesteps' , type = float , default = 1e6 ) ,
parser . add_argument ( '--network' , help = 'network type (mlp, cnn, lstm, cnn_lstm, conv_only)' , default = None )
parser . add_argument ( '--gamestate' , help = 'game state to load (so far only used in retro games)' , default = None )
parser . add_argument ( '--num_env' , help = 'Number of environment copies being run in parallel. When not specified, set to number of cpus for Atari, and to 1 for Mujoco' , default = None , type = int )
parser . add_argument ( '--reward_scale' , help = 'Reward scale factor. Default: 1.0' , default = 1.0 , type = float )
parser . add_argument ( '--save_path' , help = 'Path to save trained model to' , default = None , type = str )
parser . add_argument ( '--save_video_interval' , help = 'Save video every x steps (0 = disabled)' , default = 0 , type = int )
parser . add_argument ( '--save_video_length' , help = 'Length of recorded video. Default: 200' , default = 200 , type = int )
parser . add_argument ( '--play' , default = False , action = 'store_true' )
return parser


signature:
def robotics_arg_parser ( ) :
    """Create an argparse . ArgumentParser for run _ mujoco . py ."""

implementation:
parser = arg_parser ( )
parser . add_argument ( '--env' , help = 'environment ID' , type = str , default = 'FetchReach-v0' )
parser . add_argument ( '--seed' , help = 'RNG seed' , type = int , default = None )
parser . add_argument ( '--num-timesteps' , type = int , default = int ( 1e6 ) )
return parser


signature:
def parse_unknown_args ( args ) :
    """Parse arguments not consumed by arg parser into a dicitonary"""

implementation:
retval = { }
preceded_by_key = False
for arg in args :
    if arg . startswith ( '--' ) :
        if '=' in arg :
            key = arg . split ( '=' ) [ 0 ] [ 2 : ]
            value = arg . split ( '=' ) [ 1 ]
            retval [ key ] = value
        else :
            key = arg [ 2 : ]
            preceded_by_key = True
    elif preceded_by_key :
        retval [ key ] = arg
        preceded_by_key = False
return retval


signature:
def clear_mpi_env_vars ( ) :
    """from mpi4py import MPI will call MPI _ Init by default . If the child process has MPI environment variables , MPI will think that the child process is an MPI process just like the parent and do bad things such as hang .
 This context manager is a hacky way to clear those environment variables temporarily such as when we are starting multiprocessing
 Processes ."""

implementation:
removed_environment = { }
for k , v in list ( os . environ . items ( ) ) :
    for prefix in [ 'OMPI_' , 'PMI_' ] :
        if k . startswith ( prefix ) :
            removed_environment [ k ] = v
            del os . environ [ k ]
try :
    yield
finally :
    os . environ . update ( removed_environment )


signature:
def learn ( * , network , env , total_timesteps , eval_env = None , seed = None , nsteps = 2048 , ent_coef = 0.0 , lr = 3e-4 , vf_coef = 0.5 , max_grad_norm = 0.5 , gamma = 0.99 , lam = 0.95 , log_interval = 10 , nminibatches = 4 , noptepochs = 4 , cliprange = 0.2 , save_interval = 0 , load_path = None , model_fn = None , ** network_kwargs ) :
    '''Learn policy using PPO algorithm ( https : / / arxiv . org / abs / 1707.06347)
 Parameters :
 network : policy network architecture . Either string ( mlp , lstm , lnlstm , cnn _ lstm , cnn , cnn _ small , conv _ only - see baselines . common / models . py for full list )
 specifying the standard network architecture , or a function that takes tensorflow tensor as input and returns
 tuple ( output _ tensor , extra _ feed ) where output tensor is the last network layer output , extra _ feed is None for feed - forward
 neural nets , and extra _ feed is a dictionary describing how to feed state into the network for recurrent neural nets .
 See common / models . py / lstm for more details on using recurrent nets in policies
 env : baselines . common . vec _ env . VecEnv environment . Needs to be vectorized for parallel environment simulation .
 The environments produced by gym . make can be wrapped using baselines . common . vec _ env . DummyVecEnv class .
 nsteps : int number of steps of the vectorized environment per update ( i . e . batch size is nsteps * nenv where
 nenv is number of environment copies simulated in parallel )
 total _ timesteps : int number of timesteps ( i . e . number of actions taken in the environment )
 ent _ coef : float policy entropy coefficient in the optimization objective
 lr : float or function learning rate , constant or a schedule function [ 0,1 ] - > R + where 1 is beginning of the
 training and 0 is the end of the training .
 vf _ coef : float value function loss coefficient in the optimization objective
 max _ grad _ norm : float or None gradient norm clipping coefficient
 gamma : float discounting factor
 lam : float advantage estimation discounting factor ( lambda in the paper )
 log _ interval : int number of timesteps between logging events
 nminibatches : int number of training minibatches per update . For recurrent policies ,
 should be smaller or equal than number of environments run in parallel .
 noptepochs : int number of training epochs per update
 cliprange : float or function clipping range , constant or schedule function [ 0,1 ] - > R + where 1 is beginning of the training
 and 0 is the end of the training
 save _ interval : int number of timesteps between saving events
 load _ path : str path to load the model from
 * * network _ kwargs : keyword arguments to the policy / network builder . See baselines . common / policies . py / build _ policy and arguments to a particular type of network
 For instance , ' mlp ' network architecture has arguments num _ hidden and num _ layers .'''

implementation:
set_global_seeds ( seed )
if isinstance ( lr , float ) :
    lr = constfn ( lr )
else :
    assert callable ( lr )
if isinstance ( cliprange , float ) :
    cliprange = constfn ( cliprange )
else :
    assert callable ( cliprange )
total_timesteps = int ( total_timesteps )
policy = build_policy ( env , network , ** network_kwargs )
# Get the nb of env
nenvs = env . num_envs
# Get state _ space and action _ space
ob_space = env . observation_space
ac_space = env . action_space
# Calculate the batch _ size
nbatch = nenvs * nsteps
nbatch_train = nbatch // nminibatches
# Instantiate the model object ( that creates act _ model and train _ model )
if model_fn is None :
    from baselines . ppo2 . model import Model
    model_fn = Model
model = model_fn ( policy = policy , ob_space = ob_space , ac_space = ac_space , nbatch_act = nenvs , nbatch_train = nbatch_train , nsteps = nsteps , ent_coef = ent_coef , vf_coef = vf_coef , max_grad_norm = max_grad_norm )
if load_path is not None :
    model . load ( load_path )
# Instantiate the runner object
runner = Runner ( env = env , model = model , nsteps = nsteps , gamma = gamma , lam = lam )
if eval_env is not None :
    eval_runner = Runner ( env = eval_env , model = model , nsteps = nsteps , gamma = gamma , lam = lam )
epinfobuf = deque ( maxlen = 100 )
if eval_env is not None :
    eval_epinfobuf = deque ( maxlen = 100 )
# Start total timer
tfirststart = time . perf_counter ( )
nupdates = total_timesteps // nbatch
for update in range ( 1 , nupdates + 1 ) :
    assert nbatch % nminibatches == 0
    # Start timer
    tstart = time . perf_counter ( )
    frac = 1.0 - ( update - 1.0 ) / nupdates
    # Calculate the learning rate
    lrnow = lr ( frac )
    # Calculate the cliprange
    cliprangenow = cliprange ( frac )
    # Get minibatch
    obs , returns , masks , actions , values , neglogpacs , states , epinfos = runner . run ( )
    # pylint : disable = E0632
    if eval_env is not None :
        eval_obs , eval_returns , eval_masks , eval_actions , eval_values , eval_neglogpacs , eval_states , eval_epinfos = eval_runner . run ( )
    # pylint : disable = E0632
    epinfobuf . extend ( epinfos )
    if eval_env is not None :
        eval_epinfobuf . extend ( eval_epinfos )
    # Here what we ' re going to do is for each minibatch calculate the loss and append it .
    mblossvals = [ ]
    if states is None : # nonrecurrent version
    # Index of each element of batch _ size
    # Create the indices array
        inds = np . arange ( nbatch )
        for _ in range ( noptepochs ) : # Randomize the indexes
            np . random . shuffle ( inds )
            # 0 to batch _ size with batch _ train _ size step
            for start in range ( 0 , nbatch , nbatch_train ) :
                end = start + nbatch_train
                mbinds = inds [ start : end ]
                slices = ( arr [ mbinds ] for arr in ( obs , returns , masks , actions , values , neglogpacs ) )
                mblossvals . append ( model . train ( lrnow , cliprangenow , * slices ) )
    else : # recurrent version
        assert nenvs % nminibatches == 0
        envsperbatch = nenvs // nminibatches
        envinds = np . arange ( nenvs )
        flatinds = np . arange ( nenvs * nsteps ) . reshape ( nenvs , nsteps )
        for _ in range ( noptepochs ) :
            np . random . shuffle ( envinds )
            for start in range ( 0 , nenvs , envsperbatch ) :
                end = start + envsperbatch
                mbenvinds = envinds [ start : end ]
                mbflatinds = flatinds [ mbenvinds ] . ravel ( )
                slices = ( arr [ mbflatinds ] for arr in ( obs , returns , masks , actions , values , neglogpacs ) )
                mbstates = states [ mbenvinds ]
                mblossvals . append ( model . train ( lrnow , cliprangenow , * slices , mbstates ) )
    # Feedforward - - > get losses - - > update
    lossvals = np . mean ( mblossvals , axis = 0 )
    # End timer
    tnow = time . perf_counter ( )
    # Calculate the fps ( frame per second )
    fps = int ( nbatch / ( tnow - tstart ) )
    if update % log_interval == 0 or update == 1 : # Calculates if value function is a good predicator of the returns ( ev > 1)
    # or if it ' s just worse than predicting nothing ( ev = < 0)
        ev = explained_variance ( values , returns )
        logger . logkv ( "serial_timesteps" , update * nsteps )
        logger . logkv ( "nupdates" , update )
        logger . logkv ( "total_timesteps" , update * nbatch )
        logger . logkv ( "fps" , fps )
        logger . logkv ( "explained_variance" , float ( ev ) )
        logger . logkv ( 'eprewmean' , safemean ( [ epinfo [ 'r' ] for epinfo in epinfobuf ] ) )
        logger . logkv ( 'eplenmean' , safemean ( [ epinfo [ 'l' ] for epinfo in epinfobuf ] ) )
        if eval_env is not None :
            logger . logkv ( 'eval_eprewmean' , safemean ( [ epinfo [ 'r' ] for epinfo in eval_epinfobuf ] ) )
            logger . logkv ( 'eval_eplenmean' , safemean ( [ epinfo [ 'l' ] for epinfo in eval_epinfobuf ] ) )
        logger . logkv ( 'time_elapsed' , tnow - tfirststart )
        for ( lossval , lossname ) in zip ( lossvals , model . loss_names ) :
            logger . logkv ( lossname , lossval )
        if MPI is None or MPI . COMM_WORLD . Get_rank ( ) == 0 :
            logger . dumpkvs ( )
    if save_interval and ( update % save_interval == 0 or update == 1 ) and logger . get_dir ( ) and ( MPI is None or MPI . COMM_WORLD . Get_rank ( ) == 0 ) :
        checkdir = osp . join ( logger . get_dir ( ) , 'checkpoints' )
        os . makedirs ( checkdir , exist_ok = True )
        savepath = osp . join ( checkdir , '%.5i' % update )
        print ( 'Saving to' , savepath )
        model . save ( savepath )
return model


signature:
def cg ( f_Ax , b , cg_iters = 10 , callback = None , verbose = False , residual_tol = 1e-10 ) :
    """Demmel p 312"""

implementation:
p = b . copy ( )
r = b . copy ( )
x = np . zeros_like ( b )
rdotr = r . dot ( r )
fmtstr = "%10i %10.3g %10.3g"
titlestr = "%10s %10s %10s"
if verbose :
    print ( titlestr % ( "iter" , "residual norm" , "soln norm" ) )
for i in range ( cg_iters ) :
    if callback is not None :
        callback ( x )
    if verbose :
        print ( fmtstr % ( i , rdotr , np . linalg . norm ( x ) ) )
    z = f_Ax ( p )
    v = rdotr / p . dot ( z )
    x += v * p
    r -= v * z
    newrdotr = r . dot ( r )
    mu = newrdotr / rdotr
    p = r + mu * p
    rdotr = newrdotr
    if rdotr < residual_tol :
        break
if callback is not None :
    callback ( x )
if verbose :
    print ( fmtstr % ( i + 1 , rdotr , np . linalg . norm ( x ) ) )
    # pylint : disable = W0631
return x


signature:
def observation_placeholder ( ob_space , batch_size = None , name = 'Ob' ) :
    '''Create placeholder to feed observations into of the size appropriate to the observation space
 Parameters :
 ob _ space : gym . Space observation space
 batch _ size : int size of the batch to be fed into input . Can be left None in most cases .
 name : str name of the placeholder
 Returns :
 tensorflow placeholder tensor'''

implementation:
assert isinstance ( ob_space , Discrete ) or isinstance ( ob_space , Box ) or isinstance ( ob_space , MultiDiscrete ) , 'Can only deal with Discrete and Box observation spaces for now'
dtype = ob_space . dtype
if dtype == np . int8 :
    dtype = np . uint8
return tf . placeholder ( shape = ( batch_size , ) + ob_space . shape , dtype = dtype , name = name )


signature:
def observation_input ( ob_space , batch_size = None , name = 'Ob' ) :
    '''Create placeholder to feed observations into of the size appropriate to the observation space , and add input
 encoder of the appropriate type .'''

implementation:
placeholder = observation_placeholder ( ob_space , batch_size , name )
return placeholder , encode_observation ( ob_space , placeholder )


signature:
def encode_observation ( ob_space , placeholder ) :
    '''Encode input in the way that is appropriate to the observation space
 Parameters :
 ob _ space : gym . Space observation space
 placeholder : tf . placeholder observation input placeholder'''

implementation:
if isinstance ( ob_space , Discrete ) :
    return tf . to_float ( tf . one_hot ( placeholder , ob_space . n ) )
elif isinstance ( ob_space , Box ) :
    return tf . to_float ( placeholder )
elif isinstance ( ob_space , MultiDiscrete ) :
    placeholder = tf . cast ( placeholder , tf . int32 )
    one_hots = [ tf . to_float ( tf . one_hot ( placeholder [ ... , i ] , ob_space . nvec [ i ] ) ) for i in range ( placeholder . shape [ - 1 ] ) ]
    return tf . concat ( one_hots , axis = - 1 )
else :
    raise NotImplementedError


signature:
def generate_rollouts ( self ) :
    """Performs ` rollout _ batch _ size ` rollouts in parallel for time horizon ` T ` with the current
 policy acting on it accordingly ."""

implementation:
self . reset_all_rollouts ( )
# compute observations
o = np . empty ( ( self . rollout_batch_size , self . dims [ 'o' ] ) , np . float32 )
# observations
ag = np . empty ( ( self . rollout_batch_size , self . dims [ 'g' ] ) , np . float32 )
# achieved goals
o [ : ] = self . initial_o
ag [ : ] = self . initial_ag
# generate episodes
obs , achieved_goals , acts , goals , successes = [ ] , [ ] , [ ] , [ ] , [ ]
dones = [ ]
info_values = [ np . empty ( ( self . T - 1 , self . rollout_batch_size , self . dims [ 'info_' + key ] ) , np . float32 ) for key in self . info_keys ]
Qs = [ ]
for t in range ( self . T ) :
    policy_output = self . policy . get_actions ( o , ag , self . g , compute_Q = self . compute_Q , noise_eps = self . noise_eps if not self . exploit else 0. , random_eps = self . random_eps if not self . exploit else 0. , use_target_net = self . use_target_net )
    if self . compute_Q :
        u , Q = policy_output
        Qs . append ( Q )
    else :
        u = policy_output
    if u . ndim == 1 : # The non - batched case should still have a reasonable shape .
        u = u . reshape ( 1 , - 1 )
    o_new = np . empty ( ( self . rollout_batch_size , self . dims [ 'o' ] ) )
    ag_new = np . empty ( ( self . rollout_batch_size , self . dims [ 'g' ] ) )
    success = np . zeros ( self . rollout_batch_size )
    # compute new states and observations
    obs_dict_new , _ , done , info = self . venv . step ( u )
    o_new = obs_dict_new [ 'observation' ]
    ag_new = obs_dict_new [ 'achieved_goal' ]
    success = np . array ( [ i . get ( 'is_success' , 0.0 ) for i in info ] )
    if any ( done ) : # here we assume all environments are done is ~ same number of steps , so we terminate rollouts whenever any of the envs returns done
    # trick with using vecenvs is not to add the obs from the environments that are " done " , because those are already observations
    # after a reset
        break
    for i , info_dict in enumerate ( info ) :
        for idx , key in enumerate ( self . info_keys ) :
            info_values [ idx ] [ t , i ] = info [ i ] [ key ]
    if np . isnan ( o_new ) . any ( ) :
        self . logger . warn ( 'NaN caught during rollout generation. Trying again...' )
        self . reset_all_rollouts ( )
        return self . generate_rollouts ( )
    dones . append ( done )
    obs . append ( o . copy ( ) )
    achieved_goals . append ( ag . copy ( ) )
    successes . append ( success . copy ( ) )
    acts . append ( u . copy ( ) )
    goals . append ( self . g . copy ( ) )
    o [ ... ] = o_new
    ag [ ... ] = ag_new
obs . append ( o . copy ( ) )
achieved_goals . append ( ag . copy ( ) )
episode = dict ( o = obs , u = acts , g = goals , ag = achieved_goals )
for key , value in zip ( self . info_keys , info_values ) :
    episode [ 'info_{}' . format ( key ) ] = value
# stats
successful = np . array ( successes ) [ - 1 , : ]
assert successful . shape == ( self . rollout_batch_size , )
success_rate = np . mean ( successful )
self . success_history . append ( success_rate )
if self . compute_Q :
    self . Q_history . append ( np . mean ( Qs ) )
self . n_episodes += self . rollout_batch_size
return convert_episode_to_batch_major ( episode )


signature:
def save_policy ( self , path ) :
    """Pickles the current policy for later inspection ."""

implementation:
with open ( path , 'wb' ) as f :
    pickle . dump ( self . policy , f )


signature:
def logs ( self , prefix = 'worker' ) :
    """Generates a dictionary that contains all collected statistics ."""

implementation:
logs = [ ]
logs += [ ( 'success_rate' , np . mean ( self . success_history ) ) ]
if self . compute_Q :
    logs += [ ( 'mean_Q' , np . mean ( self . Q_history ) ) ]
logs += [ ( 'episode' , self . n_episodes ) ]
if prefix != '' and not prefix . endswith ( '/' ) :
    return [ ( prefix + '/' + key , val ) for key , val in logs ]
else :
    return logs


signature:
def smooth ( y , radius , mode = 'two_sided' , valid_only = False ) :
    '''Smooth signal y , where radius is determines the size of the window
 mode = ' twosided ' :
 average over the window [ max ( index - radius , 0 ) , min ( index + radius , len ( y ) - 1 ) ]
 mode = ' causal ' :
 average over the window [ max ( index - radius , 0 ) , index ]
 valid _ only : put nan in entries where the full - sized window is not available'''

implementation:
assert mode in ( 'two_sided' , 'causal' )
if len ( y ) < 2 * radius + 1 :
    return np . ones_like ( y ) * y . mean ( )
elif mode == 'two_sided' :
    convkernel = np . ones ( 2 * radius + 1 )
    out = np . convolve ( y , convkernel , mode = 'same' ) / np . convolve ( np . ones_like ( y ) , convkernel , mode = 'same' )
    if valid_only :
        out [ : radius ] = out [ - radius : ] = np . nan
elif mode == 'causal' :
    convkernel = np . ones ( radius )
    out = np . convolve ( y , convkernel , mode = 'full' ) / np . convolve ( np . ones_like ( y ) , convkernel , mode = 'full' )
    out = out [ : - radius + 1 ]
    if valid_only :
        out [ : radius ] = np . nan
return out


signature:
def one_sided_ema ( xolds , yolds , low = None , high = None , n = 512 , decay_steps = 1. , low_counts_threshold = 1e-8 ) :
    '''perform one - sided ( causal ) EMA ( exponential moving average )
 smoothing and resampling to an even grid with n points .
 Does not do extrapolation , so we assume
 xolds [ 0 ] < = low & & high < = xolds [ - 1]
 Arguments :
 xolds : array or list - x values of data . Needs to be sorted in ascending order
 yolds : array of list - y values of data . Has to have the same length as xolds
 low : float - min value of the new x grid . By default equals to xolds [ 0]
 high : float - max value of the new x grid . By default equals to xolds [ - 1]
 n : int - number of points in new x grid
 decay _ steps : float - EMA decay factor , expressed in new x grid steps .
 low _ counts _ threshold : float or int
 - y values with counts less than this value will be set to NaN
 Returns :
 tuple sum _ ys , count _ ys where
 xs - array with new x grid
 ys - array of EMA of y at each point of the new x grid
 count _ ys - array of EMA of y counts at each point of the new x grid'''

implementation:
low = xolds [ 0 ] if low is None else low
high = xolds [ - 1 ] if high is None else high
assert xolds [ 0 ] <= low , 'low = {} < xolds[0] = {} - extrapolation not permitted!' . format ( low , xolds [ 0 ] )
assert xolds [ - 1 ] >= high , 'high = {} > xolds[-1] = {}  - extrapolation not permitted!' . format ( high , xolds [ - 1 ] )
assert len ( xolds ) == len ( yolds ) , 'length of xolds ({}) and yolds ({}) do not match!' . format ( len ( xolds ) , len ( yolds ) )
xolds = xolds . astype ( 'float64' )
yolds = yolds . astype ( 'float64' )
luoi = 0
# last unused old index
sum_y = 0.
count_y = 0.
xnews = np . linspace ( low , high , n )
decay_period = ( high - low ) / ( n - 1 ) * decay_steps
interstep_decay = np . exp ( - 1. / decay_steps )
sum_ys = np . zeros_like ( xnews )
count_ys = np . zeros_like ( xnews )
for i in range ( n ) :
    xnew = xnews [ i ]
    sum_y *= interstep_decay
    count_y *= interstep_decay
    while True :
        xold = xolds [ luoi ]
        if xold <= xnew :
            decay = np . exp ( - ( xnew - xold ) / decay_period )
            sum_y += decay * yolds [ luoi ]
            count_y += decay
            luoi += 1
        else :
            break
        if luoi >= len ( xolds ) :
            break
    sum_ys [ i ] = sum_y
    count_ys [ i ] = count_y
ys = sum_ys / count_ys
ys [ count_ys < low_counts_threshold ] = np . nan
return xnews , ys , count_ys


signature:
def symmetric_ema ( xolds , yolds , low = None , high = None , n = 512 , decay_steps = 1. , low_counts_threshold = 1e-8 ) :
    '''perform symmetric EMA ( exponential moving average )
 smoothing and resampling to an even grid with n points .
 Does not do extrapolation , so we assume
 xolds [ 0 ] < = low & & high < = xolds [ - 1]
 Arguments :
 xolds : array or list - x values of data . Needs to be sorted in ascending order
 yolds : array of list - y values of data . Has to have the same length as xolds
 low : float - min value of the new x grid . By default equals to xolds [ 0]
 high : float - max value of the new x grid . By default equals to xolds [ - 1]
 n : int - number of points in new x grid
 decay _ steps : float - EMA decay factor , expressed in new x grid steps .
 low _ counts _ threshold : float or int
 - y values with counts less than this value will be set to NaN
 Returns :
 tuple sum _ ys , count _ ys where
 xs - array with new x grid
 ys - array of EMA of y at each point of the new x grid
 count _ ys - array of EMA of y counts at each point of the new x grid'''

implementation:
xs , ys1 , count_ys1 = one_sided_ema ( xolds , yolds , low , high , n , decay_steps , low_counts_threshold = 0 )
_ , ys2 , count_ys2 = one_sided_ema ( - xolds [ : : - 1 ] , yolds [ : : - 1 ] , - high , - low , n , decay_steps , low_counts_threshold = 0 )
ys2 = ys2 [ : : - 1 ]
count_ys2 = count_ys2 [ : : - 1 ]
count_ys = count_ys1 + count_ys2
ys = ( ys1 * count_ys1 + ys2 * count_ys2 ) / count_ys
ys [ count_ys < low_counts_threshold ] = np . nan
return xs , ys , count_ys


signature:
def load_results ( root_dir_or_dirs , enable_progress = True , enable_monitor = True , verbose = False ) :
    '''load summaries of runs from a list of directories ( including subdirectories )
 Arguments :
 enable _ progress : bool - if True , will attempt to load data from progress . csv files ( data saved by logger ) . Default : True
 enable _ monitor : bool - if True , will attempt to load data from monitor . csv files ( data saved by Monitor environment wrapper ) . Default : True
 verbose : bool - if True , will print out list of directories from which the data is loaded . Default : False
 Returns :
 List of Result objects with the following fields :
 - dirname - path to the directory data was loaded from
 - metadata - run metadata ( such as command - line arguments and anything else in metadata . json file
 - monitor - if enable _ monitor is True , this field contains pandas dataframe with loaded monitor . csv file ( or aggregate of all * . monitor . csv files in the directory )
 - progress - if enable _ progress is True , this field contains pandas dataframe with loaded progress . csv file'''

implementation:
import re
if isinstance ( root_dir_or_dirs , str ) :
    rootdirs = [ osp . expanduser ( root_dir_or_dirs ) ]
else :
    rootdirs = [ osp . expanduser ( d ) for d in root_dir_or_dirs ]
allresults = [ ]
for rootdir in rootdirs :
    assert osp . exists ( rootdir ) , "%s doesn't exist" % rootdir
    for dirname , dirs , files in os . walk ( rootdir ) :
        if '-proc' in dirname :
            files [ : ] = [ ]
            continue
        monitor_re = re . compile ( r'(\d+\.)?(\d+\.)?monitor\.csv' )
        if set ( [ 'metadata.json' , 'monitor.json' , 'progress.json' , 'progress.csv' ] ) . intersection ( files ) or any ( [ f for f in files if monitor_re . match ( f ) ] ) : # also match monitor files like 0.1 . monitor . csv
        # used to be uncommented , which means do not go deeper than current directory if any of the data files
        # are found
        # dirs [ : ] = [ ]
            result = { 'dirname' : dirname }
            if "metadata.json" in files :
                with open ( osp . join ( dirname , "metadata.json" ) , "r" ) as fh :
                    result [ 'metadata' ] = json . load ( fh )
            progjson = osp . join ( dirname , "progress.json" )
            progcsv = osp . join ( dirname , "progress.csv" )
            if enable_progress :
                if osp . exists ( progjson ) :
                    result [ 'progress' ] = pandas . DataFrame ( read_json ( progjson ) )
                elif osp . exists ( progcsv ) :
                    try :
                        result [ 'progress' ] = read_csv ( progcsv )
                    except pandas . errors . EmptyDataError :
                        print ( 'skipping progress file in ' , dirname , 'empty data' )
                else :
                    if verbose :
                        print ( 'skipping %s: no progress file' % dirname )
            if enable_monitor :
                try :
                    result [ 'monitor' ] = pandas . DataFrame ( monitor . load_results ( dirname ) )
                except monitor . LoadMonitorResultsError :
                    print ( 'skipping %s: no monitor files' % dirname )
                except Exception as e :
                    print ( 'exception loading monitor file in %s: %s' % ( dirname , e ) )
            if result . get ( 'monitor' ) is not None or result . get ( 'progress' ) is not None :
                allresults . append ( Result ( ** result ) )
                if verbose :
                    print ( 'successfully loaded %s' % dirname )
if verbose :
    print ( 'loaded %i results' % len ( allresults ) )
return allresults


signature:
def plot_results ( allresults , * , xy_fn = default_xy_fn , split_fn = default_split_fn , group_fn = default_split_fn , average_group = False , shaded_std = True , shaded_err = True , figsize = None , legend_outside = False , resample = 0 , smooth_step = 1.0 ) :
    '''Plot multiple Results objects
 xy _ fn : function Result - > x , y - function that converts results objects into tuple of x and y values .
 By default , x is cumsum of episode lengths , and y is episode rewards
 split _ fn : function Result - > hashable - function that converts results objects into keys to split curves into sub - panels by .
 That is , the results r for which split _ fn ( r ) is different will be put on different sub - panels .
 By default , the portion of r . dirname between last / and - < digits > is returned . The sub - panels are
 stacked vertically in the figure .
 group _ fn : function Result - > hashable - function that converts results objects into keys to group curves by .
 That is , the results r for which group _ fn ( r ) is the same will be put into the same group .
 Curves in the same group have the same color ( if average _ group is False ) , or averaged over
 ( if average _ group is True ) . The default value is the same as default value for split _ fn
 average _ group : bool - if True , will average the curves in the same group and plot the mean . Enables resampling
 ( if resample = 0 , will use 512 steps )
 shaded _ std : bool - if True ( default ) , the shaded region corresponding to standard deviation of the group of curves will be
 shown ( only applicable if average _ group = True )
 shaded _ err : bool - if True ( default ) , the shaded region corresponding to error in mean estimate of the group of curves
 ( that is , standard deviation divided by square root of number of curves ) will be
 shown ( only applicable if average _ group = True )
 figsize : tuple or None - size of the resulting figure ( including sub - panels ) . By default , width is 6 and height is 6 times number of
 sub - panels .
 legend _ outside : bool - if True , will place the legend outside of the sub - panels .
 resample : int - if not zero , size of the uniform grid in x direction to resample onto . Resampling is performed via symmetric
 EMA smoothing ( see the docstring for symmetric _ ema ) .
 Default is zero ( no resampling ) . Note that if average _ group is True , resampling is necessary ; in that case , default
 value is 512.
 smooth _ step : float - when resampling ( i . e . when resample > 0 or average _ group is True ) , use this EMA decay parameter ( in units of the new grid step ) .
 See docstrings for decay _ steps in symmetric _ ema or one _ sided _ ema functions .'''

implementation:
if split_fn is None :
    split_fn = lambda _ : ''
if group_fn is None :
    group_fn = lambda _ : ''
sk2r = defaultdict ( list )
# splitkey2results
for result in allresults :
    splitkey = split_fn ( result )
    sk2r [ splitkey ] . append ( result )
assert len ( sk2r ) > 0
assert isinstance ( resample , int ) , "0: don't resample. <integer>: that many samples"
nrows = len ( sk2r )
ncols = 1
figsize = figsize or ( 6 , 6 * nrows )
f , axarr = plt . subplots ( nrows , ncols , sharex = False , squeeze = False , figsize = figsize )
groups = list ( set ( group_fn ( result ) for result in allresults ) )
default_samples = 512
if average_group :
    resample = resample or default_samples
for ( isplit , sk ) in enumerate ( sorted ( sk2r . keys ( ) ) ) :
    g2l = { }
    g2c = defaultdict ( int )
    sresults = sk2r [ sk ]
    gresults = defaultdict ( list )
    ax = axarr [ isplit ] [ 0 ]
    for result in sresults :
        group = group_fn ( result )
        g2c [ group ] += 1
        x , y = xy_fn ( result )
        if x is None :
            x = np . arange ( len ( y ) )
        x , y = map ( np . asarray , ( x , y ) )
        if average_group :
            gresults [ group ] . append ( ( x , y ) )
        else :
            if resample :
                x , y , counts = symmetric_ema ( x , y , x [ 0 ] , x [ - 1 ] , resample , decay_steps = smooth_step )
            l , = ax . plot ( x , y , color = COLORS [ groups . index ( group ) % len ( COLORS ) ] )
            g2l [ group ] = l
    if average_group :
        for group in sorted ( groups ) :
            xys = gresults [ group ]
            if not any ( xys ) :
                continue
            color = COLORS [ groups . index ( group ) % len ( COLORS ) ]
            origxs = [ xy [ 0 ] for xy in xys ]
            minxlen = min ( map ( len , origxs ) )
            def allequal ( qs ) :
                return all ( ( q == qs [ 0 ] ) . all ( ) for q in qs [ 1 : ] )
            if resample :
                low = max ( x [ 0 ] for x in origxs )
                high = min ( x [ - 1 ] for x in origxs )
                usex = np . linspace ( low , high , resample )
                ys = [ ]
                for ( x , y ) in xys :
                    ys . append ( symmetric_ema ( x , y , low , high , resample , decay_steps = smooth_step ) [ 1 ] )
            else :
                assert allequal ( [ x [ : minxlen ] for x in origxs ] ) , 'If you want to average unevenly sampled data, set resample=<number of samples you want>'
                usex = origxs [ 0 ]
                ys = [ xy [ 1 ] [ : minxlen ] for xy in xys ]
            ymean = np . mean ( ys , axis = 0 )
            ystd = np . std ( ys , axis = 0 )
            ystderr = ystd / np . sqrt ( len ( ys ) )
            l , = axarr [ isplit ] [ 0 ] . plot ( usex , ymean , color = color )
            g2l [ group ] = l
            if shaded_err :
                ax . fill_between ( usex , ymean - ystderr , ymean + ystderr , color = color , alpha = .4 )
            if shaded_std :
                ax . fill_between ( usex , ymean - ystd , ymean + ystd , color = color , alpha = .2 )
    # https : / / matplotlib . org / users / legend _ guide . html
    plt . tight_layout ( )
    if any ( g2l . keys ( ) ) :
        ax . legend ( g2l . values ( ) , [ '%s (%i)' % ( g , g2c [ g ] ) for g in g2l ] if average_group else g2l . keys ( ) , loc = 2 if legend_outside else None , bbox_to_anchor = ( 1 , 1 ) if legend_outside else None )
    ax . set_title ( sk )
return f , axarr


signature:
def check_synced ( localval , comm = None ) :
    """It ' s common to forget to initialize your variables to the same values , or
 ( less commonly ) if you update them in some other way than adam , to get them out of sync .
 This function checks that variables on all MPI workers are the same , and raises
 an AssertionError otherwise
 Arguments :
 comm : MPI communicator
 localval : list of local variables ( list of variables on current worker to be compared with the other workers )"""

implementation:
comm = comm or MPI . COMM_WORLD
vals = comm . gather ( localval )
if comm . rank == 0 :
    assert all ( val == vals [ 0 ] for val in vals [ 1 : ] )


signature:
def copy_obs_dict ( obs ) :
    """Deep - copy an observation dict ."""

implementation: return { k : np . copy ( v ) for k , v in obs . items ( ) }


signature:
def obs_space_info ( obs_space ) :
    """Get dict - structured information about a gym . Space .
 Returns :
 A tuple ( keys , shapes , dtypes ) :
 keys : a list of dict keys .
 shapes : a dict mapping keys to shapes .
 dtypes : a dict mapping keys to dtypes ."""

implementation:
if isinstance ( obs_space , gym . spaces . Dict ) :
    assert isinstance ( obs_space . spaces , OrderedDict )
    subspaces = obs_space . spaces
else :
    subspaces = { None : obs_space }
keys = [ ]
shapes = { }
dtypes = { }
for key , box in subspaces . items ( ) :
    keys . append ( key )
    shapes [ key ] = box . shape
    dtypes [ key ] = box . dtype
return keys , shapes , dtypes


signature:
def q_retrace ( R , D , q_i , v , rho_i , nenvs , nsteps , gamma ) :
    """Calculates q _ retrace targets
 : param R : Rewards
 : param D : Dones
 : param q _ i : Q values for actions taken
 : param v : V values
 : param rho _ i : Importance weight for each action
 : return : Q _ retrace values"""

implementation:
rho_bar = batch_to_seq ( tf . minimum ( 1.0 , rho_i ) , nenvs , nsteps , True )
# list of len steps , shape [ nenvs ]
rs = batch_to_seq ( R , nenvs , nsteps , True )
# list of len steps , shape [ nenvs ]
ds = batch_to_seq ( D , nenvs , nsteps , True )
# list of len steps , shape [ nenvs ]
q_is = batch_to_seq ( q_i , nenvs , nsteps , True )
vs = batch_to_seq ( v , nenvs , nsteps + 1 , True )
v_final = vs [ - 1 ]
qret = v_final
qrets = [ ]
for i in range ( nsteps - 1 , - 1 , - 1 ) :
    check_shape ( [ qret , ds [ i ] , rs [ i ] , rho_bar [ i ] , q_is [ i ] , vs [ i ] ] , [ [ nenvs ] ] * 6 )
    qret = rs [ i ] + gamma * qret * ( 1.0 - ds [ i ] )
    qrets . append ( qret )
    qret = ( rho_bar [ i ] * ( qret - q_is [ i ] ) ) + vs [ i ]
qrets = qrets [ : : - 1 ]
qret = seq_to_batch ( qrets , flat = True )
return qret


signature:
def learn ( network , env , seed = None , nsteps = 20 , total_timesteps = int ( 80e6 ) , q_coef = 0.5 , ent_coef = 0.01 , max_grad_norm = 10 , lr = 7e-4 , lrschedule = 'linear' , rprop_epsilon = 1e-5 , rprop_alpha = 0.99 , gamma = 0.99 , log_interval = 100 , buffer_size = 50000 , replay_ratio = 4 , replay_start = 10000 , c = 10.0 , trust_region = True , alpha = 0.99 , delta = 1 , load_path = None , ** network_kwargs ) :
    '''Main entrypoint for ACER ( Actor - Critic with Experience Replay ) algorithm ( https : / / arxiv . org / pdf / 1611.01224 . pdf )
 Train an agent with given network architecture on a given environment using ACER .
 Parameters :
 network : policy network architecture . Either string ( mlp , lstm , lnlstm , cnn _ lstm , cnn , cnn _ small , conv _ only - see baselines . common / models . py for full list )
 specifying the standard network architecture , or a function that takes tensorflow tensor as input and returns
 tuple ( output _ tensor , extra _ feed ) where output tensor is the last network layer output , extra _ feed is None for feed - forward
 neural nets , and extra _ feed is a dictionary describing how to feed state into the network for recurrent neural nets .
 See baselines . common / policies . py / lstm for more details on using recurrent nets in policies
 env : environment . Needs to be vectorized for parallel environment simulation .
 The environments produced by gym . make can be wrapped using baselines . common . vec _ env . DummyVecEnv class .
 nsteps : int , number of steps of the vectorized environment per update ( i . e . batch size is nsteps * nenv where
 nenv is number of environment copies simulated in parallel ) ( default : 20)
 nstack : int , size of the frame stack , i . e . number of the frames passed to the step model . Frames are stacked along channel dimension
 ( last image dimension ) ( default : 4)
 total _ timesteps : int , number of timesteps ( i . e . number of actions taken in the environment ) ( default : 80M )
 q _ coef : float , value function loss coefficient in the optimization objective ( analog of vf _ coef for other actor - critic methods )
 ent _ coef : float , policy entropy coefficient in the optimization objective ( default : 0.01)
 max _ grad _ norm : float , gradient norm clipping coefficient . If set to None , no clipping . ( default : 10 ) ,
 lr : float , learning rate for RMSProp ( current implementation has RMSProp hardcoded in ) ( default : 7e - 4)
 lrschedule : schedule of learning rate . Can be ' linear ' , ' constant ' , or a function [ 0 . . 1 ] - > [ 0 . . 1 ] that takes fraction of the training progress as input and
 returns fraction of the learning rate ( specified as lr ) as output
 rprop _ epsilon : float , RMSProp epsilon ( stabilizes square root computation in denominator of RMSProp update ) ( default : 1e - 5)
 rprop _ alpha : float , RMSProp decay parameter ( default : 0.99)
 gamma : float , reward discounting factor ( default : 0.99)
 log _ interval : int , number of updates between logging events ( default : 100)
 buffer _ size : int , size of the replay buffer ( default : 50k )
 replay _ ratio : int , now many ( on average ) batches of data to sample from the replay buffer take after batch from the environment ( default : 4)
 replay _ start : int , the sampling from the replay buffer does not start until replay buffer has at least that many samples ( default : 10k )
 c : float , importance weight clipping factor ( default : 10)
 trust _ region bool , whether or not algorithms estimates the gradient KL divergence between the old and updated policy and uses it to determine step size ( default : True )
 delta : float , max KL divergence between the old policy and updated policy ( default : 1)
 alpha : float , momentum factor in the Polyak ( exponential moving average ) averaging of the model parameters ( default : 0.99)
 load _ path : str , path to load the model from ( default : None )
 * * network _ kwargs : keyword arguments to the policy / network builder . See baselines . common / policies . py / build _ policy and arguments to a particular type of network
 For instance , ' mlp ' network architecture has arguments num _ hidden and num _ layers .'''

implementation:
print ( "Running Acer Simple" )
print ( locals ( ) )
set_global_seeds ( seed )
if not isinstance ( env , VecFrameStack ) :
    env = VecFrameStack ( env , 1 )
policy = build_policy ( env , network , estimate_q = True , ** network_kwargs )
nenvs = env . num_envs
ob_space = env . observation_space
ac_space = env . action_space
nstack = env . nstack
model = Model ( policy = policy , ob_space = ob_space , ac_space = ac_space , nenvs = nenvs , nsteps = nsteps , ent_coef = ent_coef , q_coef = q_coef , gamma = gamma , max_grad_norm = max_grad_norm , lr = lr , rprop_alpha = rprop_alpha , rprop_epsilon = rprop_epsilon , total_timesteps = total_timesteps , lrschedule = lrschedule , c = c , trust_region = trust_region , alpha = alpha , delta = delta )
runner = Runner ( env = env , model = model , nsteps = nsteps )
if replay_ratio > 0 :
    buffer = Buffer ( env = env , nsteps = nsteps , size = buffer_size )
else :
    buffer = None
nbatch = nenvs * nsteps
acer = Acer ( runner , model , buffer , log_interval )
acer . tstart = time . time ( )
for acer . steps in range ( 0 , total_timesteps , nbatch ) : # nbatch samples , 1 on _ policy call and multiple off - policy calls
    acer . call ( on_policy = True )
    if replay_ratio > 0 and buffer . has_atleast ( replay_start ) :
        n = np . random . poisson ( replay_ratio )
        for _ in range ( n ) :
            acer . call ( on_policy = False )
# no simulation steps in this
return model


signature:
def apply_stats ( self , statsUpdates ) :
    """compute stats and update / apply the new stats to the running average"""

implementation:
def updateAccumStats ( ) :
    if self . _full_stats_init :
        return tf . cond ( tf . greater ( self . sgd_step , self . _cold_iter ) , lambda : tf . group ( * self . _apply_stats ( statsUpdates , accumulate = True , accumulateCoeff = 1. / self . _stats_accum_iter ) ) , tf . no_op )
    else :
        return tf . group ( * self . _apply_stats ( statsUpdates , accumulate = True , accumulateCoeff = 1. / self . _stats_accum_iter ) )
def updateRunningAvgStats ( statsUpdates , fac_iter = 1 ) : # return tf . cond ( tf . greater _ equal ( self . factor _ step ,
# tf . convert _ to _ tensor ( fac _ iter ) ) , lambda :
# tf . group ( * self . _ apply _ stats ( stats _ list , varlist ) ) , tf . no _ op )
    return tf . group ( * self . _apply_stats ( statsUpdates ) )
if self . _async_stats : # asynchronous stats update
    update_stats = self . _apply_stats ( statsUpdates )
    queue = tf . FIFOQueue ( 1 , [ item . dtype for item in update_stats ] , shapes = [ item . get_shape ( ) for item in update_stats ] )
    enqueue_op = queue . enqueue ( update_stats )
    def dequeue_stats_op ( ) :
        return queue . dequeue ( )
    self . qr_stats = tf . train . QueueRunner ( queue , [ enqueue_op ] )
    update_stats_op = tf . cond ( tf . equal ( queue . size ( ) , tf . convert_to_tensor ( 0 ) ) , tf . no_op , lambda : tf . group ( * [ dequeue_stats_op ( ) , ] ) )
else : # synchronous stats update
    update_stats_op = tf . cond ( tf . greater_equal ( self . stats_step , self . _stats_accum_iter ) , lambda : updateRunningAvgStats ( statsUpdates ) , updateAccumStats )
self . _update_stats_op = update_stats_op
return update_stats_op


signature:
def tile_images ( img_nhwc ) :
    """Tile N images into one big PxQ image
 ( P , Q ) are chosen to be as close as possible , and if N
 is square , then P = Q .
 input : img _ nhwc , list or array of images , ndim = 4 once turned into array
 n = batch index , h = height , w = width , c = channel
 returns :
 bigim _ HWc , ndarray with ndim = 3"""

implementation:
img_nhwc = np . asarray ( img_nhwc )
N , h , w , c = img_nhwc . shape
H = int ( np . ceil ( np . sqrt ( N ) ) )
W = int ( np . ceil ( float ( N ) / H ) )
img_nhwc = np . array ( list ( img_nhwc ) + [ img_nhwc [ 0 ] * 0 for _ in range ( N , H * W ) ] )
img_HWhwc = img_nhwc . reshape ( H , W , h , w , c )
img_HhWwc = img_HWhwc . transpose ( 0 , 2 , 1 , 3 , 4 )
img_Hh_Ww_c = img_HhWwc . reshape ( H * h , W * w , c )
return img_Hh_Ww_c


signature:
def sum ( self , start = 0 , end = None ) :
    """Returns arr [ start ] + . . . + arr [ end ]"""

implementation: return super ( SumSegmentTree , self ) . reduce ( start , end )


signature:
def find_prefixsum_idx ( self , prefixsum ) :
    """Find the highest index ` i ` in the array such that
 sum ( arr [ 0 ] + arr [ 1 ] + . . . + arr [ i - i ] ) < = prefixsum
 if array values are probabilities , this function
 allows to sample indexes according to the discrete
 probability efficiently .
 Parameters
 perfixsum : float
 upperbound on the sum of array prefix
 Returns
 idx : int
 highest index satisfying the prefixsum constraint"""

implementation:
assert 0 <= prefixsum <= self . sum ( ) + 1e-5
idx = 1
while idx < self . _capacity : # while non - leaf
    if self . _value [ 2 * idx ] > prefixsum :
        idx = 2 * idx
    else :
        prefixsum -= self . _value [ 2 * idx ]
        idx = 2 * idx + 1
return idx - self . _capacity


signature:
def min ( self , start = 0 , end = None ) :
    """Returns min ( arr [ start ] , . . . , arr [ end ] )"""

implementation: return super ( MinSegmentTree , self ) . reduce ( start , end )


signature:
def value ( self , t ) :
    """See Schedule . value"""

implementation:
for ( l_t , l ) , ( r_t , r ) in zip ( self . _endpoints [ : - 1 ] , self . _endpoints [ 1 : ] ) :
    if l_t <= t and t < r_t :
        alpha = float ( t - l_t ) / ( r_t - l_t )
        return self . _interpolation ( l , r , alpha )
# t does not belong to any of the pieces , so doom .
assert self . _outside_value is not None
return self . _outside_value


signature:
def _subproc_worker ( pipe , parent_pipe , env_fn_wrapper , obs_bufs , obs_shapes , obs_dtypes , keys ) :
    """Control a single environment instance using IPC and
 shared memory ."""

implementation:
def _write_obs ( maybe_dict_obs ) :
    flatdict = obs_to_dict ( maybe_dict_obs )
    for k in keys :
        dst = obs_bufs [ k ] . get_obj ( )
        dst_np = np . frombuffer ( dst , dtype = obs_dtypes [ k ] ) . reshape ( obs_shapes [ k ] )
        # pylint : disable = W0212
        np . copyto ( dst_np , flatdict [ k ] )
env = env_fn_wrapper . x ( )
parent_pipe . close ( )
try :
    while True :
        cmd , data = pipe . recv ( )
        if cmd == 'reset' :
            pipe . send ( _write_obs ( env . reset ( ) ) )
        elif cmd == 'step' :
            obs , reward , done , info = env . step ( data )
            if done :
                obs = env . reset ( )
            pipe . send ( ( _write_obs ( obs ) , reward , done , info ) )
        elif cmd == 'render' :
            pipe . send ( env . render ( mode = 'rgb_array' ) )
        elif cmd == 'close' :
            pipe . send ( None )
            break
        else :
            raise RuntimeError ( 'Got unrecognized cmd %s' % cmd )
except KeyboardInterrupt :
    print ( 'ShmemVecEnv worker: got KeyboardInterrupt' )
finally :
    env . close ( )


signature:
def learn ( network , env , seed = None , nsteps = 5 , total_timesteps = int ( 80e6 ) , vf_coef = 0.5 , ent_coef = 0.01 , max_grad_norm = 0.5 , lr = 7e-4 , lrschedule = 'linear' , epsilon = 1e-5 , alpha = 0.99 , gamma = 0.99 , log_interval = 100 , load_path = None , ** network_kwargs ) :
    '''Main entrypoint for A2C algorithm . Train a policy with given network architecture on a given environment using a2c algorithm .
 Parameters :
 network : policy network architecture . Either string ( mlp , lstm , lnlstm , cnn _ lstm , cnn , cnn _ small , conv _ only - see baselines . common / models . py for full list )
 specifying the standard network architecture , or a function that takes tensorflow tensor as input and returns
 tuple ( output _ tensor , extra _ feed ) where output tensor is the last network layer output , extra _ feed is None for feed - forward
 neural nets , and extra _ feed is a dictionary describing how to feed state into the network for recurrent neural nets .
 See baselines . common / policies . py / lstm for more details on using recurrent nets in policies
 env : RL environment . Should implement interface similar to VecEnv ( baselines . common / vec _ env ) or be wrapped with DummyVecEnv ( baselines . common / vec _ env / dummy _ vec _ env . py )
 seed : seed to make random number sequence in the alorightm reproducible . By default is None which means seed from system noise generator ( not reproducible )
 nsteps : int , number of steps of the vectorized environment per update ( i . e . batch size is nsteps * nenv where
 nenv is number of environment copies simulated in parallel )
 total _ timesteps : int , total number of timesteps to train on ( default : 80M )
 vf _ coef : float , coefficient in front of value function loss in the total loss function ( default : 0.5)
 ent _ coef : float , coeffictiant in front of the policy entropy in the total loss function ( default : 0.01)
 max _ gradient _ norm : float , gradient is clipped to have global L2 norm no more than this value ( default : 0.5)
 lr : float , learning rate for RMSProp ( current implementation has RMSProp hardcoded in ) ( default : 7e - 4)
 lrschedule : schedule of learning rate . Can be ' linear ' , ' constant ' , or a function [ 0 . . 1 ] - > [ 0 . . 1 ] that takes fraction of the training progress as input and
 returns fraction of the learning rate ( specified as lr ) as output
 epsilon : float , RMSProp epsilon ( stabilizes square root computation in denominator of RMSProp update ) ( default : 1e - 5)
 alpha : float , RMSProp decay parameter ( default : 0.99)
 gamma : float , reward discounting parameter ( default : 0.99)
 log _ interval : int , specifies how frequently the logs are printed out ( default : 100)
 * * network _ kwargs : keyword arguments to the policy / network builder . See baselines . common / policies . py / build _ policy and arguments to a particular type of network
 For instance , ' mlp ' network architecture has arguments num _ hidden and num _ layers .'''

implementation:
set_global_seeds ( seed )
# Get the nb of env
nenvs = env . num_envs
policy = build_policy ( env , network , ** network_kwargs )
# Instantiate the model object ( that creates step _ model and train _ model )
model = Model ( policy = policy , env = env , nsteps = nsteps , ent_coef = ent_coef , vf_coef = vf_coef , max_grad_norm = max_grad_norm , lr = lr , alpha = alpha , epsilon = epsilon , total_timesteps = total_timesteps , lrschedule = lrschedule )
if load_path is not None :
    model . load ( load_path )
# Instantiate the runner object
runner = Runner ( env , model , nsteps = nsteps , gamma = gamma )
epinfobuf = deque ( maxlen = 100 )
# Calculate the batch _ size
nbatch = nenvs * nsteps
# Start total timer
tstart = time . time ( )
for update in range ( 1 , total_timesteps // nbatch + 1 ) : # Get mini batch of experiences
    obs , states , rewards , masks , actions , values , epinfos = runner . run ( )
    epinfobuf . extend ( epinfos )
    policy_loss , value_loss , policy_entropy = model . train ( obs , states , rewards , masks , actions , values )
    nseconds = time . time ( ) - tstart
    # Calculate the fps ( frame per second )
    fps = int ( ( update * nbatch ) / nseconds )
    if update % log_interval == 0 or update == 1 : # Calculates if value function is a good predicator of the returns ( ev > 1)
    # or if it ' s just worse than predicting nothing ( ev = < 0)
        ev = explained_variance ( values , rewards )
        logger . record_tabular ( "nupdates" , update )
        logger . record_tabular ( "total_timesteps" , update * nbatch )
        logger . record_tabular ( "fps" , fps )
        logger . record_tabular ( "policy_entropy" , float ( policy_entropy ) )
        logger . record_tabular ( "value_loss" , float ( value_loss ) )
        logger . record_tabular ( "explained_variance" , float ( ev ) )
        logger . record_tabular ( "eprewmean" , safemean ( [ epinfo [ 'r' ] for epinfo in epinfobuf ] ) )
        logger . record_tabular ( "eplenmean" , safemean ( [ epinfo [ 'l' ] for epinfo in epinfobuf ] ) )
        logger . dump_tabular ( )
return model


signature:
def sf01 ( arr ) :
    """swap and then flatten axes 0 and 1"""

implementation:
s = arr . shape
return arr . swapaxes ( 0 , 1 ) . reshape ( s [ 0 ] * s [ 1 ] , * s [ 2 : ] )


signature:
def step ( self , observation , ** extra_feed ) :
    """Compute next action ( s ) given the observation ( s )
 Parameters :
 observation observation data ( either single or a batch )
 * * extra _ feed additional data such as state or mask ( names of the arguments should match the ones in constructor , see _ _ init _ _ )
 Returns :
 ( action , value estimate , next state , negative log likelihood of the action under current policy parameters ) tuple"""

implementation:
a , v , state , neglogp = self . _evaluate ( [ self . action , self . vf , self . state , self . neglogp ] , observation , ** extra_feed )
if state . size == 0 :
    state = None
return a , v , state , neglogp


signature:
def value ( self , ob , * args , ** kwargs ) :
    """Compute value estimate ( s ) given the observation ( s )
 Parameters :
 observation observation data ( either single or a batch )
 * * extra _ feed additional data such as state or mask ( names of the arguments should match the ones in constructor , see _ _ init _ _ )
 Returns :
 value estimate"""

implementation: return self . _evaluate ( self . vf , ob , * args , ** kwargs )


signature:
def pretty_eta ( seconds_left ) :
    """Print the number of seconds in human readable format .
 Examples :
 2 days
 2 hours and 37 minutes
 less than a minute
 Paramters
 seconds _ left : int
 Number of seconds to be converted to the ETA
 Returns
 eta : str
 String representing the pretty ETA ."""

implementation:
minutes_left = seconds_left // 60
seconds_left %= 60
hours_left = minutes_left // 60
minutes_left %= 60
days_left = hours_left // 24
hours_left %= 24
def helper ( cnt , name ) :
    return "{} {}{}" . format ( str ( cnt ) , name , ( 's' if cnt > 1 else '' ) )
if days_left > 0 :
    msg = helper ( days_left , 'day' )
    if hours_left > 0 :
        msg += ' and ' + helper ( hours_left , 'hour' )
    return msg
if hours_left > 0 :
    msg = helper ( hours_left , 'hour' )
    if minutes_left > 0 :
        msg += ' and ' + helper ( minutes_left , 'minute' )
    return msg
if minutes_left > 0 :
    return helper ( minutes_left , 'minute' )
return 'less than a minute'


signature:
def boolean_flag ( parser , name , default = False , help = None ) :
    """Add a boolean flag to argparse parser .
 Parameters
 parser : argparse . Parser
 parser to add the flag to
 name : str
 - - < name > will enable the flag , while - - no - < name > will disable it
 default : bool or None
 default value of the flag
 help : str
 help string for the flag"""

implementation:
dest = name . replace ( '-' , '_' )
parser . add_argument ( "--" + name , action = "store_true" , default = default , dest = dest , help = help )
parser . add_argument ( "--no-" + name , action = "store_false" , dest = dest )


signature:
def get_wrapper_by_name ( env , classname ) :
    """Given an a gym environment possibly wrapped multiple times , returns a wrapper
 of class named classname or raises ValueError if no such wrapper was applied
 Parameters
 env : gym . Env of gym . Wrapper
 gym environment
 classname : str
 name of the wrapper
 Returns
 wrapper : gym . Wrapper
 wrapper named classname"""

implementation:
currentenv = env
while True :
    if classname == currentenv . class_name ( ) :
        return currentenv
    elif isinstance ( currentenv , gym . Wrapper ) :
        currentenv = currentenv . env
    else :
        raise ValueError ( "Couldn't find wrapper named %s" % classname )


signature:
def relatively_safe_pickle_dump ( obj , path , compression = False ) :
    """This is just like regular pickle dump , except from the fact that failure cases are
 different :
 - It ' s never possible that we end up with a pickle in corrupted state .
 - If a there was a different file at the path , that file will remain unchanged in the
 even of failure ( provided that filesystem rename is atomic ) .
 - it is sometimes possible that we end up with useless temp file which needs to be
 deleted manually ( it will be removed automatically on the next function call )
 The indended use case is periodic checkpoints of experiment state , such that we never
 corrupt previous checkpoints if the current one fails .
 Parameters
 obj : object
 object to pickle
 path : str
 path to the output file
 compression : bool
 if true pickle will be compressed"""

implementation:
temp_storage = path + ".relatively_safe"
if compression : # Using gzip here would be simpler , but the size is limited to 2GB
    with tempfile . NamedTemporaryFile ( ) as uncompressed_file :
        pickle . dump ( obj , uncompressed_file )
        uncompressed_file . file . flush ( )
        with zipfile . ZipFile ( temp_storage , "w" , compression = zipfile . ZIP_DEFLATED ) as myzip :
            myzip . write ( uncompressed_file . name , "data" )
else :
    with open ( temp_storage , "wb" ) as f :
        pickle . dump ( obj , f )
os . rename ( temp_storage , path )


signature:
def pickle_load ( path , compression = False ) :
    """Unpickle a possible compressed pickle .
 Parameters
 path : str
 path to the output file
 compression : bool
 if true assumes that pickle was compressed when created and attempts decompression .
 Returns
 obj : object
 the unpickled object"""

implementation:
if compression :
    with zipfile . ZipFile ( path , "r" , compression = zipfile . ZIP_DEFLATED ) as myzip :
        with myzip . open ( "data" ) as f :
            return pickle . load ( f )
else :
    with open ( path , "rb" ) as f :
        return pickle . load ( f )


signature:
def update ( self , new_val ) :
    """Update the estimate .
 Parameters
 new _ val : float
 new observated value of estimated quantity ."""

implementation:
if self . _value is None :
    self . _value = new_val
else :
    self . _value = self . _gamma * self . _value + ( 1.0 - self . _gamma ) * new_val


signature:
def store_args ( method ) :
    """Stores provided method args as instance attributes ."""

implementation:
argspec = inspect . getfullargspec ( method )
defaults = { }
if argspec . defaults is not None :
    defaults = dict ( zip ( argspec . args [ - len ( argspec . defaults ) : ] , argspec . defaults ) )
if argspec . kwonlydefaults is not None :
    defaults . update ( argspec . kwonlydefaults )
arg_names = argspec . args [ 1 : ]
@ functools . wraps ( method )
def wrapper ( * positional_args , ** keyword_args ) :
    self = positional_args [ 0 ]
    # Get default arg values
    args = defaults . copy ( )
    # Add provided arg values
    for name , value in zip ( arg_names , positional_args [ 1 : ] ) :
        args [ name ] = value
    args . update ( keyword_args )
    self . __dict__ . update ( args )
    return method ( * positional_args , ** keyword_args )
return wrapper


signature:
def import_function ( spec ) :
    """Import a function identified by a string like " pkg . module : fn _ name " ."""

implementation:
mod_name , fn_name = spec . split ( ':' )
module = importlib . import_module ( mod_name )
fn = getattr ( module , fn_name )
return fn


signature:
def flatten_grads ( var_list , grads ) :
    """Flattens a variables and their gradients ."""

implementation: return tf . concat ( [ tf . reshape ( grad , [ U . numel ( v ) ] ) for ( v , grad ) in zip ( var_list , grads ) ] , 0 )


signature:
def nn ( input , layers_sizes , reuse = None , flatten = False , name = "" ) :
    """Creates a simple neural network"""

implementation:
for i , size in enumerate ( layers_sizes ) :
    activation = tf . nn . relu if i < len ( layers_sizes ) - 1 else None
    input = tf . layers . dense ( inputs = input , units = size , kernel_initializer = tf . contrib . layers . xavier_initializer ( ) , reuse = reuse , name = name + '_' + str ( i ) )
    if activation :
        input = activation ( input )
if flatten :
    assert layers_sizes [ - 1 ] == 1
    input = tf . reshape ( input , [ - 1 ] )
return input


signature:
def mpi_fork ( n , extra_mpi_args = [ ] ) :
    """Re - launches the current script with workers
 Returns " parent " for original parent , " child " for MPI children"""

implementation:
if n <= 1 :
    return "child"
if os . getenv ( "IN_MPI" ) is None :
    env = os . environ . copy ( )
    env . update ( MKL_NUM_THREADS = "1" , OMP_NUM_THREADS = "1" , IN_MPI = "1" )
    # " - bind - to core " is crucial for good performance
    args = [ "mpirun" , "-np" , str ( n ) ] + extra_mpi_args + [ sys . executable ]
    args += sys . argv
    subprocess . check_call ( args , env = env )
    return "parent"
else :
    install_mpi_excepthook ( )
    return "child"


signature:
def convert_episode_to_batch_major ( episode ) :
    """Converts an episode to have the batch dimension in the major ( first )
 dimension ."""

implementation:
episode_batch = { }
for key in episode . keys ( ) :
    val = np . array ( episode [ key ] ) . copy ( )
    # make inputs batch - major instead of time - major
    episode_batch [ key ] = val . swapaxes ( 0 , 1 )
return episode_batch


signature:
def reshape_for_broadcasting ( source , target ) :
    """Reshapes a tensor ( source ) to have the correct shape and dtype of the target
 before broadcasting it with MPI ."""

implementation:
dim = len ( target . get_shape ( ) )
shape = ( [ 1 ] * ( dim - 1 ) ) + [ - 1 ]
return tf . reshape ( tf . cast ( source , target . dtype ) , shape )


signature:
def add_vtarg_and_adv ( seg , gamma , lam ) :
    """Compute target value using TD ( lambda ) estimator , and advantage with GAE ( lambda )"""

implementation:
new = np . append ( seg [ "new" ] , 0 )
# last element is only used for last vtarg , but we already zeroed it if last new = 1
vpred = np . append ( seg [ "vpred" ] , seg [ "nextvpred" ] )
T = len ( seg [ "rew" ] )
seg [ "adv" ] = gaelam = np . empty ( T , 'float32' )
rew = seg [ "rew" ]
lastgaelam = 0
for t in reversed ( range ( T ) ) :
    nonterminal = 1 - new [ t + 1 ]
    delta = rew [ t ] + gamma * vpred [ t + 1 ] * nonterminal - vpred [ t ]
    gaelam [ t ] = lastgaelam = delta + gamma * lam * nonterminal * lastgaelam
seg [ "tdlamret" ] = seg [ "adv" ] + seg [ "vpred" ]


signature:
def switch ( condition , then_expression , else_expression ) :
    """Switches between two operations depending on a scalar value ( int or bool ) .
 Note that both ` then _ expression ` and ` else _ expression `
 should be symbolic tensors of the * same shape * .
 # Arguments
 condition : scalar tensor .
 then _ expression : TensorFlow operation .
 else _ expression : TensorFlow operation ."""

implementation:
x_shape = copy . copy ( then_expression . get_shape ( ) )
x = tf . cond ( tf . cast ( condition , 'bool' ) , lambda : then_expression , lambda : else_expression )
x . set_shape ( x_shape )
return x


signature:
def huber_loss ( x , delta = 1.0 ) :
    """Reference : https : / / en . wikipedia . org / wiki / Huber _ loss"""

implementation: return tf . where ( tf . abs ( x ) < delta , tf . square ( x ) * 0.5 , delta * ( tf . abs ( x ) - 0.5 * delta ) )


signature:
def get_session ( config = None ) :
    """Get default session or create one with a given config"""

implementation:
sess = tf . get_default_session ( )
if sess is None :
    sess = make_session ( config = config , make_default = True )
return sess


signature:
def make_session ( config = None , num_cpu = None , make_default = False , graph = None ) :
    """Returns a session that will use < num _ cpu > CPU ' s only"""

implementation:
if num_cpu is None :
    num_cpu = int ( os . getenv ( 'RCALL_NUM_CPU' , multiprocessing . cpu_count ( ) ) )
if config is None :
    config = tf . ConfigProto ( allow_soft_placement = True , inter_op_parallelism_threads = num_cpu , intra_op_parallelism_threads = num_cpu )
    config . gpu_options . allow_growth = True
if make_default :
    return tf . InteractiveSession ( config = config , graph = graph )
else :
    return tf . Session ( config = config , graph = graph )


signature:
def initialize ( ) :
    """Initialize all the uninitialized variables in the global scope ."""

implementation:
new_variables = set ( tf . global_variables ( ) ) - ALREADY_INITIALIZED
get_session ( ) . run ( tf . variables_initializer ( new_variables ) )
ALREADY_INITIALIZED . update ( new_variables )


signature:
def function ( inputs , outputs , updates = None , givens = None ) :
    """Just like Theano function . Take a bunch of tensorflow placeholders and expressions
 computed based on those placeholders and produces f ( inputs ) - > outputs . Function f takes
 values to be fed to the input ' s placeholders and produces the values of the expressions
 in outputs .
 Input values can be passed in the same order as inputs or can be provided as kwargs based
 on placeholder name ( passed to constructor or accessible via placeholder . op . name ) .
 Example :
 x = tf . placeholder ( tf . int32 , ( ) , name = " x " )
 y = tf . placeholder ( tf . int32 , ( ) , name = " y " )
 z = 3 * x + 2 * y
 lin = function ( [ x , y ] , z , givens = { y : 0 } )
 with single _ threaded _ session ( ) :
 initialize ( )
 assert lin ( 2 ) = = 6
 assert lin ( x = 3 ) = = 9
 assert lin ( 2 , 2 ) = = 10
 assert lin ( x = 2 , y = 3 ) = = 12
 Parameters
 inputs : [ tf . placeholder , tf . constant , or object with make _ feed _ dict method ]
 list of input arguments
 outputs : [ tf . Variable ] or tf . Variable
 list of outputs or a single output to be returned from function . Returned
 value will also have the same shape .
 updates : [ tf . Operation ] or tf . Operation
 list of update functions or single update function that will be run whenever
 the function is called . The return is ignored ."""

implementation:
if isinstance ( outputs , list ) :
    return _Function ( inputs , outputs , updates , givens = givens )
elif isinstance ( outputs , ( dict , collections . OrderedDict ) ) :
    f = _Function ( inputs , outputs . values ( ) , updates , givens = givens )
    return lambda * args , ** kwargs : type ( outputs ) ( zip ( outputs . keys ( ) , f ( * args , ** kwargs ) ) )
else :
    f = _Function ( inputs , [ outputs ] , updates , givens = givens )
    return lambda * args , ** kwargs : f ( * args , ** kwargs ) [ 0 ]


signature:
def adjust_shape ( placeholder , data ) :
    '''adjust shape of the data to the shape of the placeholder if possible .
 If shape is incompatible , AssertionError is thrown
 Parameters :
 placeholder tensorflow input placeholder
 data input data to be ( potentially ) reshaped to be fed into placeholder
 Returns :
 reshaped data'''

implementation:
if not isinstance ( data , np . ndarray ) and not isinstance ( data , list ) :
    return data
if isinstance ( data , list ) :
    data = np . array ( data )
placeholder_shape = [ x or - 1 for x in placeholder . shape . as_list ( ) ]
assert _check_shape ( placeholder_shape , data . shape ) , 'Shape of data {} is not compatible with shape of the placeholder {}' . format ( data . shape , placeholder_shape )
return np . reshape ( data , placeholder_shape )


signature:
def _check_shape ( placeholder_shape , data_shape ) :
    '''check if two shapes are compatible ( i . e . differ only by dimensions of size 1 , or by the batch dimension )'''

implementation:
return True
squeezed_placeholder_shape = _squeeze_shape ( placeholder_shape )
squeezed_data_shape = _squeeze_shape ( data_shape )
for i , s_data in enumerate ( squeezed_data_shape ) :
    s_placeholder = squeezed_placeholder_shape [ i ]
    if s_placeholder != - 1 and s_data != s_placeholder :
        return False
return True


signature:
def profile ( n ) :
    """Usage :
 @ profile ( " my _ func " )
 def my _ func ( ) : code"""

implementation:
def decorator_with_name ( func ) :
    def func_wrapper ( * args , ** kwargs ) :
        with profile_kv ( n ) :
            return func ( * args , ** kwargs )
    return func_wrapper
return decorator_with_name


signature:
def wrap_deepmind ( env , episode_life = True , clip_rewards = True , frame_stack = False , scale = False ) :
    """Configure environment for DeepMind - style Atari ."""

implementation:
if episode_life :
    env = EpisodicLifeEnv ( env )
if 'FIRE' in env . unwrapped . get_action_meanings ( ) :
    env = FireResetEnv ( env )
env = WarpFrame ( env )
if scale :
    env = ScaledFloatFrame ( env )
if clip_rewards :
    env = ClipRewardEnv ( env )
if frame_stack :
    env = FrameStack ( env , 4 )
return env


signature:
def reset ( self , ** kwargs ) :
    """Reset only when lives are exhausted .
 This way all states are still reachable even though lives are episodic ,
 and the learner need not know about any of this behind - the - scenes ."""

implementation:
if self . was_real_done :
    obs = self . env . reset ( ** kwargs )
else : # no - op step to advance from terminal / lost life state
    obs , _ , _ , _ = self . env . step ( 0 )
self . lives = self . env . unwrapped . ale . lives ( )
return obs


signature:
def sync_from_root ( sess , variables , comm = None ) :
    """Send the root node ' s parameters to every worker .
 Arguments :
 sess : the TensorFlow session .
 variables : all parameter variables including optimizer ' s"""

implementation:
if comm is None :
    comm = MPI . COMM_WORLD
import tensorflow as tf
values = comm . bcast ( sess . run ( variables ) )
sess . run ( [ tf . assign ( var , val ) for ( var , val ) in zip ( variables , values ) ] )


signature:
def gpu_count ( ) :
    """Count the GPUs on this machine ."""

implementation:
if shutil . which ( 'nvidia-smi' ) is None :
    return 0
output = subprocess . check_output ( [ 'nvidia-smi' , '--query-gpu=gpu_name' , '--format=csv' ] )
return max ( 0 , len ( output . split ( b'\n' ) ) - 2 )


signature:
def setup_mpi_gpus ( ) :
    """Set CUDA _ VISIBLE _ DEVICES to MPI rank if not already set"""

implementation:
if 'CUDA_VISIBLE_DEVICES' not in os . environ :
    if sys . platform == 'darwin' : # This Assumes if you ' re on OSX you ' re just
        ids = [ ]
        # doing a smoke test and don ' t want GPUs
    else :
        lrank , _lsize = get_local_rank_size ( MPI . COMM_WORLD )
        ids = [ lrank ]
    os . environ [ "CUDA_VISIBLE_DEVICES" ] = "," . join ( map ( str , ids ) )


signature:
def get_local_rank_size ( comm ) :
    """Returns the rank of each process on its machine
 The processes on a given machine will be assigned ranks
 0 , 1 , 2 , . . . , N - 1,
 where N is the number of processes on this machine .
 Useful if you want to assign one gpu per machine"""

implementation:
this_node = platform . node ( )
ranks_nodes = comm . allgather ( ( comm . Get_rank ( ) , this_node ) )
node2rankssofar = defaultdict ( int )
local_rank = None
for ( rank , node ) in ranks_nodes :
    if rank == comm . Get_rank ( ) :
        local_rank = node2rankssofar [ node ]
    node2rankssofar [ node ] += 1
assert local_rank is not None
return local_rank , node2rankssofar [ this_node ]


signature:
def share_file ( comm , path ) :
    """Copies the file from rank 0 to all other ranks
 Puts it in the same place on all machines"""

implementation:
localrank , _ = get_local_rank_size ( comm )
if comm . Get_rank ( ) == 0 :
    with open ( path , 'rb' ) as fh :
        data = fh . read ( )
    comm . bcast ( data )
else :
    data = comm . bcast ( None )
    if localrank == 0 :
        os . makedirs ( os . path . dirname ( path ) , exist_ok = True )
        with open ( path , 'wb' ) as fh :
            fh . write ( data )
comm . Barrier ( )


signature:
def dict_gather ( comm , d , op = 'mean' , assert_all_have_data = True ) :
    """Perform a reduction operation over dicts"""

implementation:
if comm is None :
    return d
alldicts = comm . allgather ( d )
size = comm . size
k2li = defaultdict ( list )
for d in alldicts :
    for ( k , v ) in d . items ( ) :
        k2li [ k ] . append ( v )
result = { }
for ( k , li ) in k2li . items ( ) :
    if assert_all_have_data :
        assert len ( li ) == size , "only %i out of %i MPI workers have sent '%s'" % ( len ( li ) , size , k )
    if op == 'mean' :
        result [ k ] = np . mean ( li , axis = 0 )
    elif op == 'sum' :
        result [ k ] = np . sum ( li , axis = 0 )
    else :
        assert 0 , op
return result


signature:
def mpi_weighted_mean ( comm , local_name2valcount ) :
    """Perform a weighted average over dicts that are each on a different node
 Input : local _ name2valcount : dict mapping key - > ( value , count )
 Returns : key - > mean"""

implementation:
all_name2valcount = comm . gather ( local_name2valcount )
if comm . rank == 0 :
    name2sum = defaultdict ( float )
    name2count = defaultdict ( float )
    for n2vc in all_name2valcount :
        for ( name , ( val , count ) ) in n2vc . items ( ) :
            try :
                val = float ( val )
            except ValueError :
                if comm . rank == 0 :
                    warnings . warn ( 'WARNING: tried to compute mean on non-float {}={}' . format ( name , val ) )
            else :
                name2sum [ name ] += val * count
                name2count [ name ] += count
    return { name : name2sum [ name ] / name2count [ name ] for name in name2sum }
else :
    return { }