(self)
return tl.Fn('A2CLoss', f)
Definition of the Advantage Actor Critic (A2C) loss.
Definition of the Advantage Actor Critic (A2C) loss.
Definition of the Advantage Actor Critic ( A2C ) loss .
def policy_loss_given_log_probs(self):
    """Definition of the Advantage Actor Critic (A2C) loss."""
    # A2C is one of the most basic actor-critic RL algorithms.
    # TODO(henrykm) re-factor f into rl_layers and finally share code between
    # actor_critic.py and actor_critic_joint.py - requires change of inputs
    # in actor_critic_joint.py from dist_inputs to log_probs.
    def f(log_probs, advantages, old_log_probs, mask):
      del old_log_probs  # Not used in A2C.
      # log_probs of the shape float32[128,1]
      # advantages of the shape int32[128,1]
      # mask of the shape int32[128,1]
      if log_probs.shape != advantages.shape:
        raise ValueError('New log-probs and advantages shapes '
                         'should be the same, %s != %s' % (log_probs.shape,
                                                           advantages.shape))
      if log_probs.shape != mask.shape:
        raise ValueError('New log-probs and mask shapes should be the same'
                         ', %s != %s' % (log_probs.shape, mask.shape))

      a2c_objective = -jnp.sum(log_probs * advantages * mask) / jnp.sum(mask)

      entropy_vec = self._policy_dist.entropy(log_probs) * self._entropy_coeff
      entropy_loss = jnp.mean(entropy_vec)

      combined_loss = a2c_objective - entropy_loss

      return combined_loss

    return tl.Fn('A2CLoss', f)
def policy_loss_given_log_probs ( self ) : # A2C is one of the most basic actor-critic RL algorithms. # TODO(henrykm) re-factor f into rl_layers and finally share code between # actor_critic.py and actor_critic_joint.py - requires change of inputs # in actor_critic_joint.py from dist_inputs to log_probs. def f ( log_probs , advantages , old_log_probs , mask ) : del old_log_probs # Not used in A2C. # log_probs of the shape float32[128,1] # advantages of the shape int32[128,1] # mask of the shape int32[128,1] if log_probs . shape != advantages . shape : raise ValueError ( 'New log-probs and advantages shapes ' 'should be the same, %s != %s' % ( log_probs . shape , advantages . shape ) ) if log_probs . shape != mask . shape : raise ValueError ( 'New log-probs and mask shapes should be the same' ', %s != %s' % ( log_probs . shape , mask . shape ) ) a2c_objective = - jnp . sum ( log_probs * advantages * mask ) / jnp . sum ( mask ) entropy_vec = self . _policy_dist . entropy ( log_probs ) * self . _entropy_coeff entropy_loss = jnp . mean ( entropy_vec ) combined_loss = a2c_objective - entropy_loss return combined_loss return tl . Fn ( 'A2CLoss' , f )