(
    collect_model,
    policy_distribution,
    loop,
    trajectory_np,
    head_index=0,
    temperature=1.0,
)
return result
Policy function powered by a neural network.

  Used to implement Agent.policy() in policy-based agents.

  Args:
    collect_model: the model used for collecting trajectories
    policy_distribution: an instance of trax.rl.distributions.Distribution
    loop: trax.supervised.training.Loop used to train the policy network
    trajectory_np: an instance of trax.rl.task.TimeStepBatch
    head_index: index of the policy head a multihead model.
    temperature: temperature used to sample from the policy (default=1.0)

  Returns:
    a pair (action, dist_inputs) where action is the action taken and
    dist_inputs is the parameters of the policy distribution, that will later
    be used for training.
Policy function powered by a neural network.
Policy function powered by a neural network .
def network_policy(
    collect_model,
    policy_distribution,
    loop,
    trajectory_np,
    head_index=0,
    temperature=1.0,
):
  """Policy function powered by a neural network.

  Used to implement Agent.policy() in policy-based agents.

  Args:
    collect_model: the model used for collecting trajectories
    policy_distribution: an instance of trax.rl.distributions.Distribution
    loop: trax.supervised.training.Loop used to train the policy network
    trajectory_np: an instance of trax.rl.task.TimeStepBatch
    head_index: index of the policy head a multihead model.
    temperature: temperature used to sample from the policy (default=1.0)

  Returns:
    a pair (action, dist_inputs) where action is the action taken and
    dist_inputs is the parameters of the policy distribution, that will later
    be used for training.
  """
  if temperature == 1.0:
    model = collect_model
  else:
    # When evaluating (t != 1.0), use the evaluation model instead of the
    # collection model - some models accumulate normalization statistics
    # during data collection, and we don't want to do it in eval to avoid data
    # leakage.
    model = loop.eval_model
    model.state = collect_model.state
  # Copying weights from loop.model should work, because the raw model's
  # weights should be updated automatically during training, but it doesn't.
  # TODO(pkozakowski): Debug.
  acc = loop._trainer_per_task[0].accelerated_model_with_loss  # pylint: disable=protected-access
  model.weights = acc._unreplicate(acc.weights[0])  # pylint: disable=protected-access
  # Add batch dimension to trajectory_np and run the model.
  pred = model(trajectory_np.observation[None, ...])
  if isinstance(pred, (tuple, list)):
    # For multihead models, extract the policy head output.
    pred = pred[head_index]
  assert pred.shape == (
      1, trajectory_np.observation.shape[0], policy_distribution.n_inputs
  )
  # Pick element 0 from the batch (the only one), last (current) timestep.
  pred = pred[0, -1, :]
  sample = policy_distribution.sample(pred, temperature=temperature)
  result = (sample, pred)
  if fastmath.is_backend(fastmath.Backend.JAX):
    # The result is composed of mutable numpy arrays. We copy them to avoid
    # accidental modification.
    result = fastmath.nested_map(lambda x: x.copy(), result)
  return result
def network_policy ( collect_model , policy_distribution , loop , trajectory_np , head_index = 0 , temperature = 1.0 , ) : if temperature == 1.0 : model = collect_model else : # When evaluating (t != 1.0), use the evaluation model instead of the # collection model - some models accumulate normalization statistics # during data collection, and we don't want to do it in eval to avoid data # leakage. model = loop . eval_model model . state = collect_model . state # Copying weights from loop.model should work, because the raw model's # weights should be updated automatically during training, but it doesn't. # TODO(pkozakowski): Debug. acc = loop . _trainer_per_task [ 0 ] . accelerated_model_with_loss # pylint: disable=protected-access model . weights = acc . _unreplicate ( acc . weights [ 0 ] ) # pylint: disable=protected-access # Add batch dimension to trajectory_np and run the model. pred = model ( trajectory_np . observation [ None , ... ] ) if isinstance ( pred , ( tuple , list ) ) : # For multihead models, extract the policy head output. pred = pred [ head_index ] assert pred . shape == ( 1 , trajectory_np . observation . shape [ 0 ] , policy_distribution . n_inputs ) # Pick element 0 from the batch (the only one), last (current) timestep. pred = pred [ 0 , - 1 , : ] sample = policy_distribution . sample ( pred , temperature = temperature ) result = ( sample , pred ) if fastmath . is_backend ( fastmath . Backend . JAX ) : # The result is composed of mutable numpy arrays. We copy them to avoid # accidental modification. result = fastmath . nested_map ( lambda x : x . copy ( ) , result ) return result