(self, task, policy_model=None, policy_optimizer=None,
               policy_lr_schedule=lr.multifactor, policy_batch_size=64,
               policy_train_steps_per_epoch=500, policy_evals_per_epoch=1,
               policy_eval_steps=1, n_eval_episodes=0,
               only_eval=False, max_slice_length=1, output_dir=None, **kwargs)
Configures the policy trainer.

    Args:
      task: RLTask instance, which defines the environment to train on.
      policy_model: Trax layer, representing the policy model.
          functions and eval functions (a.k.a. metrics) are considered to be
          outside the core model, taking core model output and data labels as
          their two inputs.
      policy_optimizer: the optimizer to use to train the policy model.
      policy_lr_schedule: learning rate schedule to use to train the policy.
      policy_batch_size: batch size used to train the policy model.
      policy_train_steps_per_epoch: how long to train policy in each RL epoch.
      policy_evals_per_epoch: number of policy trainer evaluations per RL epoch
          - only affects metric reporting.
      policy_eval_steps: number of policy trainer steps per evaluation - only
          affects metric reporting.
      n_eval_episodes: number of episodes to play with policy at
        temperature 0 in each epoch -- used for evaluation only
      only_eval: If set to True, then trajectories are collected only for
        for evaluation purposes, but they are not recorded.
      max_slice_length: the maximum length of trajectory slices to use.
      output_dir: Path telling where to save outputs (evals and checkpoints).
      **kwargs: arguments for the superclass Agent.
Configures the policy trainer.
Configures the policy trainer .
def __init__(self, task, policy_model=None, policy_optimizer=None,
               policy_lr_schedule=lr.multifactor, policy_batch_size=64,
               policy_train_steps_per_epoch=500, policy_evals_per_epoch=1,
               policy_eval_steps=1, n_eval_episodes=0,
               only_eval=False, max_slice_length=1, output_dir=None, **kwargs):
    """Configures the policy trainer.

    Args:
      task: RLTask instance, which defines the environment to train on.
      policy_model: Trax layer, representing the policy model.
          functions and eval functions (a.k.a. metrics) are considered to be
          outside the core model, taking core model output and data labels as
          their two inputs.
      policy_optimizer: the optimizer to use to train the policy model.
      policy_lr_schedule: learning rate schedule to use to train the policy.
      policy_batch_size: batch size used to train the policy model.
      policy_train_steps_per_epoch: how long to train policy in each RL epoch.
      policy_evals_per_epoch: number of policy trainer evaluations per RL epoch
          - only affects metric reporting.
      policy_eval_steps: number of policy trainer steps per evaluation - only
          affects metric reporting.
      n_eval_episodes: number of episodes to play with policy at
        temperature 0 in each epoch -- used for evaluation only
      only_eval: If set to True, then trajectories are collected only for
        for evaluation purposes, but they are not recorded.
      max_slice_length: the maximum length of trajectory slices to use.
      output_dir: Path telling where to save outputs (evals and checkpoints).
      **kwargs: arguments for the superclass Agent.
    """
    super().__init__(
        task,
        n_eval_episodes=n_eval_episodes,
        output_dir=output_dir,
        **kwargs
    )
    self._policy_batch_size = policy_batch_size
    self._policy_train_steps_per_epoch = policy_train_steps_per_epoch
    self._policy_evals_per_epoch = policy_evals_per_epoch
    self._policy_eval_steps = policy_eval_steps
    self._only_eval = only_eval
    self._max_slice_length = max_slice_length
    self._policy_dist = distributions.create_distribution(task.action_space)

    # Inputs to the policy model are produced by self._policy_batches_stream.
    self._policy_inputs = data.inputs.Inputs(
        train_stream=lambda _: self.policy_batches_stream())

    policy_model = functools.partial(
        policy_model,
        policy_distribution=self._policy_dist,
    )

    # This is the policy Trainer that will be used to train the policy model.
    # * inputs to the trainer come from self.policy_batches_stream
    # * outputs, targets and weights are passed to self.policy_loss
    self._policy_trainer = supervised.Trainer(
        model=policy_model,
        optimizer=policy_optimizer,
        lr_schedule=policy_lr_schedule(),
        loss_fn=self.policy_loss,
        inputs=self._policy_inputs,
        output_dir=output_dir,
        metrics=self.policy_metrics,
    )
    self._policy_collect_model = tl.Accelerate(
        policy_model(mode='collect'), n_devices=1)
    policy_batch = next(self.policy_batches_stream())
    self._policy_collect_model.init(shapes.signature(policy_batch))
    self._policy_eval_model = tl.Accelerate(
        policy_model(mode='eval'), n_devices=1)  # Not collecting stats
    self._policy_eval_model.init(shapes.signature(policy_batch))
def __init__ ( self , task , policy_model = None , policy_optimizer = None , policy_lr_schedule = lr . multifactor , policy_batch_size = 64 , policy_train_steps_per_epoch = 500 , policy_evals_per_epoch = 1 , policy_eval_steps = 1 , n_eval_episodes = 0 , only_eval = False , max_slice_length = 1 , output_dir = None , * * kwargs ) : super ( ) . __init__ ( task , n_eval_episodes = n_eval_episodes , output_dir = output_dir , * * kwargs ) self . _policy_batch_size = policy_batch_size self . _policy_train_steps_per_epoch = policy_train_steps_per_epoch self . _policy_evals_per_epoch = policy_evals_per_epoch self . _policy_eval_steps = policy_eval_steps self . _only_eval = only_eval self . _max_slice_length = max_slice_length self . _policy_dist = distributions . create_distribution ( task . action_space ) # Inputs to the policy model are produced by self._policy_batches_stream. self . _policy_inputs = data . inputs . Inputs ( train_stream = lambda _ : self . policy_batches_stream ( ) ) policy_model = functools . partial ( policy_model , policy_distribution = self . _policy_dist , ) # This is the policy Trainer that will be used to train the policy model. # * inputs to the trainer come from self.policy_batches_stream # * outputs, targets and weights are passed to self.policy_loss self . _policy_trainer = supervised . Trainer ( model = policy_model , optimizer = policy_optimizer , lr_schedule = policy_lr_schedule ( ) , loss_fn = self . policy_loss , inputs = self . _policy_inputs , output_dir = output_dir , metrics = self . policy_metrics , ) self . _policy_collect_model = tl . Accelerate ( policy_model ( mode = 'collect' ) , n_devices = 1 ) policy_batch = next ( self . policy_batches_stream ( ) ) self . _policy_collect_model . init ( shapes . signature ( policy_batch ) ) self . _policy_eval_model = tl . Accelerate ( policy_model ( mode = 'eval' ) , n_devices = 1 ) # Not collecting stats self . _policy_eval_model . init ( shapes . signature ( policy_batch ) )