(self, task)
return optimizers.ReversibleSerialTrainer(
        blocks, loss_layer, task.optimizer,
        free_accelerators_on_step=(self._use_memory_efficient_trainer == 2),
        adasum=self._adasum)
Initializes the per-task trainer.
Initializes the per-task trainer.
Initializes the per - task trainer .
def _init_trainer(self, task):
    """Initializes the per-task trainer."""
    # Build the per-task model, sharing weights with other tasks.
    if not self._use_memory_efficient_trainer:
      model_in_training = _model_with_ends(
          self._model,
          [task.loss_layer],
          shapes.signature(task.sample_batch)
      )
      if base.N_WEIGHTS_SHARDS > 1:
        sharded_weights = fastmath.nested_map(
            lambda x: x[0], tl.shard(model_in_training.weights))
        task.optimizer.tree_init(sharded_weights)
      else:
        task.optimizer.tree_init(model_in_training.weights)
      return optimizers.Trainer(
          model_in_training, task.optimizer, adasum=self._adasum)
    # In the memory-efficient path, we initialize the model here.
    blocks, loss_layer = optimizers.trainer.extract_reversible_blocks(
        [self._model, task.loss_layer], loss_chunk_size=self._loss_chunk_size)
    rng = self._model.rng
    sig = shapes.signature(task.sample_batch)
    optimizers.trainer.init_reversible_blocks(blocks, loss_layer, sig, rng)
    # TODO(lukaszkaiser): here optimizer is a function, revisit this.
    return optimizers.ReversibleSerialTrainer(
        blocks, loss_layer, task.optimizer,
        free_accelerators_on_step=(self._use_memory_efficient_trainer == 2),
        adasum=self._adasum)
def _init_trainer ( self , task ) : # Build the per-task model, sharing weights with other tasks. if not self . _use_memory_efficient_trainer : model_in_training = _model_with_ends ( self . _model , [ task . loss_layer ] , shapes . signature ( task . sample_batch ) ) if base . N_WEIGHTS_SHARDS > 1 : sharded_weights = fastmath . nested_map ( lambda x : x [ 0 ] , tl . shard ( model_in_training . weights ) ) task . optimizer . tree_init ( sharded_weights ) else : task . optimizer . tree_init ( model_in_training . weights ) return optimizers . Trainer ( model_in_training , task . optimizer , adasum = self . _adasum ) # In the memory-efficient path, we initialize the model here. blocks , loss_layer = optimizers . trainer . extract_reversible_blocks ( [ self . _model , task . loss_layer ] , loss_chunk_size = self . _loss_chunk_size ) rng = self . _model . rng sig = shapes . signature ( task . sample_batch ) optimizers . trainer . init_reversible_blocks ( blocks , loss_layer , sig , rng ) # TODO(lukaszkaiser): here optimizer is a function, revisit this. return optimizers . ReversibleSerialTrainer ( blocks , loss_layer , task . optimizer , free_accelerators_on_step = ( self . _use_memory_efficient_trainer == 2 ) , adasum = self . _adasum )