(self, grad_stack, step, layer, inp, state,
                             fbo_fn, rng, optimizer, replicated_opt_params)
return stats, grad_stack
Run reversible layers backwards.
Run reversible layers backwards.
Run reversible layers backwards .
def _run_backward_standard(self, grad_stack, step, layer, inp, state,
                             fbo_fn, rng, optimizer, replicated_opt_params):
    """Run reversible layers backwards."""
    step_int = int(step) if self._n_devices < 2 else int(step[0])
    if step_int % self._n_steps_per_log == 1:
      logging.info('running backward standard layer %s', str(layer))
    if grad_stack is not None:
      grads = cb.inputs_from_stack(grad_stack, layer.n_out)
    else:
      grads = None
    slots = self._replicate(optimizer.slots)
    weights = self._replicate(layer.weights)
    # Ensure all arguments are on accelerator.
    state = tl.on_accelerator(state)
    replicated_opt_params = tl.on_accelerator(replicated_opt_params)
    rng = tl.on_accelerator(rng)
    grads = tl.on_accelerator(grads)
    inp = tl.on_accelerator(inp)
    new_weights, new_state, new_slots, new_grads, stats = fbo_fn(
        inp, weights, grads, state, slots, replicated_opt_params, rng, step)
    layer.weights = self._lazy_unreplicate(new_weights)
    layer.state = self._unreplicate(new_state)
    optimizer.slots = self._unreplicate(new_slots)
    if grad_stack is not None:
      grad_stack = cb.outputs_onto_stack(new_grads, grad_stack, layer.n_out)
    else:
      grad_stack = new_grads
    return stats, grad_stack
def _run_backward_standard ( self , grad_stack , step , layer , inp , state , fbo_fn , rng , optimizer , replicated_opt_params ) : step_int = int ( step ) if self . _n_devices < 2 else int ( step [ 0 ] ) if step_int % self . _n_steps_per_log == 1 : logging . info ( 'running backward standard layer %s' , str ( layer ) ) if grad_stack is not None : grads = cb . inputs_from_stack ( grad_stack , layer . n_out ) else : grads = None slots = self . _replicate ( optimizer . slots ) weights = self . _replicate ( layer . weights ) # Ensure all arguments are on accelerator. state = tl . on_accelerator ( state ) replicated_opt_params = tl . on_accelerator ( replicated_opt_params ) rng = tl . on_accelerator ( rng ) grads = tl . on_accelerator ( grads ) inp = tl . on_accelerator ( inp ) new_weights , new_state , new_slots , new_grads , stats = fbo_fn ( inp , weights , grads , state , slots , replicated_opt_params , rng , step ) layer . weights = self . _lazy_unreplicate ( new_weights ) layer . state = self . _unreplicate ( new_state ) optimizer . slots = self . _unreplicate ( new_slots ) if grad_stack is not None : grad_stack = cb . outputs_onto_stack ( new_grads , grad_stack , layer . n_out ) else : grad_stack = new_grads return stats , grad_stack