(predict_fn, loss_fn, n_devices, jit=True)
return compute_loss
Returns a (JIT-compiled) function that computes the loss for one step.
Returns a (JIT-compiled) function that computes the loss for one step.
Returns a ( JIT - compiled ) function that computes the loss for one step .
def _jit_compute_loss_fn(predict_fn, loss_fn, n_devices, jit=True):
  """Returns a (JIT-compiled) function that computes the loss for one step."""
  if n_devices == 1:  # TODO(lukaszkaiser): remove branch when not needed.
    def single_compute_loss(opt_state, batch, state, rng):
      rng, subrng = jax_random.split(rng[0])
      loss_val, state = loss_fn(opt_state[0], batch, predict_fn, state, rng)
      return loss_val, state, [subrng]
    return fastmath.jit(single_compute_loss) if jit else single_compute_loss

  # Else, for n_devices > 1:
  @functools.partial(fastmath.pmap, axis_name='batch')
  def mapped_compute_loss(opt_state, batch, state, rng):
    """This is a multi-device version of the update function above."""
    # We assume all tensors have the first dimension = n_devices.
    rng, subrng = jax_random.split(rng)
    loss_val, state = loss_fn(opt_state[0], batch, predict_fn, state, rng)
    return loss_val, state, subrng

  def compute_loss(opt_state, batch, state, rng):
    return mapped_compute_loss(
        opt_state, _reshape_by_device(batch, n_devices), state, rng)

  return compute_loss
def _jit_compute_loss_fn ( predict_fn , loss_fn , n_devices , jit = True ) : if n_devices == 1 : # TODO(lukaszkaiser): remove branch when not needed. def single_compute_loss ( opt_state , batch , state , rng ) : rng , subrng = jax_random . split ( rng [ 0 ] ) loss_val , state = loss_fn ( opt_state [ 0 ] , batch , predict_fn , state , rng ) return loss_val , state , [ subrng ] return fastmath . jit ( single_compute_loss ) if jit else single_compute_loss # Else, for n_devices > 1: @ functools . partial ( fastmath . pmap , axis_name = 'batch' ) def mapped_compute_loss ( opt_state , batch , state , rng ) : """This is a multi-device version of the update function above.""" # We assume all tensors have the first dimension = n_devices. rng , subrng = jax_random . split ( rng ) loss_val , state = loss_fn ( opt_state [ 0 ] , batch , predict_fn , state , rng ) return loss_val , state , subrng def compute_loss ( opt_state , batch , state , rng ) : return mapped_compute_loss ( opt_state , _reshape_by_device ( batch , n_devices ) , state , rng ) return compute_loss