(val, permutation, inverse_permutation, axis=0)
return permute(permutation, inverse_permutation, val)
Permutation helper for LSH attention.
Permutation helper for LSH attention.
Permutation helper for LSH attention .
def permute_via_gather(val, permutation, inverse_permutation, axis=0):
  """Permutation helper for LSH attention."""
  def permute_impl(p, unused_ip, val):
    return np.take(val, p, axis=axis)
  def permute_fwd(p, ip, val):
    return np.take(val, p, axis=axis), ip
  def permute_bwd(ip, permuted_grad):
    # JAX autodiff would synthesize a scatter operation because it doesn't
    # know that the indices are a permutation. However on TPU, gathers are
    # faster than scatters (at least in the regime the LSH attention uses).
    return (None, None, np.take(permuted_grad, ip, axis=axis))
  permute = fastmath.custom_vjp(permute_impl, permute_fwd, permute_bwd)
  return permute(permutation, inverse_permutation, val)
def permute_via_gather ( val , permutation , inverse_permutation , axis = 0 ) : def permute_impl ( p , unused_ip , val ) : return np . take ( val , p , axis = axis ) def permute_fwd ( p , ip , val ) : return np . take ( val , p , axis = axis ) , ip def permute_bwd ( ip , permuted_grad ) : # JAX autodiff would synthesize a scatter operation because it doesn't # know that the indices are a permutation. However on TPU, gathers are # faster than scatters (at least in the regime the LSH attention uses). return ( None , None , np . take ( permuted_grad , ip , axis = axis ) ) permute = fastmath . custom_vjp ( permute_impl , permute_fwd , permute_bwd ) return permute ( permutation , inverse_permutation , val )