(val, keys, inverse_keys, axis=0)
return permute(keys, inverse_keys, val)
Permutation helper for LSH attention.
Permutation helper for LSH attention.
Permutation helper for LSH attention .
def permute_via_sort(val, keys, inverse_keys, axis=0):
  """Permutation helper for LSH attention."""
  def permute_impl(k, unused_ik, val):
    # On TPU, sorting scalars by key is faster than a gather.
    _, permuted = fastmath.sort_key_val(k, val, dimension=axis)
    return permuted
  def permute_fwd(k, ik, val):
    # On TPU, sorting scalars by key is faster than a gather.
    _, permuted = fastmath.sort_key_val(k, val, dimension=axis)
    return permuted, ik
  def permute_bwd(ik, permuted_grad):
    _, val_grad = fastmath.sort_key_val(
        ik, permuted_grad, dimension=axis)
    return (None, None, val_grad)
  permute = fastmath.custom_vjp(permute_impl, permute_fwd, permute_bwd)
  return permute(keys, inverse_keys, val)
def permute_via_sort ( val , keys , inverse_keys , axis = 0 ) : def permute_impl ( k , unused_ik , val ) : # On TPU, sorting scalars by key is faster than a gather. _ , permuted = fastmath . sort_key_val ( k , val , dimension = axis ) return permuted def permute_fwd ( k , ik , val ) : # On TPU, sorting scalars by key is faster than a gather. _ , permuted = fastmath . sort_key_val ( k , val , dimension = axis ) return permuted , ik def permute_bwd ( ik , permuted_grad ) : _ , val_grad = fastmath . sort_key_val ( ik , permuted_grad , dimension = axis ) return ( None , None , val_grad ) permute = fastmath . custom_vjp ( permute_impl , permute_fwd , permute_bwd ) return permute ( keys , inverse_keys , val )