(tensors, n_shards=None)
return fastmath.nested_map(_shard_fn, tensors)
Shard tensors across n_shards.
Shard tensors across n_shards.
Shard tensors across n_shards .
def shard(tensors, n_shards=None):
  """Shard tensors across n_shards."""
  n_shards = N_WEIGHTS_SHARDS if n_shards is None else n_shards
  indices = _axis_index(np.zeros(fastmath.local_device_count()))
  def _shard_fn(x):
    axis = _axis_to_shard_heuristic(x.shape)
    if int(x.shape[axis]) % n_shards != 0:
      raise ValueError(f'Cannot split x with shape {x.shape} into {n_shards}.')
    split_x = jnp.split(x, n_shards, axis=axis)
    split_x = [split_x[i % n_shards] for i in indices]
    return np.stack(split_x, axis=0)
  return fastmath.nested_map(_shard_fn, tensors)
def shard ( tensors , n_shards = None ) : n_shards = N_WEIGHTS_SHARDS if n_shards is None else n_shards indices = _axis_index ( np . zeros ( fastmath . local_device_count ( ) ) ) def _shard_fn ( x ) : axis = _axis_to_shard_heuristic ( x . shape ) if int ( x . shape [ axis ] ) % n_shards != 0 : raise ValueError ( f'Cannot split x with shape {x.shape} into {n_shards}.' ) split_x = jnp . split ( x , n_shards , axis = axis ) split_x = [ split_x [ i % n_shards ] for i in indices ] return np . stack ( split_x , axis = 0 ) return fastmath . nested_map ( _shard_fn , tensors )