(stream, spm_path=None, extra_ids=0)
Sentencepiece tokenization.
Sentencepiece tokenization.
Sentencepiece tokenization .
def sentencepiece_tokenize(stream, spm_path=None, extra_ids=0):
  """Sentencepiece tokenization."""
  spm_path = spm_path or t5_data().DEFAULT_SPM_PATH
  vocab_file = os.path.basename(spm_path)
  vocab_dir = os.path.dirname(spm_path)
  vocab = _get_vocab(vocab_type='sentencepiece',
                     vocab_file=vocab_file,
                     vocab_dir=vocab_dir,
                     extra_ids=extra_ids)
  for example in stream:
    # example could either be str or (str,)
    if isinstance(example, tuple):
      example = example[0]
    yield np.array(vocab.encode(example))
def sentencepiece_tokenize ( stream , spm_path = None , extra_ids = 0 ) : spm_path = spm_path or t5_data ( ) . DEFAULT_SPM_PATH vocab_file = os . path . basename ( spm_path ) vocab_dir = os . path . dirname ( spm_path ) vocab = _get_vocab ( vocab_type = 'sentencepiece' , vocab_file = vocab_file , vocab_dir = vocab_dir , extra_ids = extra_ids ) for example in stream : # example could either be str or (str,) if isinstance ( example , tuple ) : example = example [ 0 ] yield np . array ( vocab . encode ( example ) )