(x,
               vocab_type='subword',
               vocab_file=None,
               vocab_dir=None,
               n_reserved_ids=0)
return str(vocab.decode(x_unreserved.tolist()))
Maps integer arrays to text; the opposite of `tokenize`.

  In many cases (all char- and subword-type vocabularies and most sentencepiece
  ones) the tokenization is invertible, so detokenize(tokenize(x)) = x. In some
  more rare cases this can remove some spacing, but it is still often useful
  to run detokenize to get a readable version for a tokenized string.

  Args:
    x: a list or numpy array of integers.
    vocab_type: Type of vocabulary, one of: 'subword', 'sentencepiece', 'char'.
    vocab_file: Name of the vocabulary file.
    vocab_dir: Directory which contains the vocabulary file.
    n_reserved_ids: An int, offset added so 0, ..., n_reserved_ids-1 are unused;
      This is common for example when reserving the 0 for padding and 1 for EOS,
      but it's only needed if these symbols are not already included (and thus
      reserved) in the vocab_file.

  Returns:
    A string corresponding to the de-tokenized version of x.
Maps integer arrays to text; the opposite of `tokenize`.
Maps integer arrays to text ; the opposite of tokenize .
def detokenize(x,
               vocab_type='subword',
               vocab_file=None,
               vocab_dir=None,
               n_reserved_ids=0):
  """Maps integer arrays to text; the opposite of `tokenize`.

  In many cases (all char- and subword-type vocabularies and most sentencepiece
  ones) the tokenization is invertible, so detokenize(tokenize(x)) = x. In some
  more rare cases this can remove some spacing, but it is still often useful
  to run detokenize to get a readable version for a tokenized string.

  Args:
    x: a list or numpy array of integers.
    vocab_type: Type of vocabulary, one of: 'subword', 'sentencepiece', 'char'.
    vocab_file: Name of the vocabulary file.
    vocab_dir: Directory which contains the vocabulary file.
    n_reserved_ids: An int, offset added so 0, ..., n_reserved_ids-1 are unused;
      This is common for example when reserving the 0 for padding and 1 for EOS,
      but it's only needed if these symbols are not already included (and thus
      reserved) in the vocab_file.

  Returns:
    A string corresponding to the de-tokenized version of x.
  """
  vocab = _get_vocab(vocab_type, vocab_file, vocab_dir)
  x_unreserved = np.array(x) - n_reserved_ids
  return str(vocab.decode(x_unreserved.tolist()))
def detokenize ( x , vocab_type = 'subword' , vocab_file = None , vocab_dir = None , n_reserved_ids = 0 ) : vocab = _get_vocab ( vocab_type , vocab_file , vocab_dir ) x_unreserved = np . array ( x ) - n_reserved_ids return str ( vocab . decode ( x_unreserved . tolist ( ) ) )