(ids: tf.Tensor, vocab_filename: str, encoder_type: str)
return text
DecodeOp.
DecodeOp.
DecodeOp .
def decode(ids: tf.Tensor, vocab_filename: str, encoder_type: str):
  """DecodeOp."""
  if encoder_type not in ["sentencepiece", "sentencepiece_newline"]:
    raise ValueError("Unsupported encoder type: %s" % encoder_type)
  sp_model = tf.gfile.GFile(vocab_filename, "rb").read()
  tokenizer = tf_text.SentencepieceTokenizer(model=sp_model)
  ids = tf.where(ids > 1 + _SHIFT_RESERVED_TOKENS, ids - _SHIFT_RESERVED_TOKENS,
                 ids)
  ids = tf.cast(ids, tf.int32)
  text = tokenizer.detokenize(ids)
  text = tf.reshape(text, [-1])
  if encoder_type == "sentencepiece_newline":
    text = tf.strings.regex_replace(text, _NEWLINE_SYMBOL, "\n")
  return text
def decode ( ids : tf . Tensor , vocab_filename : str , encoder_type : str ) : if encoder_type not in [ "sentencepiece" , "sentencepiece_newline" ] : raise ValueError ( "Unsupported encoder type: %s" % encoder_type ) sp_model = tf . gfile . GFile ( vocab_filename , "rb" ) . read ( ) tokenizer = tf_text . SentencepieceTokenizer ( model = sp_model ) ids = tf . where ( ids > 1 + _SHIFT_RESERVED_TOKENS , ids - _SHIFT_RESERVED_TOKENS , ids ) ids = tf . cast ( ids , tf . int32 ) text = tokenizer . detokenize ( ids ) text = tf . reshape ( text , [ - 1 ] ) if encoder_type == "sentencepiece_newline" : text = tf . strings . regex_replace ( text , _NEWLINE_SYMBOL , "\n" ) return text