(self, sequence)
return detok
Detokenize a single sequence

           Same args/returns as `tokenize`
Detokenize a single sequence
Detokenize a single sequence
def detokenize(self, sequence):
        """Detokenize a single sequence

           Same args/returns as `tokenize`
        """
        if self.tokenizer is None:
            raise ValueError("No tokenizer loaded")

        if self.tokenizer_opt["type"] == "sentencepiece":
            detok = self.tokenizer.DecodePieces(sequence.split())
        elif self.tokenizer_opt["type"] == "pyonmttok":
            detok = self.tokenizer.detokenize(sequence.split())

        return detok
def detokenize ( self , sequence ) : if self . tokenizer is None : raise ValueError ( "No tokenizer loaded" ) if self . tokenizer_opt [ "type" ] == "sentencepiece" : detok = self . tokenizer . DecodePieces ( sequence . split ( ) ) elif self . tokenizer_opt [ "type" ] == "pyonmttok" : detok = self . tokenizer . detokenize ( sequence . split ( ) ) return detok