(self, sequence)
return tok
Tokenize a single sequence

            Args:
                sequence: (str) the sequence to tokenize

            Returns:
                tok: (str) the tokenized sequence
Tokenize a single sequence
Tokenize a single sequence
def tokenize(self, sequence):
        """Tokenize a single sequence

            Args:
                sequence: (str) the sequence to tokenize

            Returns:
                tok: (str) the tokenized sequence

        """
        if self.tokenizer is None:
            raise ValueError("No tokenizer loaded")

        if self.tokenizer_opt["type"] == "sentencepiece":
            tok = self.tokenizer.EncodeAsPieces(sequence)
            tok = " ".join(tok)
        elif self.tokenizer_opt["type"] == "pyonmttok":
            tok, _ = self.tokenizer.tokenize(sequence)
            tok = " ".join(tok)
        return tok
def tokenize ( self , sequence ) : if self . tokenizer is None : raise ValueError ( "No tokenizer loaded" ) if self . tokenizer_opt [ "type" ] == "sentencepiece" : tok = self . tokenizer . EncodeAsPieces ( sequence ) tok = " " . join ( tok ) elif self . tokenizer_opt [ "type" ] == "pyonmttok" : tok , _ = self . tokenizer . tokenize ( sequence ) tok = " " . join ( tok ) return tok