(self, text)
return (torch.IntTensor(text), torch.IntTensor(length))
Support batch or single str.

        Args:
            text (str or list of str): texts to convert.

        Returns:
            torch.IntTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
            torch.IntTensor [n]: length of each text.
Support batch or single str.
Support batch or single str .
def encode(self, text):
        """Support batch or single str.

        Args:
            text (str or list of str): texts to convert.

        Returns:
            torch.IntTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
            torch.IntTensor [n]: length of each text.
        """
        if isinstance(text, str):
            text = text.split(self.sep)
            text = [self.dict[item] for item in text]
            length = [len(text)]
        elif isinstance(text, collections.Iterable):
            length = [len(s.split(self.sep)) for s in text]
            text = self.sep.join(text)
            text, _ = self.encode(text)
        return (torch.IntTensor(text), torch.IntTensor(length))
def encode ( self , text ) : if isinstance ( text , str ) : text = text . split ( self . sep ) text = [ self . dict [ item ] for item in text ] length = [ len ( text ) ] elif isinstance ( text , collections . Iterable ) : length = [ len ( s . split ( self . sep ) ) for s in text ] text = self . sep . join ( text ) text , _ = self . encode ( text ) return ( torch . IntTensor ( text ) , torch . IntTensor ( length ) )