(self, tokens: Union[str, List[str]])
Looks up indices of text tokens according to the vocabulary
        Args:
            tokens (Union[str, List[str]]): a source token or tokens to be converted
        Returns:
            Union[int, List[int]]: a token index or a list of token indices according to the vocabulary
Looks up indices of text tokens according to the vocabulary
        Args:
            tokens (Union[str, List[str]]): a source token or tokens to be converted
        Returns:
            Union[int, List[int]]: a token index or a list of token indices according to the vocabulary
Looks up indices of text tokens according to the vocabulary Args : tokens ( Union [ str List [ str ]] ) : a source token or tokens to be converted Returns : Union [ int List [ int ]] : a token index or a list of token indices according to the vocabulary
def to_indices(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:
        """Looks up indices of text tokens according to the vocabulary
        Args:
            tokens (Union[str, List[str]]): a source token or tokens to be converted
        Returns:
            Union[int, List[int]]: a token index or a list of token indices according to the vocabulary
        """
        if isinstance(tokens, list):
            return [
                self._token_to_idx[tkn]
                if tkn in self._token_to_idx
                else self._token_to_idx[self._unknown_token]
                for tkn in tokens
            ]
        else:
            return (
                self._token_to_idx[tokens]
                if tokens in self._token_to_idx
                else self._token_to_idx[self._unknown_token]
            )
def to_indices ( self , tokens : Union [ str , List [ str ] ] ) -> Union [ int , List [ int ] ] : if isinstance ( tokens , list ) : return [ self . _token_to_idx [ tkn ] if tkn in self . _token_to_idx else self . _token_to_idx [ self . _unknown_token ] for tkn in tokens ] else : return ( self . _token_to_idx [ tokens ] if tokens in self . _token_to_idx else self . _token_to_idx [ self . _unknown_token ] )