(self, tokens)
Initialize alphabet from an iterable of token or subtoken strings.
Initialize alphabet from an iterable of token or subtoken strings.
Initialize alphabet from an iterable of token or subtoken strings .
def _init_alphabet_from_tokens(self, tokens):
    """Initialize alphabet from an iterable of token or subtoken strings."""
    # Include all characters from all tokens in the alphabet to guarantee that
    # any token can be encoded. Additionally, include all escaping characters.
    self._alphabet = {c for token in tokens for c in token}  # pylint: disable=g-complex-comprehension
    self._alphabet |= _ESCAPE_CHARS
def _init_alphabet_from_tokens ( self , tokens ) : # Include all characters from all tokens in the alphabet to guarantee that # any token can be encoded. Additionally, include all escaping characters. self . _alphabet = { c for token in tokens for c in token } # pylint: disable=g-complex-comprehension self . _alphabet |= _ESCAPE_CHARS