(self, vocab_list)
Initialize tokens from a list of tokens.

    It is ok if reserved tokens appear in the vocab list. They will be
    removed. The set of tokens in vocab_list should be unique.

    Args:
      vocab_list: A list of tokens.
Initialize tokens from a list of tokens.
Initialize tokens from a list of tokens .
def _init_vocab_from_list(self, vocab_list):
    """Initialize tokens from a list of tokens.

    It is ok if reserved tokens appear in the vocab list. They will be
    removed. The set of tokens in vocab_list should be unique.

    Args:
      vocab_list: A list of tokens.
    """

    def token_gen():
      for token in vocab_list:
        if token not in RESERVED_TOKENS:
          yield token

    self._init_vocab(token_gen())
def _init_vocab_from_list ( self , vocab_list ) : def token_gen ( ) : for token in vocab_list : if token not in RESERVED_TOKENS : yield token self . _init_vocab ( token_gen ( ) )