(self,
               vocab_filename,
               reverse=False,
               vocab_list=None,
               replace_oov=None,
               num_reserved_ids=NUM_RESERVED_TOKENS)
Initialize from a file or list, one token per line.

    Handling of reserved tokens works as follows:
    - When initializing from a list, we add reserved tokens to the vocab.
    - When initializing from a file, we do not add reserved tokens to the vocab.
    - When saving vocab files, we save reserved tokens to the file.

    Args:
      vocab_filename: If not None, the full filename to read vocab from. If this
        is not None, then vocab_list should be None.
      reverse: Boolean indicating if tokens should be reversed during encoding
        and decoding.
      vocab_list: If not None, a list of elements of the vocabulary. If this is
        not None, then vocab_filename should be None.
      replace_oov: If not None, every out-of-vocabulary token seen when encoding
        will be replaced by this string (which must be in vocab).
      num_reserved_ids: Number of IDs to save for reserved tokens like <EOS>.
Initialize from a file or list, one token per line.
Initialize from a file or list one token per line .
def __init__(self,
               vocab_filename,
               reverse=False,
               vocab_list=None,
               replace_oov=None,
               num_reserved_ids=NUM_RESERVED_TOKENS):
    """Initialize from a file or list, one token per line.

    Handling of reserved tokens works as follows:
    - When initializing from a list, we add reserved tokens to the vocab.
    - When initializing from a file, we do not add reserved tokens to the vocab.
    - When saving vocab files, we save reserved tokens to the file.

    Args:
      vocab_filename: If not None, the full filename to read vocab from. If this
        is not None, then vocab_list should be None.
      reverse: Boolean indicating if tokens should be reversed during encoding
        and decoding.
      vocab_list: If not None, a list of elements of the vocabulary. If this is
        not None, then vocab_filename should be None.
      replace_oov: If not None, every out-of-vocabulary token seen when encoding
        will be replaced by this string (which must be in vocab).
      num_reserved_ids: Number of IDs to save for reserved tokens like <EOS>.
    """
    super(TokenTextEncoder, self).__init__(num_reserved_ids=num_reserved_ids)
    self._reverse = reverse
    self._replace_oov = replace_oov
    if vocab_filename:
      self._init_vocab_from_file(vocab_filename)
    else:
      assert vocab_list is not None
      self._init_vocab_from_list(vocab_list)
def __init__ ( self , vocab_filename , reverse = False , vocab_list = None , replace_oov = None , num_reserved_ids = NUM_RESERVED_TOKENS ) : super ( TokenTextEncoder , self ) . __init__ ( num_reserved_ids = num_reserved_ids ) self . _reverse = reverse self . _replace_oov = replace_oov if vocab_filename : self . _init_vocab_from_file ( vocab_filename ) else : assert vocab_list is not None self . _init_vocab_from_list ( vocab_list )