(self,
               vocab_file,
               start_word="<S>",
               end_word="</S>",
               unk_word="<UNK>")
Initializes the vocabulary.

    Args:
      vocab_file: File containing the vocabulary, where the words are the first
        whitespace-separated token on each line (other tokens are ignored) and
        the word ids are the corresponding line numbers.
      start_word: Special word denoting sentence start.
      end_word: Special word denoting sentence end.
      unk_word: Special word denoting unknown words.
Initializes the vocabulary.
Initializes the vocabulary .
def __init__(self,
               vocab_file,
               start_word="<S>",
               end_word="</S>",
               unk_word="<UNK>"):
    """Initializes the vocabulary.

    Args:
      vocab_file: File containing the vocabulary, where the words are the first
        whitespace-separated token on each line (other tokens are ignored) and
        the word ids are the corresponding line numbers.
      start_word: Special word denoting sentence start.
      end_word: Special word denoting sentence end.
      unk_word: Special word denoting unknown words.
    """
    if not tf.gfile.Exists(vocab_file):
      tf.logging.fatal("Vocab file %s not found.", vocab_file)
    tf.logging.info("Initializing vocabulary from file: %s", vocab_file)

    with tf.gfile.GFile(vocab_file, mode="r") as f:
      reverse_vocab = list(f.readlines())
    reverse_vocab = [line.split()[0] for line in reverse_vocab]
    assert start_word in reverse_vocab
    assert end_word in reverse_vocab
    if unk_word not in reverse_vocab:
      reverse_vocab.append(unk_word)
    vocab = dict([(x, y) for (y, x) in enumerate(reverse_vocab)])

    tf.logging.info("Created vocabulary with %d words" % len(vocab))

    self.vocab = vocab  # vocab[word] = id
    self.reverse_vocab = reverse_vocab  # reverse_vocab[id] = word

    # Save special word ids.
    self.start_id = vocab[start_word]
    self.end_id = vocab[end_word]
    self.unk_id = vocab[unk_word]
def __init__ ( self , vocab_file , start_word = "<S>" , end_word = "</S>" , unk_word = "<UNK>" ) : if not tf . gfile . Exists ( vocab_file ) : tf . logging . fatal ( "Vocab file %s not found." , vocab_file ) tf . logging . info ( "Initializing vocabulary from file: %s" , vocab_file ) with tf . gfile . GFile ( vocab_file , mode = "r" ) as f : reverse_vocab = list ( f . readlines ( ) ) reverse_vocab = [ line . split ( ) [ 0 ] for line in reverse_vocab ] assert start_word in reverse_vocab assert end_word in reverse_vocab if unk_word not in reverse_vocab : reverse_vocab . append ( unk_word ) vocab = dict ( [ ( x , y ) for ( y , x ) in enumerate ( reverse_vocab ) ] ) tf . logging . info ( "Created vocabulary with %d words" % len ( vocab ) ) self . vocab = vocab # vocab[word] = id self . reverse_vocab = reverse_vocab # reverse_vocab[id] = word # Save special word ids. self . start_id = vocab [ start_word ] self . end_id = vocab [ end_word ] self . unk_id = vocab [ unk_word ]