(self, vocab_file, max_size)
Creates a vocab of up to max_size words, reading from the vocab_file. If max_size is 0, reads the entire vocab file.

    Args:
      vocab_file: path to the vocab file, which is assumed to contain "<word> <frequency>" on each line, sorted with most frequent word first. This code doesn't actually use the frequencies, though.
      max_size: integer. The maximum size of the resulting Vocabulary.
Creates a vocab of up to max_size words, reading from the vocab_file. If max_size is 0, reads the entire vocab file.
Creates a vocab of up to max_size words reading from the vocab_file . If max_size is 0 reads the entire vocab file .
def __init__(self, vocab_file, max_size):
    """Creates a vocab of up to max_size words, reading from the vocab_file. If max_size is 0, reads the entire vocab file.

    Args:
      vocab_file: path to the vocab file, which is assumed to contain "<word> <frequency>" on each line, sorted with most frequent word first. This code doesn't actually use the frequencies, though.
      max_size: integer. The maximum size of the resulting Vocabulary."""
    self._word_to_id = {}
    self._id_to_word = {}
    self._count = 0 # keeps track of total number of words in the Vocab

    # [UNK], [PAD], [START] and [STOP] get the ids 0,1,2,3.
    for w in [UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
      self._word_to_id[w] = self._count
      self._id_to_word[self._count] = w
      self._count += 1

    # Read the vocab file and add words up to max_size
    with open(vocab_file, 'r') as vocab_f:
      for line in vocab_f:
        pieces = line.split()
        if len(pieces) != 2:
          print 'Warning: incorrectly formatted line in vocabulary file: %s\n' % line
          continue
        w = pieces[0]
        if w in [SENTENCE_START, SENTENCE_END, UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
          raise Exception('<s>, </s>, [UNK], [PAD], [START] and [STOP] shouldn\'t be in the vocab file, but %s is' % w)
        if w in self._word_to_id:
          raise Exception('Duplicated word in vocabulary file: %s' % w)
        self._word_to_id[w] = self._count
        self._id_to_word[self._count] = w
        self._count += 1
        if max_size != 0 and self._count >= max_size:
          print "max_size of vocab was specified as %i; we now have %i words. Stopping reading." % (max_size, self._count)
          break

    print "Finished constructing vocabulary of %i total words. Last word added: %s" % (self._count, self._id_to_word[self._count-1])
def __init__ ( self , vocab_file , max_size ) : self . _word_to_id = { } self . _id_to_word = { } self . _count = 0 # keeps track of total number of words in the Vocab # [UNK], [PAD], [START] and [STOP] get the ids 0,1,2,3. for w in [ UNKNOWN_TOKEN , PAD_TOKEN , START_DECODING , STOP_DECODING ] : self . _word_to_id [ w ] = self . _count self . _id_to_word [ self . _count ] = w self . _count += 1 # Read the vocab file and add words up to max_size with open ( vocab_file , 'r' ) as vocab_f : for line in vocab_f : pieces = line . split ( ) if len ( pieces ) != 2 : print 'Warning: incorrectly formatted line in vocabulary file: %s\n' % line continue w = pieces [ 0 ] if w in [ SENTENCE_START , SENTENCE_END , UNKNOWN_TOKEN , PAD_TOKEN , START_DECODING , STOP_DECODING ] : raise Exception ( '<s>, </s>, [UNK], [PAD], [START] and [STOP] shouldn\'t be in the vocab file, but %s is' % w ) if w in self . _word_to_id : raise Exception ( 'Duplicated word in vocabulary file: %s' % w ) self . _word_to_id [ w ] = self . _count self . _id_to_word [ self . _count ] = w self . _count += 1 if max_size != 0 and self . _count >= max_size : print "max_size of vocab was specified as %i; we now have %i words. Stopping reading." % ( max_size , self . _count ) break print "Finished constructing vocabulary of %i total words. Last word added: %s" % ( self . _count , self . _id_to_word [ self . _count - 1 ] )