(self)
Reads data from file and processes into Examples which are then placed into the example queue.
Reads data from file and processes into Examples which are then placed into the example queue.
Reads data from file and processes into Examples which are then placed into the example queue .
def fill_example_queue(self):
    """Reads data from file and processes into Examples which are then placed into the example queue."""

    input_gen = self.text_generator(data.example_generator(self._data_path, self._single_pass))

    while True:
      try:
        (article, abstract) = input_gen.next() # read the next example from file. article and abstract are both strings.
      except StopIteration: # if there are no more examples:
        tf.logging.info("The example generator for this example queue filling thread has exhausted data.")
        if self._single_pass:
          tf.logging.info("single_pass mode is on, so we've finished reading dataset. This thread is stopping.")
          self._finished_reading = True
          break
        else:
          raise Exception("single_pass mode is off but the example generator is out of data; error.")

      abstract_sentences = [sent.strip() for sent in data.abstract2sents(abstract)] # Use the <s> and </s> tags in abstract to get a list of sentences.
      example = Example(article, abstract_sentences, self._vocab, self._hps) # Process into an Example.
      self._example_queue.put(example)
def fill_example_queue ( self ) : input_gen = self . text_generator ( data . example_generator ( self . _data_path , self . _single_pass ) ) while True : try : ( article , abstract ) = input_gen . next ( ) # read the next example from file. article and abstract are both strings. except StopIteration : # if there are no more examples: tf . logging . info ( "The example generator for this example queue filling thread has exhausted data." ) if self . _single_pass : tf . logging . info ( "single_pass mode is on, so we've finished reading dataset. This thread is stopping." ) self . _finished_reading = True break else : raise Exception ( "single_pass mode is off but the example generator is out of data; error." ) abstract_sentences = [ sent . strip ( ) for sent in data . abstract2sents ( abstract ) ] # Use the <s> and </s> tags in abstract to get a list of sentences. example = Example ( article , abstract_sentences , self . _vocab , self . _hps ) # Process into an Example. self . _example_queue . put ( example )