(self, data_path, vocab, hps, single_pass)
Initialize the batcher. Start threads that process the data into batches.

    Args:
      data_path: tf.Example filepattern.
      vocab: Vocabulary object
      hps: hyperparameters
      single_pass: If True, run through the dataset exactly once (useful for when you want to run evaluation on the dev or test set). Otherwise generate random batches indefinitely (useful for training).
Initialize the batcher. Start threads that process the data into batches.
Initialize the batcher . Start threads that process the data into batches .
def __init__(self, data_path, vocab, hps, single_pass):
    """Initialize the batcher. Start threads that process the data into batches.

    Args:
      data_path: tf.Example filepattern.
      vocab: Vocabulary object
      hps: hyperparameters
      single_pass: If True, run through the dataset exactly once (useful for when you want to run evaluation on the dev or test set). Otherwise generate random batches indefinitely (useful for training).
    """
    self._data_path = data_path
    self._vocab = vocab
    self._hps = hps
    self._single_pass = single_pass

    # Initialize a queue of Batches waiting to be used, and a queue of Examples waiting to be batched
    self._batch_queue = Queue.Queue(self.BATCH_QUEUE_MAX)
    self._example_queue = Queue.Queue(self.BATCH_QUEUE_MAX * self._hps.batch_size)

    # Different settings depending on whether we're in single_pass mode or not
    if single_pass:
      self._num_example_q_threads = 1 # just one thread, so we read through the dataset just once
      self._num_batch_q_threads = 1  # just one thread to batch examples
      self._bucketing_cache_size = 1 # only load one batch's worth of examples before bucketing; this essentially means no bucketing
      self._finished_reading = False # this will tell us when we're finished reading the dataset
    else:
      self._num_example_q_threads = 16 # num threads to fill example queue
      self._num_batch_q_threads = 4  # num threads to fill batch queue
      self._bucketing_cache_size = 100 # how many batches-worth of examples to load into cache before bucketing

    # Start the threads that load the queues
    self._example_q_threads = []
    for _ in xrange(self._num_example_q_threads):
      self._example_q_threads.append(Thread(target=self.fill_example_queue))
      self._example_q_threads[-1].daemon = True
      self._example_q_threads[-1].start()
    self._batch_q_threads = []
    for _ in xrange(self._num_batch_q_threads):
      self._batch_q_threads.append(Thread(target=self.fill_batch_queue))
      self._batch_q_threads[-1].daemon = True
      self._batch_q_threads[-1].start()

    # Start a thread that watches the other threads and restarts them if they're dead
    if not single_pass: # We don't want a watcher in single_pass mode because the threads shouldn't run forever
      self._watch_thread = Thread(target=self.watch_threads)
      self._watch_thread.daemon = True
      self._watch_thread.start()
def __init__ ( self , data_path , vocab , hps , single_pass ) : self . _data_path = data_path self . _vocab = vocab self . _hps = hps self . _single_pass = single_pass # Initialize a queue of Batches waiting to be used, and a queue of Examples waiting to be batched self . _batch_queue = Queue . Queue ( self . BATCH_QUEUE_MAX ) self . _example_queue = Queue . Queue ( self . BATCH_QUEUE_MAX * self . _hps . batch_size ) # Different settings depending on whether we're in single_pass mode or not if single_pass : self . _num_example_q_threads = 1 # just one thread, so we read through the dataset just once self . _num_batch_q_threads = 1 # just one thread to batch examples self . _bucketing_cache_size = 1 # only load one batch's worth of examples before bucketing; this essentially means no bucketing self . _finished_reading = False # this will tell us when we're finished reading the dataset else : self . _num_example_q_threads = 16 # num threads to fill example queue self . _num_batch_q_threads = 4 # num threads to fill batch queue self . _bucketing_cache_size = 100 # how many batches-worth of examples to load into cache before bucketing # Start the threads that load the queues self . _example_q_threads = [ ] for _ in xrange ( self . _num_example_q_threads ) : self . _example_q_threads . append ( Thread ( target = self . fill_example_queue ) ) self . _example_q_threads [ - 1 ] . daemon = True self . _example_q_threads [ - 1 ] . start ( ) self . _batch_q_threads = [ ] for _ in xrange ( self . _num_batch_q_threads ) : self . _batch_q_threads . append ( Thread ( target = self . fill_batch_queue ) ) self . _batch_q_threads [ - 1 ] . daemon = True self . _batch_q_threads [ - 1 ] . start ( ) # Start a thread that watches the other threads and restarts them if they're dead if not single_pass : # We don't want a watcher in single_pass mode because the threads shouldn't run forever self . _watch_thread = Thread ( target = self . watch_threads ) self . _watch_thread . daemon = True self . _watch_thread . start ( )