(model, batcher, sess_context_manager, sv, summary_writer)
Repeatedly runs training iterations, logging loss to screen and writing summaries
Repeatedly runs training iterations, logging loss to screen and writing summaries
Repeatedly runs training iterations logging loss to screen and writing summaries
def run_training(model, batcher, sess_context_manager, sv, summary_writer):
  """Repeatedly runs training iterations, logging loss to screen and writing summaries"""
  tf.logging.info("starting run_training")
  with sess_context_manager as sess:
    if FLAGS.debug: # start the tensorflow debugger
      sess = tf_debug.LocalCLIDebugWrapperSession(sess)
      sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
    while True: # repeats until interrupted
      batch = batcher.next_batch()

      tf.logging.info('running training step...')
      t0=time.time()
      results = model.run_train_step(sess, batch)
      t1=time.time()
      tf.logging.info('seconds for training step: %.3f', t1-t0)

      loss = results['loss']
      tf.logging.info('loss: %f', loss) # print the loss to screen

      if not np.isfinite(loss):
        raise Exception("Loss is not finite. Stopping.")

      if FLAGS.coverage:
        coverage_loss = results['coverage_loss']
        tf.logging.info("coverage_loss: %f", coverage_loss) # print the coverage loss to screen

      # get the summaries and iteration number so we can write summaries to tensorboard
      summaries = results['summaries'] # we will write these summaries to tensorboard using summary_writer
      train_step = results['global_step'] # we need this to update our running average loss

      summary_writer.add_summary(summaries, train_step) # write the summaries
      if train_step % 100 == 0: # flush the summary writer every so often
        summary_writer.flush()
def run_training ( model , batcher , sess_context_manager , sv , summary_writer ) : tf . logging . info ( "starting run_training" ) with sess_context_manager as sess : if FLAGS . debug : # start the tensorflow debugger sess = tf_debug . LocalCLIDebugWrapperSession ( sess ) sess . add_tensor_filter ( "has_inf_or_nan" , tf_debug . has_inf_or_nan ) while True : # repeats until interrupted batch = batcher . next_batch ( ) tf . logging . info ( 'running training step...' ) t0 = time . time ( ) results = model . run_train_step ( sess , batch ) t1 = time . time ( ) tf . logging . info ( 'seconds for training step: %.3f' , t1 - t0 ) loss = results [ 'loss' ] tf . logging . info ( 'loss: %f' , loss ) # print the loss to screen if not np . isfinite ( loss ) : raise Exception ( "Loss is not finite. Stopping." ) if FLAGS . coverage : coverage_loss = results [ 'coverage_loss' ] tf . logging . info ( "coverage_loss: %f" , coverage_loss ) # print the coverage loss to screen # get the summaries and iteration number so we can write summaries to tensorboard summaries = results [ 'summaries' ] # we will write these summaries to tensorboard using summary_writer train_step = results [ 'global_step' ] # we need this to update our running average loss summary_writer . add_summary ( summaries , train_step ) # write the summaries if train_step % 100 == 0 : # flush the summary writer every so often summary_writer . flush ( )