docs: def _run_job(self, job_n, job):
          """
          Runs a single job.
  
          Args:
              job_n: job number (1 index)
              job: Custodian job
  
  
          Raises:
              ValidationError: if a job fails validation
              ReturnCodeError: if the process has a return code different from 0
              NonRecoverableError: if an unrecoverable occurs
              MaxCorrectionsPerJobError: if max_errors_per_job is reached
              MaxCorrectionsError: if max_errors is reached
              MaxCorrectionsPerHandlerError: if max_errors_per_handler is reached
          """
          self.run_log.append({"job": job.as_dict(), "corrections": [],
                               "handler": None, "validator": None,
                               "max_errors": False, "max_errors_per_job": False,
                               "max_errors_per_handler": False,
                               "nonzero_return_code": False})
          self.errors_current_job = 0
          # reset the counters of the number of times a correction has been
          # applied for each handler
          for h in self.handlers:
              h.n_applied_corrections = 0
  
          job.setup()
  
          attempt = 0
          while (self.total_errors < self.max_errors and
                 self.errors_current_job < self.max_errors_per_job):
              attempt += 1
              logger.info(
                  "Starting job no. {} ({}) attempt no. {}. Total errors and "
                  "errors in job thus far = {}, {}.".format(
                      job_n, job.name, attempt, self.total_errors,
                      self.errors_current_job))
  
              p = job.run()
              # Check for errors using the error handlers and perform
              # corrections.
              has_error = False
              zero_return_code = True
  
              # While the job is running, we use the handlers that are
              # monitors to monitor the job.
              if isinstance(p, subprocess.Popen):
                  if self.monitors:
                      n = 0
                      while True:
                          n += 1
                          time.sleep(self.polling_time_step)
                          if p.poll() is not None:
                              break
                          terminate = self.terminate_func or p.terminate
                          if n % self.monitor_freq == 0:
                              has_error = self._do_check(self.monitors,
                                                         terminate)
                          if terminate is not None and terminate != p.terminate:
                              time.sleep(self.polling_time_step)
                  else:
                      p.wait()
                      if self.terminate_func is not None and \
                              self.terminate_func != p.terminate:
                          self.terminate_func()
                          time.sleep(self.polling_time_step)
  
                  zero_return_code = p.returncode == 0
  
              logger.info("{}.run has completed. "
                          "Checking remaining handlers".format(job.name))
              # Check for errors again, since in some cases non-monitor
              # handlers fix the problems detected by monitors
              # if an error has been found, not all handlers need to run
              if has_error:
                  self._do_check([h for h in self.handlers
                                  if not h.is_monitor])
              else:
                  has_error = self._do_check(self.handlers)
  
              if has_error:
                  # This makes sure the job is killed cleanly for certain systems.
                  job.terminate()
  
              # If there are no errors detected, perform
              # postprocessing and exit.
              if not has_error:
                  for v in self.validators:
                      if v.check():
                          self.run_log[-1]["validator"] = v
                          s = "Validation failed: {}".format(v)
                          raise ValidationError(s, True, v)
                  if not zero_return_code:
                      if self.terminate_on_nonzero_returncode:
                          self.run_log[-1]["nonzero_return_code"] = True
                          s = "Job return code is %d. Terminating..." % \
                              p.returncode
                          logger.info(s)
                          raise ReturnCodeError(s, True)
                      else:
                          warnings.warn("subprocess returned a non-zero return "
                                        "code. Check outputs carefully...")
                  job.postprocess()
                  return
  
              # Check that all errors could be handled
              for x in self.run_log[-1]["corrections"]:
                  if not x["actions"] and x["handler"].raises_runtime_error:
                      self.run_log[-1]["handler"] = x["handler"]
                      s = "Unrecoverable error for handler: {}".format(x["handler"])
                      raise NonRecoverableError(s, True, x["handler"])
              for x in self.run_log[-1]["corrections"]:
                  if not x["actions"]:
                      self.run_log[-1]["handler"] = x["handler"]
                      s = "Unrecoverable error for handler: %s" % x["handler"]
                      raise NonRecoverableError(s, False, x["handler"])
  
          if self.errors_current_job >= self.max_errors_per_job:
              self.run_log[-1]["max_errors_per_job"] = True
              msg = "Max errors per job reached: {}.".format(self.max_errors_per_job)
              logger.info(msg)
              raise MaxCorrectionsPerJobError(msg, True, self.max_errors_per_job, job)
          else:
              self.run_log[-1]["max_errors"] = True
              msg = "Max errors reached: {}.".format(self.max_errors)
              logger.info(msg)
              raise MaxCorrectionsError(msg, True, self.max_errors)