docs: def _update_file_frontiers(
              self,
              frontier_list,
              revision,
              max_csets_proc=30,
              going_forward=False
          ):
          '''
          Update the frontier for all given files, up to the given revision.
  
          Built for quick continuous _forward_ updating of large sets
          of files of TUIDs. Backward updating should be done through
          get_tuids(files, revision). If we cannot find a frontier, we will
          stop looking after max_csets_proc and update all files at the given
          revision.
  
          :param frontier_list: list of files to update
          :param revision: revision to update files to
          :param max_csets_proc: maximum number of changeset logs to look through
                                 to find past frontiers.
          :param going_forward: If we know the requested revision is in front
                                of the latest revision use this flag. Used when
                                the frontier is too far away. If this is not set and
                                a frontier is too far, the latest revision will not
                                be updated.
          :return: list of (file, list(tuids)) tuples
          '''
  
          # Get the changelogs and revisions until we find the
          # last one we've seen, and get the modified files in
          # each one.
  
          # Holds the files modified up to the last frontiers.
          files_to_process = {}
  
          # Holds all frontiers to find
          remaining_frontiers = {cset for cset in list(set([frontier for _, frontier in frontier_list]))}
  
          if len(remaining_frontiers) <= 1 and frontier_list[0][1] == revision:
              # If the latest revision is the requested revision,
              # and there is only one frontier requested
              # continue to the tuid querys.
              remaining_frontiers = {}
  
          # Revision we are searching from
          final_rev = revision
  
          # If this exceeds max_csets_proc,
          # all frontiers not found are considered lost
          csets_proced = 0
  
          # Holds info on how to apply the diffs onto each frontier,
          # and all known frontiers.
          diffs_to_frontier = {cset: [] for cset in remaining_frontiers}
  
          Log.note("Searching for frontier(s): {{frontier}} ", frontier=str(list(remaining_frontiers)))
          Log.note(
              "Running on revision with HG URL: {{url}}",
              url=self.hg_url / self.config.hg.branch / 'rev' / revision
          )
          while remaining_frontiers:
              # Get a changelog
              clog_url = self.hg_url / self.config.hg.branch / 'json-log' / final_rev
              try:
                  Log.note("Searching through changelog {{url}}", url=clog_url)
                  clog_obj = http.get_json(clog_url, retry=RETRY)
                  if isinstance(clog_obj, (text_type, str)):
                      Log.error(
                          "Revision {{cset}} does not exist in the {{branch}} branch",
                          cset=final_rev, branch=self.config.hg.branch
                      )
              except Exception as e:
                  Log.error(
                      "Unexpected error getting changset-log for {{url}}: {{error}}",
                      url=clog_url,
                      error=e
                  )
  
              # For each changeset in the log (except the last one
              # which is duplicated on the next log page requested.
              clog_obj_list = list(clog_obj['changesets'])
              for clog_cset in clog_obj_list[:-1]:
                  nodes_cset = clog_cset['node'][:12]
  
                  if remaining_frontiers:
                      if nodes_cset in remaining_frontiers:
                          # Found a frontier, remove it from search list.
                          remaining_frontiers.remove(nodes_cset)
  
                          if not remaining_frontiers:
                              # Found all frontiers, get out of the loop before
                              # we add the diff to a frontier update list.
                              break
  
                      # Add this diff to the processing list
                      # for each remaining frontier
                      for cset in diffs_to_frontier:
                          if cset in remaining_frontiers:
                              diffs_to_frontier[cset].append(nodes_cset)
  
              csets_proced += 1
              if not remaining_frontiers:
                  # End searching
                  break
              elif csets_proced >= max_csets_proc:
                  # In this case, all files need to be updated to this revision to ensure
                  # line ordering consistency (between past, and future) when a revision
                  # that is in the past is asked for.
                  files_to_process = {file: [revision] for file, _ in frontier_list}
                  break
              else:
                  # Go to the next log page
                  last_entry = clog_obj_list[-1]
                  final_rev = last_entry['node'][:12]
  
          if not remaining_frontiers:
              Log.note("Found all frontiers: {{frontiers_list}}", frontiers_list=str(list(diffs_to_frontier.keys())))
          else:
              found_frontiers = [
                  frontier for frontier in diffs_to_frontier if frontier not in remaining_frontiers
              ]
              Log.note("Found frontiers: {{found}}", found=str(found_frontiers))
              Log.note("Did not find frontiers: {{not_found}}", not_found=str(list(remaining_frontiers)))
  
          added_files = {}
          removed_files = {}
          parsed_diffs = {}
  
          # This list is used to determine what files
          file_to_frontier = {file: frontier for file, frontier in frontier_list}
          if len(remaining_frontiers) != len(diffs_to_frontier.keys()):
              # If there is at least one frontier that was found
              # Only get diffs that are needed (if any frontiers were not found)
              diffs_cache = []
              for cset in diffs_to_frontier:
                  if cset not in remaining_frontiers:
                      diffs_cache.extend(diffs_to_frontier[cset])
  
              Log.note("Gathering diffs for: {{csets}}", csets=str(diffs_cache))
              all_diffs = self.get_diffs(diffs_cache)
  
              # Build a dict for faster access to the diffs,
              # to be used later when applying them.
              parsed_diffs = {diff_entry['cset']: diff_entry['diff'] for diff_entry in all_diffs}
  
              # In case the file name changes, this will map
              # the requested file to the new file name so
              # diffs can all be gathered.
              filenames_to_seek = {}
  
              # Parse diffs for files to process and store diffs to
              # apply for each file in files_to_process.
              added_and_removed_counts = {file: 1 for file in file_to_frontier}
              for csets_diff in all_diffs:
                  cset_len12 = csets_diff['cset']
                  parsed_diff = csets_diff['diff']['diffs']
  
                  for f_added in parsed_diff:
                      # Get new entries for removed files.
                      new_name = f_added['new'].name.lstrip('/')
                      old_name = f_added['old'].name.lstrip('/')
  
                      # If we don't need this file, skip it
                      if new_name not in file_to_frontier and \
                         new_name not in filenames_to_seek:
                          if old_name not in file_to_frontier and \
                             old_name not in filenames_to_seek:
                              # File not requested
                              continue
                          if new_name == 'dev/null':
                              frontier_filename = old_name
                              while frontier_filename in filenames_to_seek:
                                  frontier_filename = filenames_to_seek[frontier_filename]
  
                              if frontier_filename not in removed_files:
                                  removed_files[frontier_filename] = 0
                              removed_files[frontier_filename] = added_and_removed_counts[frontier_filename]
                              added_and_removed_counts[frontier_filename] += 1
                              continue
  
                      if old_name == 'dev/null':
                          frontier_filename = new_name
                          while frontier_filename in filenames_to_seek:
                              frontier_filename = filenames_to_seek[frontier_filename]
  
                          if frontier_filename not in added_files:
                              added_files[frontier_filename] = 0
                          added_files[frontier_filename] = added_and_removed_counts[frontier_filename]
                          added_and_removed_counts[frontier_filename] += 1
                          continue
                      if new_name != old_name:
                          # File name was changed, keep the diff anyway
                          # to add any changes it makes.
                          filenames_to_seek[new_name] = old_name
  
                      # Get the originally requested file name
                      # by following filenames_to_seek entries
                      frontier_filename = new_name
                      while frontier_filename in filenames_to_seek:
                          frontier_filename = filenames_to_seek[frontier_filename]
  
                      # If we are past the frontier for this file,
                      # or if we are at the frontier skip it.
                      if file_to_frontier[frontier_filename] == '':
                          # Previously found frontier, skip
                          continue
  
                      # At this point, file is in the database, is
                      # asked to be processed, and we are still
                      # searching for the last frontier.
                      if file_to_frontier[frontier_filename] == cset_len12:
                          file_to_frontier[frontier_filename] = ''
                          # Found the frontier, skip
                          continue
  
                      if old_name != new_name:
                          Log.note(
                              "{{cset}} changes a requested file's name: {{file}} from {{oldfile}}. ",
                              file=new_name,
                              oldfile=old_name,
                              cset=cset
                          )
  
                      # Store the diff as it needs to be applied
                      if frontier_filename in files_to_process:
                          files_to_process[frontier_filename].append(cset_len12)
                      else:
                          files_to_process[frontier_filename] = [cset_len12]
  
          # Process each file that needs it based on the
          # files_to_process list.
          result = []
          ann_inserts = []
          latestFileMod_inserts = {}
          anns_to_get = []
          total = len(frontier_list)
          tmp_results = {}
  
          with self.conn.transaction() as transaction:
              for count, (file, old_frontier) in enumerate(frontier_list):
                  if old_frontier in remaining_frontiers:
                      # If we were still looking for the frontier by the end, get a new
                      # annotation for this file.
                      anns_to_get.append(file)
  
                      if going_forward:
                          # If we are always going forward, update the frontier
                          latestFileMod_inserts[file] = (file, revision)
  
                      Log.note(
                          "Frontier update - can't find frontier {{lost_frontier}}: "
                          "{{count}}/{{total}} - {{percent|percent(decimal=0)}} | {{rev}}|{{file}} ",
                          count=count,
                          total=total,
                          file=file,
                          rev=revision,
                          percent=count / total,
                          lost_frontier=old_frontier
                      )
                      continue
                  elif file in removed_files or file in added_files:
                      if file not in removed_files:
                          removed_files[file] = 0
                      if file not in added_files:
                          added_files[file] = 0
  
                      if removed_files[file] <= added_files[file]:
                          # For it to still exist it has to be
                          # added last (to give it a larger count)
                          anns_to_get.append(file)
                          Log.note(
                              "Frontier update - adding: "
                              "{{count}}/{{total}} - {{percent|percent(decimal=0)}} | {{rev}}|{{file}} ",
                              count=count,
                              total=total,
                              file=file,
                              rev=revision,
                              percent=count / total,
                              lost_frontier=old_frontier
                          )
                      else:
                          Log.note(
                              "Frontier update - deleting: "
                              "{{count}}/{{total}} - {{percent|percent(decimal=0)}} | {{rev}}|{{file}} ",
                              count=count,
                              total=total,
                              file=file,
                              rev=revision,
                              percent=count / total,
                              lost_frontier=old_frontier
                          )
                          tmp_results[file] = []
                      if going_forward:
                          # If we are always going forward, update the frontier
                          latestFileMod_inserts[file] = (file, revision)
  
                      continue
  
                  # If the file was modified, get it's newest
                  # annotation and update the file.
                  tmp_res = None
                  if file in files_to_process:
                      # Process this file using the diffs found
                      tmp_ann = self._get_annotation(old_frontier, file, transaction)
                      if tmp_ann is None or tmp_ann == '' or self.destringify_tuids(tmp_ann) is None:
                          Log.warning(
                              "{{file}} has frontier but can't find old annotation for it in {{rev}}, "
                              "restarting it's frontier.",
                              rev=old_frontier,
                              file=file
                          )
                          anns_to_get.append(file)
                      else:
                          # File was modified, apply it's diffs
                          # Reverse the diff list, we always find the newest diff first
                          csets_to_proc = files_to_process[file][::-1]
                          tmp_res = self.destringify_tuids(tmp_ann)
                          new_fname = file
                          for i in csets_to_proc:
                              tmp_res, new_fname = self._apply_diff(transaction, tmp_res, parsed_diffs[i], i, new_fname)
  
                          ann_inserts.append((revision, file, self.stringify_tuids(tmp_res)))
                          Log.note(
                              "Frontier update - modified: {{count}}/{{total}} - {{percent|percent(decimal=0)}} "
                              "| {{rev}}|{{file}} ",
                              count=count,
                              total=total,
                              file=file,
                              rev=revision,
                              percent=count / total
                          )
                  else:
                      old_ann = self._get_annotation(old_frontier, file, transaction)
                      if old_ann is None or (old_ann == '' and file in added_files):
                          # File is new (likely from an error), or re-added - we need to create
                          # a new initial entry for this file.
                          anns_to_get.append(file)
                          Log.note(
                              "Frontier update - readded: {{count}}/{{total}} - {{percent|percent(decimal=0)}} "
                              "| {{rev}}|{{file}} ",
                              count=count,
                              total=total,
                              file=file,
                              rev=revision,
                              percent=count / total
                          )
                      else:
                          # File was not modified since last
                          # known revision
                          tmp_res = self.destringify_tuids(old_ann) if old_ann != '' else []
                          ann_inserts.append((revision, file, old_ann))
                          Log.note(
                              "Frontier update - not modified: {{count}}/{{total}} - {{percent|percent(decimal=0)}} "
                              "| {{rev}}|{{file}} ",
                              count=count,
                              total=total,
                              file=file,
                              rev=revision,
                              percent=count / total
                          )
  
                  if tmp_res:
                      tmp_results[file] = tmp_res
                  else:
                      Log.note(
                          "Error occured for file {{file}} in revision {{revision}}",
                          file=file,
                          revision=revision
                      )
                      tmp_results[file] = []
  
                  # If we have found all frontiers, update to the
                  # latest revision. Otherwise, the requested
                  # revision is too far away (can't be sure
                  # if it's past). Unless we are told that we are
                  # going forward.
                  if going_forward or not remaining_frontiers:
                      latest_rev = revision
                  else:
                      latest_rev = old_frontier
                  latestFileMod_inserts[file] = (file, latest_rev)
  
              Log.note("Updating DB tables `latestFileMod` and `annotations`...")
  
              # No need to double-check if latesteFileMods has been updated before,
              # we perform an insert or replace any way.
              if len(latestFileMod_inserts) > 0:
                  for _, inserts_list in jx.groupby(latestFileMod_inserts.values(), size=SQL_BATCH_SIZE):
                      transaction.execute(
                          "INSERT OR REPLACE INTO latestFileMod (file, revision) VALUES " +
                          sql_list(quote_list(i) for i in inserts_list)
                      )
  
              anns_added_by_other_thread = {}
              if len(ann_inserts) > 0:
                  ann_inserts = list(set(ann_inserts))
                  for _, tmp_inserts in jx.groupby(ann_inserts, size=SQL_ANN_BATCH_SIZE):
                      # Check if any were added in the mean time by another thread
                      recomputed_inserts = []
                      for rev, filename, string_tuids in tmp_inserts:
                          tmp_ann = self._get_annotation(rev, filename, transaction)
                          if not tmp_ann or tmp_ann == '':
                              recomputed_inserts.append((rev, filename, string_tuids))
                          else:
                              anns_added_by_other_thread[filename] = self.destringify_tuids(tmp_ann)
  
                      if len(recomputed_inserts) <= 0:
                          continue
  
                      try:
                          for rev, filename, tuids_ann in recomputed_inserts:
                              tmp_ann = self.destringify_tuids(tuids_ann)
                              for tuid_map in tmp_ann:
                                  if tuid_map is None or tuid_map.tuid is None or tuid_map.line is None:
                                      Log.warning(
                                          "None value encountered in annotation insertion in {{rev}} for {{file}}: {{tuids}}" ,
                                          rev=rev, file=filename, tuids=str(tuid_map)
                                      )
                          self.insert_annotations(transaction, recomputed_inserts)
                      except Exception as e:
                          Log.error("Error inserting into annotations table: {{inserting}}", inserting=recomputed_inserts, cause=e)
  
          if len(anns_to_get) > 0:
              result.extend(self.get_tuids(anns_to_get, revision, commit=False))
  
          for f in tmp_results:
              tuids = tmp_results[f]
              if f in anns_added_by_other_thread:
                  tuids = anns_added_by_other_thread[f]
              result.append((f, tuids))
          return result