docs: def diffs2persistence(rev_docs, window_size=50, revert_radius=15, sunset=None,
                        verbose=False):
      """
      Processes a sorted and page-partitioned sequence of revision documents into
      and adds a 'persistence' field to them containing statistics about how each
      token "added" in the revision persisted through future revisions.
  
      :Parameters:
          rev_docs : `iterable` ( `dict` )
              JSON documents of revision data containing a 'diff' field as
              generated by ``dump2diffs``.  It's assumed that rev_docs are
              partitioned by page and otherwise in chronological order.
          window_size : `int`
              The size of the window of revisions from which persistence data
              will be generated.
          revert_radius : `int`
              The number of revisions back that a revert can reference.
          sunset : :class:`mwtypes.Timestamp`
              The date of the database dump we are generating from.  This is
              used to apply a 'time visible' statistic.  If not set, now() will
              be assumed.
          keep_diff : `bool`
              Do not drop the `diff` field from the revision document after
              processing is complete.
          verbose : `bool`
              Prints out dots and stuff to stderr
  
      :Returns:
          A generator of rev_docs with a 'persistence' field containing
          statistics about individual tokens.
      """
      rev_docs = mwxml.utilities.normalize(rev_docs)
      window_size = int(window_size)
      revert_radius = int(revert_radius)
      sunset = Timestamp(sunset) if sunset is not None \
                                 else Timestamp(time.time())
  
      # Group the docs by page
      page_docs = groupby(rev_docs, key=lambda d: d['page']['title'])
  
      for page_title, rev_docs in page_docs:
  
          if verbose:
              sys.stderr.write(page_title + ": ")
  
          # We need a look-ahead to know how long this revision was visible
          rev_docs = peekable(rev_docs)
  
          # The window allows us to manage memory
          window = deque(maxlen=window_size)
  
          # The state does the actual processing work
          state = DiffState(revert_radius=revert_radius)
  
          while rev_docs:
              rev_doc = next(rev_docs)
              next_doc = rev_docs.peek(None)
  
              if next_doc is not None:
                  seconds_visible = Timestamp(next_doc['timestamp']) - \
                                    Timestamp(rev_doc['timestamp'])
              else:
                  seconds_visible = sunset - Timestamp(rev_doc['timestamp'])
  
              if seconds_visible < 0:
                  logger.warn("Seconds visible {0} is less than zero."
                              .format(seconds_visible))
                  seconds_visible = 0
  
              _, tokens_added, _ = \
                  state.update_opdocs(rev_doc['sha1'], rev_doc['diff']['ops'],
                                      (rev_doc['user'], seconds_visible))
  
              if len(window) == window_size:
                  # Time to start writing some stats
                  old_doc, old_added = window[0]
                  window.append((rev_doc, tokens_added))
                  persistence = token_persistence(old_doc, old_added, window,
                                                  None)
                  old_doc['persistence'] = persistence
                  yield old_doc
                  if verbose:
                      sys.stderr.write(".")
                      sys.stderr.flush()
              else:
                  window.append((rev_doc, tokens_added))
  
          while len(window) > 0:
              old_doc, old_added = window.popleft()
              persistence = token_persistence(old_doc, old_added, window, sunset)
              old_doc['persistence'] = persistence
              yield old_doc
              if verbose:
                  sys.stderr.write("_")
                  sys.stderr.flush()
  
          if verbose:
              sys.stderr.write("\n")