docs: def _generate_extended_ngrams(self, matches, work, siglum, label, corpus,
                                    highest_n):
          """Returns the n-grams of the largest size that exist in `siglum`
          witness to `work` under `label`, generated from adding
          together overlapping n-grams in `matches`.
  
          :param matches: n-gram matches
          :type matches: `pandas.DataFrame`
          :param work: name of work whose results are being processed
          :type work: `str`
          :param siglum: siglum of witness whose results are being processed
          :type siglum: `str`
          :param label: label of witness whose results are being processed
          :type label: `str`
          :param corpus: corpus to which `filename` belongs
          :type corpus: `Corpus`
          :param highest_n: highest degree of n-gram in `matches`
          :type highest_n: `int`
          :rtype: `list` of `str`
  
          """
          # For large result sets, this method may involve a lot of
          # processing within the for loop, so optimise even small
          # things, such as aliasing dotted calls here and below.
          t_join = self._tokenizer.joiner.join
          witness_matches = matches[
              (matches[constants.WORK_FIELDNAME] == work) &
              (matches[constants.SIGLUM_FIELDNAME] == siglum) &
              (matches[constants.LABEL_FIELDNAME] == label)]
          text = corpus.get_witness(work, siglum).get_token_content()
          ngrams = [tuple(self._tokenizer.tokenize(ngram)) for ngram in
                    list(witness_matches[constants.NGRAM_FIELDNAME])]
          # Go through the list of n-grams, and create a list of
          # extended n-grams by joining two n-grams together that
          # overlap (a[-overlap:] == b[:-1]) and checking that the result
          # occurs in text.
          working_ngrams = ngrams[:]
          extended_ngrams = set(ngrams)
          new_working_ngrams = []
          overlap = highest_n - 1
          # Create an index of n-grams by their overlapping portion,
          # pointing to the non-overlapping token.
          ngram_index = {}
          for ngram in ngrams:
              values = ngram_index.setdefault(ngram[:-1], [])
              values.append(ngram[-1:])
          extended_add = extended_ngrams.add
          new_working_append = new_working_ngrams.append
          ngram_size = highest_n
          while working_ngrams:
              removals = set()
              ngram_size += 1
              self._logger.debug(
                  'Iterating over {} n-grams to produce {}-grams'.format(
                      len(working_ngrams), ngram_size))
              for base in working_ngrams:
                  remove_base = False
                  base_overlap = base[-overlap:]
                  for next_token in ngram_index.get(base_overlap, []):
                      extension = base + next_token
                      if t_join(extension) in text:
                          extended_add(extension)
                          new_working_append(extension)
                          remove_base = True
                  if remove_base:
                      # Remove base from extended_ngrams, because it is
                      # now encompassed by extension.
                      removals.add(base)
              extended_ngrams -= removals
              working_ngrams = new_working_ngrams[:]
              new_working_ngrams = []
              new_working_append = new_working_ngrams.append
          extended_ngrams = sorted(extended_ngrams, key=len, reverse=True)
          extended_ngrams = [t_join(ngram) for ngram in extended_ngrams]
          self._logger.debug('Generated {} extended n-grams'.format(
              len(extended_ngrams)))
          self._logger.debug('Longest generated n-gram: {}'.format(
              extended_ngrams[0]))
          # In order to get the counts correct in the next step of the
          # process, these n-grams must be overlaid over the text and
          # repeated as many times as there are matches. N-grams that do
          # not match (and they may not match on previously matched
          # parts of the text) are discarded.
          ngrams = []
          for ngram in extended_ngrams:
              # Remove from the text those parts that match. Replace
              # them with a double space, which should prevent any
              # incorrect match on the text from each side of the match
              # that is now contiguous.
              text, count = re.subn(re.escape(ngram), '  ', text)
              ngrams.extend([ngram] * count)
          self._logger.debug('Aligned extended n-grams with the text; '
                             '{} distinct n-grams exist'.format(len(ngrams)))
          return ngrams