(
    tokens: List[Any])
Computes spans of repeating subsequences.

  Find (none inclusive, but possibly overlapping) consective repeating
  subsequences.
  For example, in "a b c a b c a b c a b d", consecutive repeating spans are (0,
  9), (1, 10), (2, 11).
  (0, 6) is also a consecutive repeating span, but it is included in (0. 9).
  Args:
    tokens: Tokens from the text.

  Yields:
    Spans of repeating subsequences.
Computes spans of repeating subsequences.
Computes spans of repeating subsequences .
def _find_consecutive_repeating_subsequence_spans(
    tokens: List[Any]) -> Iterable[Tuple[int, int]]:
  """Computes spans of repeating subsequences.

  Find (none inclusive, but possibly overlapping) consective repeating
  subsequences.
  For example, in "a b c a b c a b c a b d", consecutive repeating spans are (0,
  9), (1, 10), (2, 11).
  (0, 6) is also a consecutive repeating span, but it is included in (0. 9).
  Args:
    tokens: Tokens from the text.

  Yields:
    Spans of repeating subsequences.
  """
  length = len(tokens)
  # searched_repeats saves repoeats that is included in lower ngram repeats.
  searched_repeats = set()
  # For each ngram that is possibly repeated.
  for n in range(1, length // 2 + 1):
    i = 0
    while i <= length - 2 * n:
      # Skip span if already searched in lower ngram repeats.
      if tuple(tokens[i:i + n]) in searched_repeats:
        i += n
        continue
      # Greedily matches to longest repeating sequence with span size n.
      count, match_break = _longest_repeat_with_size(tokens, i, n)
      if count > 1:
        for j in range((match_break - i) % n + 1):
          yield (i + j, i + j + n * count)
          # Add repeating ngrams to searched_repeats
          for c in range(1, count):
            searched_repeats.add(tuple(tokens[i + j:i + j + c * n]))
      i = max(i, match_break - n) + 1
def _find_consecutive_repeating_subsequence_spans ( tokens : List [ Any ] ) -> Iterable [ Tuple [ int , int ] ] : length = len ( tokens ) # searched_repeats saves repoeats that is included in lower ngram repeats. searched_repeats = set ( ) # For each ngram that is possibly repeated. for n in range ( 1 , length // 2 + 1 ) : i = 0 while i <= length - 2 * n : # Skip span if already searched in lower ngram repeats. if tuple ( tokens [ i : i + n ] ) in searched_repeats : i += n continue # Greedily matches to longest repeating sequence with span size n. count , match_break = _longest_repeat_with_size ( tokens , i , n ) if count > 1 : for j in range ( ( match_break - i ) % n + 1 ) : yield ( i + j , i + j + n * count ) # Add repeating ngrams to searched_repeats for c in range ( 1 , count ) : searched_repeats . add ( tuple ( tokens [ i + j : i + j + c * n ] ) ) i = max ( i , match_break - n ) + 1