docs: def consume_token(self, tokens, index, tokens_len):
          """Consume a token.
  
          Returns tuple of (tokens, tokens_len, index) when consumption is
          completed and tokens have been merged together.
          """
          del tokens_len
  
          consumption_ended = False
  
          q_type = self.quote_type
  
          begin_literal_type = getattr(TokenType,
                                       "Begin{0}QuotedLiteral".format(q_type))
          end_literal_type = getattr(TokenType,
                                     "End{0}QuotedLiteral".format(q_type))
  
          if (index != self.begin and
                  tokens[index].type == begin_literal_type):
              # This is an edge case where a quote begins a line and matched
              # as a quoted region beginning and a quoted region ending.
              # Split the token before and after the quote, mark the
              # quote character itself as an ending and insert both
              # tokens back in, handling the ending afterwards.
              assert _RE_QUOTE_TYPE.match(tokens[index].content[0])
  
              # Mini-tokenize everything after the first token
              line_tokens = _scan_for_tokens(tokens[index].content[1:])
              end_type = getattr(TokenType,
                                 "End{0}QuotedLiteral".format(q_type))
              replacement = [Token(type=end_type,
                                   content=tokens[index].content[0],
                                   line=tokens[index].line,
                                   col=tokens[index].col)]
  
              for after in line_tokens:
                  replacement.append(Token(type=after.type,
                                           content=after.content,
                                           line=(tokens[index].line +
                                                 after.line - 1),
                                           col=(tokens[index].col +
                                                after.col - 1)))
  
              tokens = _replace_token_range(tokens,
                                            index,
                                            index + 1,
                                            replacement)
              consumption_ended = True
  
          if tokens[index].type == end_literal_type:
              consumption_ended = True
  
          if consumption_ended:
              end = index + 1
              pasted = ""
              for i in range(self.begin, end):
                  pasted += tokens[i].content
  
              tokens = _replace_token_range(tokens, self.begin, end,
                                            [Token(type=TokenType.QuotedLiteral,
                                                   content=pasted,
                                                   line=tokens[self.begin].line,
                                                   col=tokens[self.begin].col)])
  
              return (self.begin, len(tokens), tokens)