docs: def _scan_for_tokens(contents):
      """Scan a string for tokens and return immediate form tokens."""
      # Regexes are in priority order. Changing the order may alter the
      # behavior of the lexer
      scanner = re.Scanner([
          # Things inside quotes
          (r"(?<![^\s\(])([\"\'])(?:(?=(\\?))\2.)*?\1(?![^\s\)])",
           lambda s, t: (TokenType.QuotedLiteral, t)),
          # Numbers on their own
          (r"(?<![^\s\(])-?[0-9]+(?![^\s\)\(])", lambda s, t: (TokenType.Number,
                                                               t)),
          # Left Paren
          (r"\(", lambda s, t: (TokenType.LeftParen, t)),
          # Right Paren
          (r"\)", lambda s, t: (TokenType.RightParen, t)),
          # Either a valid function name or variable name.
          (r"(?<![^\s\(])[a-zA-z_][a-zA-Z0-9_]*(?![^\s\)\(])",
           lambda s, t: (TokenType.Word, t)),
          # Variable dereference.
          (r"(?<![^\s\(])\${[a-zA-z_][a-zA-Z0-9_]*}(?![^\s\)])",
           lambda s, t: (TokenType.Deref, t)),
          # Newline
          (r"\n", lambda s, t: (TokenType.Newline, t)),
          # Whitespace
          (r"\s+", lambda s, t: (TokenType.Whitespace, t)),
          # The beginning of a double-quoted string, terminating at end of line
          (r"(?<![^\s\(\\])[\"]([^\"]|\\[\"])*$",
           lambda s, t: (TokenType.BeginDoubleQuotedLiteral, t)),
          # The end of a double-quoted string
          (r"[^\s]*(?<!\\)[\"](?![^\s\)])",
           lambda s, t: (TokenType.EndDoubleQuotedLiteral, t)),
          # The beginning of a single-quoted string, terminating at end of line
          (r"(?<![^\s\(\\])[\']([^\']|\\[\'])*$",
           lambda s, t: (TokenType.BeginSingleQuotedLiteral, t)),
          # The end of a single-quoted string
          (r"[^\s]*(?<!\\)[\'](?![^\s\)])",
           lambda s, t: (TokenType.EndSingleQuotedLiteral, t)),
          # Begin-RST Comment Block
          (r"#.rst:$", lambda s, t: (TokenType.BeginRSTComment, t)),
          # Begin Inline RST
          (r"#\[=*\[.rst:$", lambda s, t: (TokenType.BeginInlineRST, t)),
          # End Inline RST
          (r"#\]=*\]$", lambda s, t: (TokenType.EndInlineRST, t)),
          # Comment
          (r"#", lambda s, t: (TokenType.Comment, t)),
          # Catch-all for literals which are compound statements.
          (r"([^\s\(\)]+|[^\s\(]*[^\)]|[^\(][^\s\)]*)",
           lambda s, t: (TokenType.UnquotedLiteral, t))
      ])
  
      tokens_return = []
  
      lines = contents.splitlines(True)
      lineno = 0
      for line in lines:
          lineno += 1
          col = 1
  
          tokens, remaining = scanner.scan(line)
          if remaining != "":
              msg = "Unknown tokens found on line {0}: {1}".format(lineno,
                                                                   remaining)
              raise RuntimeError(msg)
  
          for token_type, token_contents in tokens:
              tokens_return.append(Token(type=token_type,
                                         content=token_contents,
                                         line=lineno,
                                         col=col))
  
              col += len(token_contents)
  
      return tokens_return