docs: def _Tokenize(template_str, meta_left, meta_right, whitespace):
      """Yields tokens, which are 2-tuples (TOKEN_TYPE, token_string)."""
      
      trimlen = len(meta_left)
      token_re = MakeTokenRegex(meta_left, meta_right)
      do_strip = (whitespace == 'strip-line')  # Do this outside loop
      do_strip_part = False
      
      for line in template_str.splitlines(True):  # retain newlines
          if do_strip or do_strip_part:
              line = line.strip()
          
          tokens = token_re.split(line)
          
          # Check for a special case first.  If a comment or "block" directive is on a
          # line by itself (with only space surrounding it), then the space is
          # omitted.  For simplicity, we don't handle the case where we have 2
          # directives, say '{.end} # {#comment}' on a line.
          if len(tokens) == 3:
              # ''.isspace() == False, so work around that
              if (tokens[0].isspace() or not tokens[0]) and \
                      (tokens[2].isspace() or not tokens[2]):
                  token = tokens[1][trimlen: -trimlen]
                  
                  # Check the ones that begin with ## before #
                  if token == COMMENT_BEGIN:
                      yield COMMENT_BEGIN_TOKEN, None
                      continue
                  if token == COMMENT_END:
                      yield COMMENT_END_TOKEN, None
                      continue
                  if token == OPTION_STRIP_LINE:
                      do_strip_part = True
                      continue
                  if token == OPTION_END:
                      do_strip_part = False
                      continue
                  
                  if token.startswith('#'):
                      continue  # The whole line is omitted
                  
                  token_type, token = _MatchDirective(token)
                  if token_type is not None:
                      yield token_type, token  # Only yield the token, not space
                      continue
          
          # The line isn't special; process it normally.
          for i, token in enumerate(tokens):
              if i % 2 == 0:
                  yield LITERAL_TOKEN, token
              
              else:  # It's a "directive" in metachracters
                  assert token.startswith(meta_left), repr(token)
                  assert token.endswith(meta_right), repr(token)
                  token = token[trimlen: -trimlen]
                  
                  # Check the ones that begin with ## before #
                  if token == COMMENT_BEGIN:
                      yield COMMENT_BEGIN_TOKEN, None
                      continue
                  if token == COMMENT_END:
                      yield COMMENT_END_TOKEN, None
                      continue
                  if token == OPTION_STRIP_LINE:
                      do_strip_part = True
                      continue
                  if token == OPTION_END:
                      do_strip_part = False
                      continue
                  
                  # A single-line comment
                  if token.startswith('#'):
                      continue
                  
                  if token.startswith('.'):
                      literal = {
                          '.meta-left': meta_left,
                          '.meta-right': meta_right,
                          '.space': ' ',
                          '.tab': '\t',
                          '.newline': '\n',
                      }.get(token)
                      
                      if literal is not None:
                          yield META_LITERAL_TOKEN, literal
                          continue
                      
                      token_type, token = _MatchDirective(token)
                      if token_type is not None:
                          yield token_type, token
                  
                  else:  # Now we know the directive is a substitution.
                      yield SUBST_TOKEN, token