docs: def tokeniter(self, source, name, filename=None, state=None):
          """This method tokenizes the text and returns the tokens in a
          generator.  Use this method if you just want to tokenize a template.
          """
          source = '\n'.join(unicode(source).splitlines())
          pos = 0
          lineno = 1
          stack = ['root']
          if state is not None and state != 'root':
              assert state in ('variable', 'block'), 'invalid state'
              stack.append(state + '_begin')
          else:
              state = 'root'
          statetokens = self.rules[stack[-1]]
          source_length = len(source)
  
          balancing_stack = []
  
          while 1:
              # tokenizer loop
              for regex, tokens, new_state in statetokens:
                  m = regex.match(source, pos)
                  # if no match we try again with the next rule
                  if m is None:
                      continue
  
                  # we only match blocks and variables if brances / parentheses
                  # are balanced. continue parsing with the lower rule which
                  # is the operator rule. do this only if the end tags look
                  # like operators
                  if balancing_stack and \
                     tokens in ('variable_end', 'block_end',
                                'linestatement_end'):
                      continue
  
                  # tuples support more options
                  if isinstance(tokens, tuple):
                      for idx, token in enumerate(tokens):
                          # failure group
                          if token.__class__ is Failure:
                              raise token(lineno, filename)
                          # bygroup is a bit more complex, in that case we
                          # yield for the current token the first named
                          # group that matched
                          elif token == '#bygroup':
                              for key, value in m.groupdict().iteritems():
                                  if value is not None:
                                      yield lineno, key, value
                                      lineno += value.count('\n')
                                      break
                              else:
                                  raise RuntimeError('%r wanted to resolve '
                                                     'the token dynamically'
                                                     ' but no group matched'
                                                     % regex)
                          # normal group
                          else:
                              data = m.group(idx + 1)
                              if data or token not in ignore_if_empty:
                                  yield lineno, token, data
                              lineno += data.count('\n')
  
                  # strings as token just are yielded as it.
                  else:
                      data = m.group()
                      # update brace/parentheses balance
                      if tokens == 'operator':
                          if data == '{':
                              balancing_stack.append('}')
                          elif data == '(':
                              balancing_stack.append(')')
                          elif data == '[':
                              balancing_stack.append(']')
                          elif data in ('}', ')', ']'):
                              if not balancing_stack:
                                  raise TemplateSyntaxError('unexpected \'%s\'' %
                                                            data, lineno, name,
                                                            filename)
                              expected_op = balancing_stack.pop()
                              if expected_op != data:
                                  raise TemplateSyntaxError('unexpected \'%s\', '
                                                            'expected \'%s\'' %
                                                            (data, expected_op),
                                                            lineno, name,
                                                            filename)
                      # yield items
                      if data or tokens not in ignore_if_empty:
                          yield lineno, tokens, data
                      lineno += data.count('\n')
  
                  # fetch new position into new variable so that we can check
                  # if there is a internal parsing error which would result
                  # in an infinite loop
                  pos2 = m.end()
  
                  # handle state changes
                  if new_state is not None:
                      # remove the uppermost state
                      if new_state == '#pop':
                          stack.pop()
                      # resolve the new state by group checking
                      elif new_state == '#bygroup':
                          for key, value in m.groupdict().iteritems():
                              if value is not None:
                                  stack.append(key)
                                  break
                          else:
                              raise RuntimeError('%r wanted to resolve the '
                                                 'new state dynamically but'
                                                 ' no group matched' %
                                                 regex)
                      # direct state name given
                      else:
                          stack.append(new_state)
                      statetokens = self.rules[stack[-1]]
                  # we are still at the same position and no stack change.
                  # this means a loop without break condition, avoid that and
                  # raise error
                  elif pos2 == pos:
                      raise RuntimeError('%r yielded empty string without '
                                         'stack change' % regex)
                  # publish new function and start again
                  pos = pos2
                  break
              # if loop terminated without break we havn't found a single match
              # either we are at the end of the file or we have a problem
              else:
                  # end of text
                  if pos >= source_length:
                      return
                  # something went wrong
                  raise TemplateSyntaxError('unexpected char %r at %d' %
                                            (source[pos], pos), lineno,
                                            name, filename)