parameters: (self, text, stack=('root',))
docstring: Split ``text`` into (tokentype, text) pairs.
  
          ``stack`` is the inital stack (default: ``['root']``)
docstring_summary: Split ``text`` into (tokentype, text) pairs.
docstring_tokens: Split text into ( tokentype text ) pairs .
function: def get_tokens_unprocessed(self, text, stack=('root',)):
          """
          Split ``text`` into (tokentype, text) pairs.
  
          ``stack`` is the inital stack (default: ``['root']``)
          """
          pos = 0
          tokendefs = self._tokens
          statestack = list(stack)
          statetokens = tokendefs[statestack[-1]]
          known_names = {}
          while 1:
              for rexmatch, action, new_state in statetokens:
                  m = rexmatch(text, pos)
                  if m:
                      # print rex.pattern
                      value = m.group()
                      if value in known_names:
                          yield pos, known_names[value], value
                      elif type(action) is _TokenType:
                          yield pos, action, value
                      elif hasattr(action, '__call__'):
                          ttype, value = action(value)
                          known_names[value] = ttype
                          yield pos, ttype, value
                      else:
                          for item in action(self, m):
                              yield item
                      pos = m.end()
                      if new_state is not None:
                          # state transition
                          if isinstance(new_state, tuple):
                              for state in new_state:
                                  if state == '#pop':
                                      statestack.pop()
                                  elif state == '#push':
                                      statestack.append(statestack[-1])
                                  else:
                                      statestack.append(state)
                          elif isinstance(new_state, int):
                              # pop
                              del statestack[new_state:]
                          elif new_state == '#push':
                              statestack.append(statestack[-1])
                          else:
                              assert False, "wrong state def: %r" % new_state
                          statetokens = tokendefs[statestack[-1]]
                      break
              else:
                  try:
                      if text[pos] == '\n':
                          # at EOL, reset state to "root"
                          pos += 1
                          statestack = ['root']
                          statetokens = tokendefs['root']
                          yield pos, Text, u'\n'
                          continue
                      yield pos, Error, text[pos]
                      pos += 1
                  except IndexError:
                      break
function_tokens: def get_tokens_unprocessed ( self , text , stack = ( 'root' , ) ) : pos = 0 tokendefs = self . _tokens statestack = list ( stack ) statetokens = tokendefs [ statestack [ - 1 ] ] known_names = { } while 1 : for rexmatch , action , new_state in statetokens : m = rexmatch ( text , pos ) if m : # print rex.pattern value = m . group ( ) if value in known_names : yield pos , known_names [ value ] , value elif type ( action ) is _TokenType : yield pos , action , value elif hasattr ( action , '__call__' ) : ttype , value = action ( value ) known_names [ value ] = ttype yield pos , ttype , value else : for item in action ( self , m ) : yield item pos = m . end ( ) if new_state is not None : # state transition if isinstance ( new_state , tuple ) : for state in new_state : if state == '#pop' : statestack . pop ( ) elif state == '#push' : statestack . append ( statestack [ - 1 ] ) else : statestack . append ( state ) elif isinstance ( new_state , int ) : # pop del statestack [ new_state : ] elif new_state == '#push' : statestack . append ( statestack [ - 1 ] ) else : assert False , "wrong state def: %r" % new_state statetokens = tokendefs [ statestack [ - 1 ] ] break else : try : if text [ pos ] == '\n' : # at EOL, reset state to "root" pos += 1 statestack = [ 'root' ] statetokens = tokendefs [ 'root' ] yield pos , Text , u'\n' continue yield pos , Error , text [ pos ] pos += 1 except IndexError : break