(self, text, stack=('root',))
Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
Split ``text`` into (tokentype, text) pairs.
Split text into ( tokentype text ) pairs .
def get_tokens_unprocessed(self, text, stack=('root',)):
        """
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        pos = 0
        tokendefs = self._tokens
        statestack = list(stack)
        statetokens = tokendefs[statestack[-1]]
        known_names = {}
        while 1:
            for rexmatch, action, new_state in statetokens:
                m = rexmatch(text, pos)
                if m:
                    # print rex.pattern
                    value = m.group()
                    if value in known_names:
                        yield pos, known_names[value], value
                    elif type(action) is _TokenType:
                        yield pos, action, value
                    elif hasattr(action, '__call__'):
                        ttype, value = action(value)
                        known_names[value] = ttype
                        yield pos, ttype, value
                    else:
                        for item in action(self, m):
                            yield item
                    pos = m.end()
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            for state in new_state:
                                if state == '#pop':
                                    statestack.pop()
                                elif state == '#push':
                                    statestack.append(statestack[-1])
                                else:
                                    statestack.append(state)
                        elif isinstance(new_state, int):
                            # pop
                            del statestack[new_state:]
                        elif new_state == '#push':
                            statestack.append(statestack[-1])
                        else:
                            assert False, "wrong state def: %r" % new_state
                        statetokens = tokendefs[statestack[-1]]
                    break
            else:
                try:
                    if text[pos] == '\n':
                        # at EOL, reset state to "root"
                        pos += 1
                        statestack = ['root']
                        statetokens = tokendefs['root']
                        yield pos, Text, u'\n'
                        continue
                    yield pos, Error, text[pos]
                    pos += 1
                except IndexError:
                    break
def get_tokens_unprocessed ( self , text , stack = ( 'root' , ) ) : pos = 0 tokendefs = self . _tokens statestack = list ( stack ) statetokens = tokendefs [ statestack [ - 1 ] ] known_names = { } while 1 : for rexmatch , action , new_state in statetokens : m = rexmatch ( text , pos ) if m : # print rex.pattern value = m . group ( ) if value in known_names : yield pos , known_names [ value ] , value elif type ( action ) is _TokenType : yield pos , action , value elif hasattr ( action , '__call__' ) : ttype , value = action ( value ) known_names [ value ] = ttype yield pos , ttype , value else : for item in action ( self , m ) : yield item pos = m . end ( ) if new_state is not None : # state transition if isinstance ( new_state , tuple ) : for state in new_state : if state == '#pop' : statestack . pop ( ) elif state == '#push' : statestack . append ( statestack [ - 1 ] ) else : statestack . append ( state ) elif isinstance ( new_state , int ) : # pop del statestack [ new_state : ] elif new_state == '#push' : statestack . append ( statestack [ - 1 ] ) else : assert False , "wrong state def: %r" % new_state statetokens = tokendefs [ statestack [ - 1 ] ] break else : try : if text [ pos ] == '\n' : # at EOL, reset state to "root" pos += 1 statestack = [ 'root' ] statetokens = tokendefs [ 'root' ] yield pos , Text , u'\n' continue yield pos , Error , text [ pos ] pos += 1 except IndexError : break