(self, text=None, context=None)
Split ``text`` into (tokentype, text) pairs.
        If ``context`` is given, use this lexer context instead.
Split ``text`` into (tokentype, text) pairs.
        If ``context`` is given, use this lexer context instead.
Split text into ( tokentype text ) pairs . If context is given use this lexer context instead .
def get_tokens_unprocessed(self, text=None, context=None):
        """
        Split ``text`` into (tokentype, text) pairs.
        If ``context`` is given, use this lexer context instead.
        """
        tokendefs = self._tokens
        if not context:
            ctx = LexerContext(text, 0)
            statetokens = tokendefs['root']
        else:
            ctx = context
            statetokens = tokendefs[ctx.stack[-1]]
            text = ctx.text
        while 1:
            for rexmatch, action, new_state in statetokens:
                m = rexmatch(text, ctx.pos, ctx.end)
                if m:
                    if type(action) is _TokenType:
                        yield ctx.pos, action, m.group()
                        ctx.pos = m.end()
                    else:
                        for item in action(self, m, ctx):
                            yield item
                        if not new_state:
                            # altered the state stack?
                            statetokens = tokendefs[ctx.stack[-1]]
                    # CAUTION: callback must set ctx.pos!
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            ctx.stack.extend(new_state)
                        elif isinstance(new_state, int):
                            # pop
                            del ctx.stack[new_state:]
                        elif new_state == '#push':
                            ctx.stack.append(ctx.stack[-1])
                        else:
                            assert False, "wrong state def: %r" % new_state
                        statetokens = tokendefs[ctx.stack[-1]]
                    break
            else:
                try:
                    if ctx.pos >= ctx.end:
                        break
                    if text[ctx.pos] == '\n':
                        # at EOL, reset state to "root"
                        ctx.pos += 1
                        ctx.stack = ['root']
                        statetokens = tokendefs['root']
                        yield ctx.pos, Text, u'\n'
                        continue
                    yield ctx.pos, Error, text[ctx.pos]
                    ctx.pos += 1
                except IndexError:
                    break
def get_tokens_unprocessed ( self , text = None , context = None ) : tokendefs = self . _tokens if not context : ctx = LexerContext ( text , 0 ) statetokens = tokendefs [ 'root' ] else : ctx = context statetokens = tokendefs [ ctx . stack [ - 1 ] ] text = ctx . text while 1 : for rexmatch , action , new_state in statetokens : m = rexmatch ( text , ctx . pos , ctx . end ) if m : if type ( action ) is _TokenType : yield ctx . pos , action , m . group ( ) ctx . pos = m . end ( ) else : for item in action ( self , m , ctx ) : yield item if not new_state : # altered the state stack? statetokens = tokendefs [ ctx . stack [ - 1 ] ] # CAUTION: callback must set ctx.pos! if new_state is not None : # state transition if isinstance ( new_state , tuple ) : ctx . stack . extend ( new_state ) elif isinstance ( new_state , int ) : # pop del ctx . stack [ new_state : ] elif new_state == '#push' : ctx . stack . append ( ctx . stack [ - 1 ] ) else : assert False , "wrong state def: %r" % new_state statetokens = tokendefs [ ctx . stack [ - 1 ] ] break else : try : if ctx . pos >= ctx . end : break if text [ ctx . pos ] == '\n' : # at EOL, reset state to "root" ctx . pos += 1 ctx . stack = [ 'root' ] statetokens = tokendefs [ 'root' ] yield ctx . pos , Text , u'\n' continue yield ctx . pos , Error , text [ ctx . pos ] ctx . pos += 1 except IndexError : break