(self, text)
Lexically analyze `text`.

        Yields pairs (`name`, `tokentext`).
Lexically analyze `text`.
Lexically analyze text .
def lex(self, text):
        """Lexically analyze `text`.

        Yields pairs (`name`, `tokentext`).

        """
        end = len(text)
        state = self.state
        regexes = self.regexes
        toks = self.toks
        start = 0

        while start < end:
            for match in regexes[state].finditer(text, start):
                name = match.lastgroup
                tok = toks[name]
                toktext = match.group(name)
                start += len(toktext)
                yield (tok.name, toktext)

                if tok.next:
                    state = tok.next
                    break

        self.state = state
def lex ( self , text ) : end = len ( text ) state = self . state regexes = self . regexes toks = self . toks start = 0 while start < end : for match in regexes [ state ] . finditer ( text , start ) : name = match . lastgroup tok = toks [ name ] toktext = match . group ( name ) start += len ( toktext ) yield ( tok . name , toktext ) if tok . next : state = tok . next break self . state = state