parameters: (self, text)
docstring: Lexically analyze `text`.
  
          Yields pairs (`name`, `tokentext`).
docstring_summary: Lexically analyze `text`.
docstring_tokens: Lexically analyze text .
function: def lex(self, text):
          """Lexically analyze `text`.
  
          Yields pairs (`name`, `tokentext`).
  
          """
          end = len(text)
          state = self.state
          regexes = self.regexes
          toks = self.toks
          start = 0
  
          while start < end:
              for match in regexes[state].finditer(text, start):
                  name = match.lastgroup
                  tok = toks[name]
                  toktext = match.group(name)
                  start += len(toktext)
                  yield (tok.name, toktext)
  
                  if tok.next:
                      state = tok.next
                      break
  
          self.state = state
function_tokens: def lex ( self , text ) : end = len ( text ) state = self . state regexes = self . regexes toks = self . toks start = 0 while start < end : for match in regexes [ state ] . finditer ( text , start ) : name = match . lastgroup tok = toks [ name ] toktext = match . group ( name ) start += len ( toktext ) yield ( tok . name , toktext ) if tok . next : state = tok . next break self . state = state