code: def lex(self, text):
          
          pos = 0
          while text:
              token_class, match = self.tokens.get_token(text)
              if token_class is not None:
                  matched_text = text[match.start():match.end()]
                  yield token_class(matched_text)
                  text = text[match.end():]
                  pos += match.end()
              elif text[0] in self.blank_chars:
                  text = text[1:]
                  pos += 1
              else:
                  raise LexerError(
                          'Invalid character %s in %s' % (text[0], text),
                          position=pos)
  
          yield self.end_token()
docs: Split self.text into a list of tokens.
  
          Args:
              text (str): text to parse
  
          Yields:
              Token: the tokens generated from the given text.