parameters: (self)
docstring: Tokenize the file, run physical line checks and yield tokens.
docstring_summary: Tokenize the file, run physical line checks and yield tokens.
docstring_tokens: Tokenize the file run physical line checks and yield tokens .
function: def generate_tokens(self):
          """Tokenize the file, run physical line checks and yield tokens."""
          if self._io_error:
              self.report_error(1, 0, 'E902 %s' % self._io_error, readlines)
          tokengen = tokenize.generate_tokens(self.readline)
          try:
              for token in tokengen:
                  if token[2][0] > self.total_lines:
                      return
                  self.noqa = token[4] and noqa(token[4])
                  self.maybe_check_physical(token)
                  yield token
          except (SyntaxError, tokenize.TokenError):
              self.report_invalid_syntax()
function_tokens: def generate_tokens ( self ) : if self . _io_error : self . report_error ( 1 , 0 , 'E902 %s' % self . _io_error , readlines ) tokengen = tokenize . generate_tokens ( self . readline ) try : for token in tokengen : if token [ 2 ] [ 0 ] > self . total_lines : return self . noqa = token [ 4 ] and noqa ( token [ 4 ] ) self . maybe_check_physical ( token ) yield token except ( SyntaxError , tokenize . TokenError ) : self . report_invalid_syntax ( )