(self)
Tokenize the file, run physical line checks and yield tokens.
Tokenize the file, run physical line checks and yield tokens.
Tokenize the file run physical line checks and yield tokens .
def generate_tokens(self):
        """Tokenize the file, run physical line checks and yield tokens."""
        if self._io_error:
            self.report_error(1, 0, 'E902 %s' % self._io_error, readlines)
        tokengen = tokenize.generate_tokens(self.readline)
        try:
            for token in tokengen:
                if token[2][0] > self.total_lines:
                    return
                self.noqa = token[4] and noqa(token[4])
                self.maybe_check_physical(token)
                yield token
        except (SyntaxError, tokenize.TokenError):
            self.report_invalid_syntax()
def generate_tokens ( self ) : if self . _io_error : self . report_error ( 1 , 0 , 'E902 %s' % self . _io_error , readlines ) tokengen = tokenize . generate_tokens ( self . readline ) try : for token in tokengen : if token [ 2 ] [ 0 ] > self . total_lines : return self . noqa = token [ 4 ] and noqa ( token [ 4 ] ) self . maybe_check_physical ( token ) yield token except ( SyntaxError , tokenize . TokenError ) : self . report_invalid_syntax ( )