(self, tokens, debug=False)
return p.rootnode
Parse a series of tokens and return the syntax tree.
Parse a series of tokens and return the syntax tree.
Parse a series of tokens and return the syntax tree .
def parse_tokens(self, tokens, debug=False):
        """Parse a series of tokens and return the syntax tree."""
        # XXX Move the prefix computation into a wrapper around tokenize.
        p = parse.Parser(self.grammar, self.convert)
        p.setup()
        lineno = 1
        column = 0
        type = value = start = end = line_text = None
        prefix = ""
        for quintuple in tokens:
            type, value, start, end, line_text = quintuple
            if start != (lineno, column):
                assert (lineno, column) <= start, ((lineno, column), start)
                s_lineno, s_column = start
                if lineno < s_lineno:
                    prefix += "\n" * (s_lineno - lineno)
                    lineno = s_lineno
                    column = 0
                if column < s_column:
                    prefix += line_text[column:s_column]
                    column = s_column
            if type in (tokenize.COMMENT, tokenize.NL):
                prefix += value
                lineno, column = end
                if value.endswith("\n"):
                    lineno += 1
                    column = 0
                continue
            if type == token.OP:
                type = grammar.opmap[value]
            if debug:
                self.logger.debug("%s %r (prefix=%r)",
                                  token.tok_name[type], value, prefix)
            if p.addtoken(type, value, (prefix, start)):
                if debug:
                    self.logger.debug("Stop.")
                break
            prefix = ""
            lineno, column = end
            if value.endswith("\n"):
                lineno += 1
                column = 0
        else:
            # We never broke out -- EOF is too soon (how can this happen???)
            raise parse.ParseError("incomplete input",
                                   type, value, (prefix, start))
        return p.rootnode
def parse_tokens ( self , tokens , debug = False ) : # XXX Move the prefix computation into a wrapper around tokenize. p = parse . Parser ( self . grammar , self . convert ) p . setup ( ) lineno = 1 column = 0 type = value = start = end = line_text = None prefix = "" for quintuple in tokens : type , value , start , end , line_text = quintuple if start != ( lineno , column ) : assert ( lineno , column ) <= start , ( ( lineno , column ) , start ) s_lineno , s_column = start if lineno < s_lineno : prefix += "\n" * ( s_lineno - lineno ) lineno = s_lineno column = 0 if column < s_column : prefix += line_text [ column : s_column ] column = s_column if type in ( tokenize . COMMENT , tokenize . NL ) : prefix += value lineno , column = end if value . endswith ( "\n" ) : lineno += 1 column = 0 continue if type == token . OP : type = grammar . opmap [ value ] if debug : self . logger . debug ( "%s %r (prefix=%r)" , token . tok_name [ type ] , value , prefix ) if p . addtoken ( type , value , ( prefix , start ) ) : if debug : self . logger . debug ( "Stop." ) break prefix = "" lineno , column = end if value . endswith ( "\n" ) : lineno += 1 column = 0 else : # We never broke out -- EOF is too soon (how can this happen???) raise parse . ParseError ( "incomplete input" , type , value , ( prefix , start ) ) return p . rootnode