parameters: (self, tokens, debug=False)
return_statement: return p.rootnode
docstring: Parse a series of tokens and return the syntax tree.
docstring_summary: Parse a series of tokens and return the syntax tree.
docstring_tokens: Parse a series of tokens and return the syntax tree .
function: def parse_tokens(self, tokens, debug=False):
          """Parse a series of tokens and return the syntax tree."""
          # XXX Move the prefix computation into a wrapper around tokenize.
          p = parse.Parser(self.grammar, self.convert)
          p.setup()
          lineno = 1
          column = 0
          type = value = start = end = line_text = None
          prefix = u""
          for quintuple in tokens:
              type, value, start, end, line_text = quintuple
              if start != (lineno, column):
                  assert (lineno, column) <= start, ((lineno, column), start)
                  s_lineno, s_column = start
                  if lineno < s_lineno:
                      prefix += "\n" * (s_lineno - lineno)
                      lineno = s_lineno
                      column = 0
                  if column < s_column:
                      prefix += line_text[column:s_column]
                      column = s_column
              if type in (tokenize.COMMENT, tokenize.NL):
                  prefix += value
                  lineno, column = end
                  if value.endswith("\n"):
                      lineno += 1
                      column = 0
                  continue
              if type == token.OP:
                  type = grammar.opmap[value]
              if debug:
                  self.logger.debug("%s %r (prefix=%r)",
                                    token.tok_name[type], value, prefix)
              if p.addtoken(type, value, (prefix, start)):
                  if debug:
                      self.logger.debug("Stop.")
                  break
              prefix = ""
              lineno, column = end
              if value.endswith("\n"):
                  lineno += 1
                  column = 0
          else:
              # We never broke out -- EOF is too soon (how can this happen???)
              raise parse.ParseError("incomplete input",
                                     type, value, (prefix, start))
          return p.rootnode
function_tokens: def parse_tokens ( self , tokens , debug = False ) : # XXX Move the prefix computation into a wrapper around tokenize. p = parse . Parser ( self . grammar , self . convert ) p . setup ( ) lineno = 1 column = 0 type = value = start = end = line_text = None prefix = u"" for quintuple in tokens : type , value , start , end , line_text = quintuple if start != ( lineno , column ) : assert ( lineno , column ) <= start , ( ( lineno , column ) , start ) s_lineno , s_column = start if lineno < s_lineno : prefix += "\n" * ( s_lineno - lineno ) lineno = s_lineno column = 0 if column < s_column : prefix += line_text [ column : s_column ] column = s_column if type in ( tokenize . COMMENT , tokenize . NL ) : prefix += value lineno , column = end if value . endswith ( "\n" ) : lineno += 1 column = 0 continue if type == token . OP : type = grammar . opmap [ value ] if debug : self . logger . debug ( "%s %r (prefix=%r)" , token . tok_name [ type ] , value , prefix ) if p . addtoken ( type , value , ( prefix , start ) ) : if debug : self . logger . debug ( "Stop." ) break prefix = "" lineno , column = end if value . endswith ( "\n" ) : lineno += 1 column = 0 else : # We never broke out -- EOF is too soon (how can this happen???) raise parse . ParseError ( "incomplete input" , type , value , ( prefix , start ) ) return p . rootnode