def tokenize(text, lexer):
    """
    Return array of lexed tokens (for debugging).
    """
    lexer.input(text)
    tokens = []
    while True:
        tok = lexer.token()
        if not tok:
            break
        tokens.append(tok)
    return tokens
