def line_tokenizer(inputstring):
    """Yield a list of tokens for each line in a file.

    If a quoted string is incomplete at the end of a physical line, the
    logical line continues to the next line.  Likewise, join two lines if
    the first line ends in a backslash.

    Setup a test file:
    >>> teststr = r'''First line.
    ... Beginning of second \\
    ... line and end.
    ... Beginning of "third
    ... line" and end.'''
    >>>

    Make sure that the lexical analyzer works:
    >>> it = line_tokenizer(teststr)
    >>> it.next()
    ['First', 'line.']
    >>> it.next()
    ['Beginning', 'of', 'second', 'line', 'and', 'end.']
    >>> it.next()
    ['Beginning', 'of', 'third\\nline', 'and', 'end.']
    >>>
    """
    from shlex import split
    logical_line = ''
    tokens = []
    for physical_line in inputstring.splitlines(True):
        logical_line += physical_line
        try:
            new_tokens = split(logical_line, comments=True)
        except ValueError:
            continue
        if new_tokens and new_tokens[-1] == '\n':
            tokens += new_tokens[:-1]
            logical_line = ''
            continue
        else:
            tokens += new_tokens
            yield tokens
        logical_line = ''
        tokens = []
    if logical_line:
        raise ValueError('Syntax error: unmatched quote at EOF')
