def token_stream(file, break_on_newline=True):
    """Word tokenizer. Also breaks on newlines."""
    while True:
        line = file.readline()
        for word in line.split(None):
            yield word
        if break_on_newline:
            yield '\n'
