@classmethod
def tokenize_with_lookahead(cls, tokenizer, input_str):
    """
        Yields (token, lookahead) one by one.
        Lookahead is None for the last token.
        """
    prev_token = None
    current_token = None
    for current_token in tokenizer(input_str):
        if prev_token is not None:
            yield prev_token, current_token
        prev_token = current_token
    if current_token is not None:
        yield current_token, None
