def _compute_char_shifts(tokens):
    """Compute the shifts in characters that occur when comparing the
    tokens string with the string consisting of all tokens separated with a
    space

    For instance, if "hello?world" is tokenized in ["hello", "?", "world"],
    then the character shifts between "hello?world" and "hello ? world" are
    [0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2]
    """
    characters_shifts = []
    if not tokens:
        return characters_shifts
    current_shift = 0
    for token_index, token in enumerate(tokens):
        if token_index == 0:
            previous_token_end = 0
            previous_space_len = 0
        else:
            previous_token_end = tokens[token_index - 1].end
            previous_space_len = 1
        offset = token.start - previous_token_end - previous_space_len
        current_shift -= offset
        token_len = token.end - token.start
        index_shift = token_len + previous_space_len
        characters_shifts += [current_shift for _ in range(index_shift)]
    return characters_shifts
