def tokenizer(fmt):
    """
    Split a format into a series of tokens, adding parens for optional values.

    For example,
      '%(%H:%)%?%M %!Z%?'
    will become
      '%(', '%H', ':', '%)', '%?', '%M', ' ', '%(', '%!Z', '%)', '%?'
    """
    i = 0
    n = len(fmt)
    while i < n:
        j = i + 1
        if fmt[i] == '%':
            j += 1
            if j > n:
                raise ValueError('Incomplete token - nothing follows %')
            if fmt[j - 1] == '!':
                j += 1
                if j > n:
                    raise ValueError('Incomplete token - nothing follows %!')
        optional = j + 1 < n and fmt[j:j + 2] == '%?'
        single = fmt[i:j] != '%)'
        if optional and single:
            yield '%('
        yield fmt[i:j]
        if optional:
            if single:
                yield '%)'
            yield '%?'
            j += 2
        i = j
