(sql, standard_quoting = False, ignore_whitespace = False,
                  fqident = False, show_location = False)
r"""Parser SQL to tokens.

    Iterator, returns (toktype, tokstr) tuples.

    Example
    >>> [x for x in sql_tokenizer("select * from a.b", ignore_whitespace=True)]
    [('ident', 'select'), ('sym', '*'), ('ident', 'from'), ('ident', 'a'), ('sym', '.'), ('ident', 'b')]
    >>> [x for x in sql_tokenizer("\"c olumn\",'str''val'")]
    [('ident', '"c olumn"'), ('sym', ','), ('str', "'str''val'")]
    >>> list(sql_tokenizer('a.b a."b "" c" a.1', fqident=True, ignore_whitespace=True))
    [('ident', 'a.b'), ('ident', 'a."b "" c"'), ('ident', 'a'), ('sym', '.'), ('num', '1')]
r"""Parser SQL to tokens.
r Parser SQL to tokens .
def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False,
                  fqident = False, show_location = False):
    r"""Parser SQL to tokens.

    Iterator, returns (toktype, tokstr) tuples.

    Example
    >>> [x for x in sql_tokenizer("select * from a.b", ignore_whitespace=True)]
    [('ident', 'select'), ('sym', '*'), ('ident', 'from'), ('ident', 'a'), ('sym', '.'), ('ident', 'b')]
    >>> [x for x in sql_tokenizer("\"c olumn\",'str''val'")]
    [('ident', '"c olumn"'), ('sym', ','), ('str', "'str''val'")]
    >>> list(sql_tokenizer('a.b a."b "" c" a.1', fqident=True, ignore_whitespace=True))
    [('ident', 'a.b'), ('ident', 'a."b "" c"'), ('ident', 'a'), ('sym', '.'), ('num', '1')]
    """
    global _std_sql_rc, _ext_sql_rc, _std_sql_fq_rc, _ext_sql_fq_rc
    if not _std_sql_rc:
        _std_sql_rc = re.compile(_std_sql, re.X | re.I | re.S)
        _ext_sql_rc = re.compile(_ext_sql, re.X | re.I | re.S)
        _std_sql_fq_rc = re.compile(_std_sql_fq, re.X | re.I | re.S)
        _ext_sql_fq_rc = re.compile(_ext_sql_fq, re.X | re.I | re.S)

    if standard_quoting:
        if fqident:
            rc = _std_sql_fq_rc
        else:
            rc = _std_sql_rc
    else:
        if fqident:
            rc = _ext_sql_fq_rc
        else:
            rc = _ext_sql_rc

    pos = 0
    while 1:
        m = rc.match(sql, pos)
        if not m:
            break
        pos = m.end()
        typ = m.lastgroup
        if ignore_whitespace and typ == "ws":
            continue
        tk = m.group()
        if show_location:
            yield (typ, tk, pos)
        else:
            yield (typ, tk)
def sql_tokenizer ( sql , standard_quoting = False , ignore_whitespace = False , fqident = False , show_location = False ) : global _std_sql_rc , _ext_sql_rc , _std_sql_fq_rc , _ext_sql_fq_rc if not _std_sql_rc : _std_sql_rc = re . compile ( _std_sql , re . X | re . I | re . S ) _ext_sql_rc = re . compile ( _ext_sql , re . X | re . I | re . S ) _std_sql_fq_rc = re . compile ( _std_sql_fq , re . X | re . I | re . S ) _ext_sql_fq_rc = re . compile ( _ext_sql_fq , re . X | re . I | re . S ) if standard_quoting : if fqident : rc = _std_sql_fq_rc else : rc = _std_sql_rc else : if fqident : rc = _ext_sql_fq_rc else : rc = _ext_sql_rc pos = 0 while 1 : m = rc . match ( sql , pos ) if not m : break pos = m . end ( ) typ = m . lastgroup if ignore_whitespace and typ == "ws" : continue tk = m . group ( ) if show_location : yield ( typ , tk , pos ) else : yield ( typ , tk )