(code, version_info, start_pos=(1, 0))
return tokenize_lines(lines, version_info, start_pos=start_pos)
Generate tokens from a the source code (string).
Generate tokens from a the source code (string).
Generate tokens from a the source code ( string ) .
def tokenize(code, version_info, start_pos=(1, 0)):
    """Generate tokens from a the source code (string)."""
    lines = split_lines(code, keepends=True)
    return tokenize_lines(lines, version_info, start_pos=start_pos)
def tokenize ( code , version_info , start_pos = ( 1 , 0 ) ) : lines = split_lines ( code , keepends = True ) return tokenize_lines ( lines , version_info , start_pos = start_pos )