parameters: (code, version_info, start_pos=(1, 0))
return_statement: return tokenize_lines(lines, version_info, start_pos=start_pos)
docstring: Generate tokens from a the source code (string).
docstring_summary: Generate tokens from a the source code (string).
docstring_tokens: Generate tokens from a the source code ( string ) .
function: def tokenize(code, version_info, start_pos=(1, 0)):
      """Generate tokens from a the source code (string)."""
      lines = split_lines(code, keepends=True)
      return tokenize_lines(lines, version_info, start_pos=start_pos)
function_tokens: def tokenize ( code , version_info , start_pos = ( 1 , 0 ) ) : lines = split_lines ( code , keepends = True ) return tokenize_lines ( lines , version_info , start_pos = start_pos )