parameters: (source)
return_statement: return tokenize.generate_tokens(io.StringIO(source).readline)
docstring: Tokenize Python source code.
docstring_summary: Tokenize Python source code.
docstring_tokens: Tokenize Python source code .
function: def _tokenize(source):
      """Tokenize Python source code."""
      # Using an undocumented API as the documented one in Python 2.7 does not work as needed
      # cross-version.
      return tokenize.generate_tokens(io.StringIO(source).readline)
function_tokens: def _tokenize ( source ) : # Using an undocumented API as the documented one in Python 2.7 does not work as needed # cross-version. return tokenize . generate_tokens ( io . StringIO ( source ) . readline )