docs: def ml(line, cell=None):
    """Implements the datalab cell magic for MLWorkbench operations.
  
    Args:
      line: the contents of the ml command line.
    Returns:
      The results of executing the cell.
    """
    parser = google.datalab.utils.commands.CommandParser(
        prog='%ml',
        description=textwrap.dedent("""\
            Execute MLWorkbench operations
  
            Use "%ml <command> -h" for help on a specific command.
        """))
  
    dataset_parser = parser.subcommand(
        'dataset',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Create or explore datasets.')
    dataset_sub_commands = dataset_parser.add_subparsers(dest='command')
    dataset_create_parser = dataset_sub_commands.add_parser(
        'create', help='Create datasets', formatter_class=argparse.RawTextHelpFormatter,
        epilog=textwrap.dedent("""\
            Example usage:
  
            %%ml dataset
            name: mydata
            format: csv
            train: path/to/train.csv
            eval: path/to/eval.csv
            schema:
              - name: news_label
                type: STRING
              - name: text
                type: STRING"""))
  
    dataset_create_parser.add_argument('--name', required=True,
                                       help='the name of the dataset to define. ')
    dataset_create_parser.add_argument('--format', required=True,
                                       choices=['csv', 'bigquery', 'transformed'],
                                       help='The format of the data.')
    dataset_create_parser.add_argument('--train', required=True,
                                       help='The path of the training file pattern if format ' +
                                            'is csv or transformed, or table name if format ' +
                                            'is bigquery.')
    dataset_create_parser.add_argument('--eval', required=True,
                                       help='The path of the eval file pattern if format ' +
                                            'is csv or transformed, or table name if format ' +
                                            'is bigquery.')
    dataset_create_parser.add_cell_argument('schema',
                                            help='yaml representation of CSV schema, or path to ' +
                                            'schema file. Only needed if format is csv.')
    dataset_create_parser.set_defaults(func=_dataset_create)
  
    dataset_explore_parser = dataset_sub_commands.add_parser(
        'explore', help='Explore training data.')
    dataset_explore_parser.add_argument('--name', required=True,
                                        help='The name of the dataset to explore.')
  
    dataset_explore_parser.add_argument('--overview', action='store_true', default=False,
                                        help='Plot overview of sampled data. Set "sample_size" ' +
                                             'to change the default sample size.')
    dataset_explore_parser.add_argument('--facets', action='store_true', default=False,
                                        help='Plot facets view of sampled data. Set ' +
                                             '"sample_size" to change the default sample size.')
    dataset_explore_parser.add_argument('--sample_size', type=int, default=1000,
                                        help='sample size for overview or facets view. Only ' +
                                             'used if either --overview or --facets is set.')
    dataset_explore_parser.set_defaults(func=_dataset_explore)
  
    analyze_parser = parser.subcommand(
        'analyze',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Analyze training data and generate stats, such as min/max/mean '
             'for numeric values, vocabulary for text columns.',
        epilog=textwrap.dedent("""\
            Example usage:
  
            %%ml analyze [--cloud]
            output: path/to/dir
            data: $mydataset
            features:
              serialId:
                transform: key
              num1:
                transform: scale
                value: 1
              num2:
                transform: identity
              text1:
                transform: bag_of_words
  
            Also supports in-notebook variables, such as:
            %%ml analyze --output path/to/dir
            training_data: $my_csv_dataset
            features: $features_def"""))
  
    analyze_parser.add_argument('--output', required=True,
                                help='path of output directory.')
    analyze_parser.add_argument('--cloud', action='store_true', default=False,
                                help='whether to run analysis in cloud or local.')
    analyze_parser.add_argument('--package', required=False,
                                help='A local or GCS tarball path to use as the source. '
                                     'If not set, the default source package will be used.')
    analyze_parser.add_cell_argument(
      'data',
      required=True,
      help="""Training data. A dataset defined by "%%ml dataset".""")
    analyze_parser.add_cell_argument(
        'features',
        required=True,
        help=textwrap.dedent("""\
            features config indicating how to transform data into features. The
            list of supported transforms:
                "transform: identity"
                     does nothing (for numerical columns).
                "transform: scale
                 value: x"
                     scale a numerical column to [-a, a]. If value is missing, x
                     defaults to 1.
                "transform: one_hot"
                     treats the string column as categorical and makes one-hot
                     encoding of it.
                "transform: embedding
                 embedding_dim: d"
                     treats the string column as categorical and makes embeddings of
                     it with specified dimension size.
                "transform: bag_of_words"
                     treats the string column as text and make bag of words
                     transform of it.
                "transform: tfidf"
                     treats the string column as text and make TFIDF transform of it.
                "transform: image_to_vec
                 checkpoint: gs://b/o"
                     from image gs url to embeddings. "checkpoint" is a inception v3
                     checkpoint. If absent, a default checkpoint is used.
                "transform: target"
                     denotes the column is the target. If the schema type of this
                     column is string, a one_hot encoding is automatically applied.
                     If numerical, an identity transform is automatically applied.
                "transform: key"
                     column contains metadata-like information and will be output
                     as-is in prediction."""))
    analyze_parser.set_defaults(func=_analyze)
  
    transform_parser = parser.subcommand(
        'transform',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Transform the data into tf.example which is more efficient in training.',
        epilog=textwrap.dedent("""\
            Example usage:
  
            %%ml transform [--cloud] [--shuffle]
            analysis: path/to/analysis_output_folder
            output: path/to/dir
            batch_size: 100
            data: $mydataset
            cloud:
              num_workers: 3
              worker_machine_type: n1-standard-1
              project_id: my_project_id"""))
    transform_parser.add_argument('--analysis', required=True,
                                  help='path of analysis output directory.')
    transform_parser.add_argument('--output', required=True,
                                  help='path of output directory.')
    transform_parser.add_argument('--cloud', action='store_true', default=False,
                                  help='whether to run transform in cloud or local.')
    transform_parser.add_argument('--shuffle', action='store_true', default=False,
                                  help='whether to shuffle the training data in output.')
    transform_parser.add_argument('--batch_size', type=int, default=100,
                                  help='number of instances in a batch to process once. '
                                       'Larger batch is more efficient but may consume more memory.')
    transform_parser.add_argument('--package', required=False,
                                  help='A local or GCS tarball path to use as the source. '
                                       'If not set, the default source package will be used.')
    transform_parser.add_cell_argument(
        'data',
        required=True,
        help="""Training data. A dataset defined by "%%ml dataset".""")
    transform_parser.add_cell_argument(
        'cloud_config',
        help=textwrap.dedent("""\
            A dictionary of cloud config. All of them are optional.
                num_workers: Dataflow number of workers. If not set, DataFlow
                    service will determine the number.
                worker_machine_type: a machine name from
                    https://cloud.google.com/compute/docs/machine-types
                    If not given, the service uses the default machine type.
                project_id: id of the project to use for DataFlow service. If not set,
                    Datalab's default project (set by %%datalab project set) is used.
                job_name: Unique name for a Dataflow job to use. If not set, a
                    random name will be used."""))
    transform_parser.set_defaults(func=_transform)
  
    train_parser = parser.subcommand(
        'train',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Train a model.',
        epilog=textwrap.dedent("""\
            Example usage:
  
            %%ml train [--cloud]
            analysis: path/to/analysis_output
            output: path/to/dir
            data: $mydataset
            model_args:
              model: linear_regression
            cloud_config:
              region: us-central1"""))
    train_parser.add_argument('--analysis', required=True,
                              help='path of analysis output directory.')
    train_parser.add_argument('--output', required=True,
                              help='path of trained model directory.')
    train_parser.add_argument('--cloud', action='store_true', default=False,
                              help='whether to run training in cloud or local.')
    train_parser.add_argument('--notb', action='store_true', default=False,
                              help='If set, tensorboard is not automatically started.')
    train_parser.add_argument('--package', required=False,
                              help='A local or GCS tarball path to use as the source. '
                                   'If not set, the default source package will be used.')
    train_parser.add_cell_argument(
        'data',
        required=True,
        help="""Training data. A dataset defined by "%%ml dataset".""")
  
    package_model_help = subprocess.Popen(
        ['python', '-m', 'trainer.task', '--datalab-help'],
        cwd=DEFAULT_PACKAGE_PATH,
        stdout=subprocess.PIPE).communicate()[0]
    package_model_help = ('model_args: a dictionary of model specific args, including:\n\n' +
                          package_model_help.decode())
    train_parser.add_cell_argument('model_args', help=package_model_help)
  
    train_parser.add_cell_argument(
        'cloud_config',
        help=textwrap.dedent("""\
            A dictionary of cloud training config, including:
                job_id: the name of the job. If not provided, a default job name is created.
                region: see {url}
                runtime_version: see "region". Must be a string like '1.2'.
                scale_tier: see "region".""".format(
            url='https://cloud.google.com/sdk/gcloud/reference/ml-engine/jobs/submit/training')))
    train_parser.set_defaults(func=_train)
  
    predict_parser = parser.subcommand(
        'predict',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Predict with local or deployed models. (Good for small datasets).',
        epilog=textwrap.dedent("""\
            Example usage:
  
            %%ml predict
            headers: key,num
            model: path/to/model
            data:
              - key1,value1
              - key2,value2
  
            Or, in another cell, define a list of dict:
  
            my_data = [{'key': 1, 'num': 1.2}, {'key': 2, 'num': 2.8}]
  
            Then:
  
            %%ml predict
            headers: key,num
            model: path/to/model
            data: $my_data"""))
    predict_parser.add_argument('--model', required=True,
                                help='The model path.')
    predict_parser.add_argument('--no_show_image', action='store_true', default=False,
                                help='If not set, add a column of images in output.')
    predict_parser.add_cell_argument(
        'data',
        required=True,
        help=textwrap.dedent("""\
            Prediction data can be
                1) CSV lines in the input cell in yaml format or
                2) a local variable which is one of
                  a) list of dict
                  b) list of strings of csv lines
                  c) a Pandas DataFrame"""))
    predict_parser.set_defaults(func=_predict)
  
    batch_predict_parser = parser.subcommand(
        'batch_predict',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Batch prediction with local or deployed models. (Good for large datasets)',
        epilog=textwrap.dedent("""\
  
        Example usage:
  
        %%ml batch_predict [--cloud]
        model: path/to/model
        output: path/to/output
        format: csv
        data:
          csv: path/to/file_pattern"""))
    batch_predict_parser.add_argument('--model', required=True,
                                      help='The model path if not --cloud, or the id in '
                                           'the form of model.version if --cloud.')
    batch_predict_parser.add_argument('--output', required=True,
                                      help='The path of output directory with prediction results. '
                                           'If --cloud, it has to be GCS path.')
    batch_predict_parser.add_argument('--format',
                                      help='csv or json. For cloud run, '
                                           'the only supported format is json.')
    batch_predict_parser.add_argument('--batch_size', type=int, default=100,
                                      help='number of instances in a batch to process once. '
                                           'Larger batch is more efficient but may consume '
                                           'more memory. Only used in local run.')
    batch_predict_parser.add_argument('--cloud', action='store_true', default=False,
                                      help='whether to run prediction in cloud or local.')
    batch_predict_parser.add_cell_argument(
        'data',
        required=True,
        help='Data to predict with. Only csv is supported.')
    batch_predict_parser.add_cell_argument(
        'cloud_config',
        help=textwrap.dedent("""\
            A dictionary of cloud batch prediction config.
                job_id: the name of the job. If not provided, a default job name is created.
                region: see {url}
                max_worker_count: see reference in "region".""".format(
                    url='https://cloud.google.com/sdk/gcloud/reference/ml-engine/jobs/submit/prediction')))  # noqa
    batch_predict_parser.set_defaults(func=_batch_predict)
  
    explain_parser = parser.subcommand(
        'explain',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Explain a prediction with LIME tool.')
    explain_parser.add_argument('--type', default='all', choices=['text', 'image', 'tabular', 'all'],
                                help='the type of column to explain.')
    explain_parser.add_argument('--algorithm', choices=['lime', 'ig'], default='lime',
                                help='"lime" is the open sourced project for prediction explainer.' +
                                     '"ig" means integrated gradients and currently only applies ' +
                                     'to image.')
    explain_parser.add_argument('--model', required=True,
                                help='path of the model directory used for prediction.')
    explain_parser.add_argument('--labels', required=True,
                                help='comma separated labels to explain.')
    explain_parser.add_argument('--column_name',
                                help='the name of the column to explain. Optional if text type ' +
                                     'and there is only one text column, or image type and ' +
                                     'there is only one image column.')
    explain_parser.add_cell_argument('data', required=True,
                                     help='Prediction Data. Can be a csv line, or a dict.')
    explain_parser.add_cell_argument('training_data',
                                     help='A csv or bigquery dataset defined by %%ml dataset. ' +
                                          'Used by tabular explainer only to determine the ' +
                                          'distribution of numeric and categorical values. ' +
                                          'Suggest using original training dataset.')
  
    # options specific for lime
    explain_parser.add_argument('--num_features', type=int,
                                help='number of features to analyze. In text, it is number of ' +
                                     'words. In image, it is number of areas. For lime only.')
    explain_parser.add_argument('--num_samples', type=int,
                                help='size of the neighborhood to learn the linear model. ' +
                                     'For lime only.')
    explain_parser.add_argument('--hide_color', type=int, default=0,
                                help='the color to use for perturbed area. If -1, average of ' +
                                     'each channel is used for each channel. For image only.')
    explain_parser.add_argument('--include_negative', action='store_true', default=False,
                                help='whether to show only positive areas. For lime image only.')
    explain_parser.add_argument('--overview', action='store_true', default=False,
                                help='whether to show overview instead of details view.' +
                                     'For lime text and tabular only.')
    explain_parser.add_argument('--batch_size', type=int, default=100,
                                help='size of batches passed to prediction. For lime only.')
  
    # options specific for integrated gradients
    explain_parser.add_argument('--num_gradients', type=int, default=50,
                                help='the number of scaled images to get gradients from. Larger ' +
                                     'number usually produces better results but slower.')
    explain_parser.add_argument('--percent_show', type=int, default=10,
                                help='the percentage of top impactful pixels to show.')
  
    explain_parser.set_defaults(func=_explain)
  
    tensorboard_parser = parser.subcommand(
        'tensorboard',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Start/stop/list TensorBoard instances.')
    tensorboard_sub_commands = tensorboard_parser.add_subparsers(dest='command')
  
    tensorboard_start_parser = tensorboard_sub_commands.add_parser(
        'start', help='Start a tensorboard instance.')
    tensorboard_start_parser.add_argument('--logdir', required=True,
                                          help='The local or GCS logdir path.')
    tensorboard_start_parser.set_defaults(func=_tensorboard_start)
  
    tensorboard_stop_parser = tensorboard_sub_commands.add_parser(
        'stop', help='Stop a tensorboard instance.')
    tensorboard_stop_parser.add_argument('--pid', required=True, type=int,
                                         help='The pid of the tensorboard instance.')
    tensorboard_stop_parser.set_defaults(func=_tensorboard_stop)
  
    tensorboard_list_parser = tensorboard_sub_commands.add_parser(
        'list', help='List tensorboard instances.')
    tensorboard_list_parser.set_defaults(func=_tensorboard_list)
  
    evaluate_parser = parser.subcommand(
        'evaluate',
        formatter_class=argparse.RawTextHelpFormatter,
        help='Analyze model evaluation results, such as confusion matrix, ROC, RMSE.')
    evaluate_sub_commands = evaluate_parser.add_subparsers(dest='command')
  
    def _add_data_params_for_evaluate(parser):
      parser.add_argument('--csv', help='csv file path patterns.')
      parser.add_argument('--headers',
                          help='csv file headers. Required if csv is specified and ' +
                               'predict_results_schema.json does not exist in the same directory.')
      parser.add_argument('--bigquery',
                          help='can be bigquery table, query as a string, or ' +
                               'a pre-defined query (%%bq query --name).')
  
    evaluate_cm_parser = evaluate_sub_commands.add_parser(
        'confusion_matrix', help='Get confusion matrix from evaluation results.')
    _add_data_params_for_evaluate(evaluate_cm_parser)
    evaluate_cm_parser.add_argument('--plot', action='store_true', default=False,
                                    help='Whether to plot confusion matrix as graph.')
    evaluate_cm_parser.add_argument('--size', type=int, default=10,
                                    help='The size of the confusion matrix.')
    evaluate_cm_parser.set_defaults(func=_evaluate_cm)
  
    evaluate_accuracy_parser = evaluate_sub_commands.add_parser(
        'accuracy', help='Get accuracy results from classification evaluation results.')
    _add_data_params_for_evaluate(evaluate_accuracy_parser)
    evaluate_accuracy_parser.set_defaults(func=_evaluate_accuracy)
  
    evaluate_pr_parser = evaluate_sub_commands.add_parser(
        'precision_recall', help='Get precision recall metrics from evaluation results.')
    _add_data_params_for_evaluate(evaluate_pr_parser)
    evaluate_pr_parser.add_argument('--plot', action='store_true', default=False,
                                    help='Whether to plot precision recall as graph.')
    evaluate_pr_parser.add_argument('--num_thresholds', type=int, default=20,
                                    help='Number of thresholds which determines how many ' +
                                         'points in the graph.')
    evaluate_pr_parser.add_argument('--target_class', required=True,
                                    help='The target class to determine correctness of ' +
                                         'a prediction.')
    evaluate_pr_parser.add_argument('--probability_column',
                                    help='The name of the column holding the probability ' +
                                         'value of the target class. If absent, the value ' +
                                         'of target class is used.')
    evaluate_pr_parser.set_defaults(func=_evaluate_pr)
  
    evaluate_roc_parser = evaluate_sub_commands.add_parser(
        'roc', help='Get ROC metrics from evaluation results.')
    _add_data_params_for_evaluate(evaluate_roc_parser)
    evaluate_roc_parser.add_argument('--plot', action='store_true', default=False,
                                     help='Whether to plot ROC as graph.')
    evaluate_roc_parser.add_argument('--num_thresholds', type=int, default=20,
                                     help='Number of thresholds which determines how many ' +
                                          'points in the graph.')
    evaluate_roc_parser.add_argument('--target_class', required=True,
                                     help='The target class to determine correctness of ' +
                                          'a prediction.')
    evaluate_roc_parser.add_argument('--probability_column',
                                     help='The name of the column holding the probability ' +
                                          'value of the target class. If absent, the value ' +
                                          'of target class is used.')
    evaluate_roc_parser.set_defaults(func=_evaluate_roc)
  
    evaluate_regression_parser = evaluate_sub_commands.add_parser(
        'regression', help='Get regression metrics from evaluation results.')
    _add_data_params_for_evaluate(evaluate_regression_parser)
    evaluate_regression_parser.set_defaults(func=_evaluate_regression)
  
    model_parser = parser.subcommand(
        'model',
        help='Models and versions management such as deployment, deletion, listing.')
    model_sub_commands = model_parser.add_subparsers(dest='command')
    model_list_parser = model_sub_commands.add_parser(
        'list', help='List models and versions.')
    model_list_parser.add_argument('--name',
                                   help='If absent, list all models of specified or current ' +
                                        'project. If provided, list all versions of the ' +
                                        'model.')
    model_list_parser.add_argument('--project',
                                   help='The project to list model(s) or version(s). If absent, ' +
                                        'use Datalab\'s default project.')
    model_list_parser.set_defaults(func=_model_list)
  
    model_delete_parser = model_sub_commands.add_parser(
        'delete', help='Delete models or versions.')
    model_delete_parser.add_argument('--name', required=True,
                                     help='If no "." in the name, try deleting the specified ' +
                                          'model. If "model.version" is provided, try deleting ' +
                                          'the specified version.')
    model_delete_parser.add_argument('--project',
                                     help='The project to delete model or version. If absent, ' +
                                          'use Datalab\'s default project.')
    model_delete_parser.set_defaults(func=_model_delete)
  
    model_deploy_parser = model_sub_commands.add_parser(
        'deploy', help='Deploy a model version.')
    model_deploy_parser.add_argument('--name', required=True,
                                     help='Must be model.version to indicate the model ' +
                                          'and version name to deploy.')
    model_deploy_parser.add_argument('--path', required=True,
                                     help='The GCS path of the model to be deployed.')
    model_deploy_parser.add_argument('--runtime_version',
                                     help='The TensorFlow version to use for this model. ' +
                                          'For example, "1.2.1". If absent, the current ' +
                                          'TensorFlow version installed in Datalab will be used.')
    model_deploy_parser.add_argument('--project',
                                     help='The project to deploy a model version. If absent, ' +
                                          'use Datalab\'s default project.')
    model_deploy_parser.set_defaults(func=_model_deploy)
  
    return google.datalab.utils.commands.handle_magic_line(line, cell, parser)