# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"apiVersion": "batch/v1beta1"
"kind": "CronJob"
"metadata":
  "labels":
    "accelerator": "v3-32"
    "benchmarkId": "tf-r2.4.0-shapemask-conv-v3-32"
    "frameworkVersion": "tf-r2.4.0"
    "mode": "conv"
    "model": "shapemask"
  "name": "tf-r2.4.0-shapemask-conv-v3-32"
  "namespace": "automated"
"spec":
  "concurrencyPolicy": "Forbid"
  "jobTemplate":
    "metadata":
      "annotations":
        "ml-testing-accelerators/gcs-subdir": "tf-r2.4.0/shapemask/conv/v3-32"
        "ml-testing-accelerators/metric-config": |
          {
            "sources": [
              {
                "literals": {
                  "assertions": {
                    "duration": {
                      "inclusive_bounds": false,
                      "std_devs_from_mean": {
                        "comparison": "LESS",
                        "std_devs": 5
                      },
                      "wait_for_n_data_points": 10
                    }
                  }
                }
              },
              {
                "tensorboard": {
                  "aggregate_assertions": [
                    {
                      "assertion": {
                        "inclusive_bounds": true,
                        "std_devs_from_mean": {
                          "comparison": "GREATER",
                          "std_devs": 2
                        },
                        "wait_for_n_data_points": 0
                      },
                      "strategy": "AVERAGE",
                      "tag": "examples_per_second"
                    }
                  ],
                  "exclude_tags": [
          
                  ],
                  "include_tags": [
                    {
                      "strategies": [
                        "FINAL"
                      ],
                      "tag_pattern": "*"
                    }
                  ],
                  "merge_runs": false
                }
              }
            ]
          }
      "labels":
        "accelerator": "v3-32"
        "benchmarkId": "tf-r2.4.0-shapemask-conv-v3-32"
        "frameworkVersion": "tf-r2.4.0"
        "mode": "conv"
        "model": "shapemask"
    "spec":
      "activeDeadlineSeconds": 36000
      "backoffLimit": 1
      "template":
        "metadata":
          "annotations":
            "reserved.cloud-tpus.google.com": "false"
            "tf-version.cloud-tpus.google.com": "2.4.0"
        "spec":
          "containers":
          - "env":
            - "name": "POD_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.name"
            - "name": "POD_NAMESPACE"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.namespace"
            "image": "gcr.io/xl-ml-test/health-monitor:stable"
            "imagePullPolicy": "Always"
            "name": "monitor"
          - "args":
            - "/bin/bash"
            - "-c"
            - |
              set -u
              set -e
              set -x
              
              python3 official/vision/detection/main.py \
                --tpu=$(KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS) \
                --strategy_type=tpu \
                --model=shapemask \
                --model_dir=$(MODEL_DIR) \
               --mode=train \
              --params_override=""eval":
                "batch_size": 64
                "eval_file_pattern": "$(COCO_DIR)/val*"
                "val_json_file": "$(COCO_DIR)/instances_val2017.json"
              "shapemask_head":
                "shape_prior_path": "gs://cloud-tpu-checkpoints/shapemask/kmeans_class_priors_91x20x32x32.npy"
                "use_category_for_mask": true
              "shapemask_parser":
                "output_size":
                - 1024
                - 1024
              "train":
                "batch_size": 256
                "checkpoint":
                  "path": "$(RESNET_PRETRAIN_DIR)/resnet50-checkpoint-2018-02-07"
                  "prefix": "resnet50/"
                "iterations_per_loop": 5000
                "learning_rate":
                  "learning_rate_levels":
                  - 0.0080000000000000002
                  - 0.00080000000000000004
                  "learning_rate_steps":
                  - 15000
                  - 20000
                "total_steps": 22500
                "train_file_pattern": "$(COCO_DIR)/train*""
              python3 official/vision/detection/main.py \
                --tpu=$(KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS) \
                --strategy_type=tpu \
                --model=shapemask \
                --model_dir=$(MODEL_DIR) \
               --mode=eval \
              --params_override=""eval":
                "batch_size": 64
                "eval_file_pattern": "$(COCO_DIR)/val*"
                "val_json_file": "$(COCO_DIR)/instances_val2017.json"
              "shapemask_head":
                "shape_prior_path": "gs://cloud-tpu-checkpoints/shapemask/kmeans_class_priors_91x20x32x32.npy"
                "use_category_for_mask": true
              "shapemask_parser":
                "output_size":
                - 1024
                - 1024
              "train":
                "batch_size": 256
                "checkpoint":
                  "path": "$(RESNET_PRETRAIN_DIR)/resnet50-checkpoint-2018-02-07"
                  "prefix": "resnet50/"
                "iterations_per_loop": 5000
                "learning_rate":
                  "learning_rate_levels":
                  - 0.0080000000000000002
                  - 0.00080000000000000004
                  "learning_rate_steps":
                  - 15000
                  - 20000
                "total_steps": 22500
                "train_file_pattern": "$(COCO_DIR)/train*""
              
            "env":
            - "name": "POD_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.name"
            - "name": "POD_UID"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.uid"
            - "name": "POD_NAMESPACE"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.namespace"
            - "name": "JOB_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.labels['job-name']"
            - "name": "MODEL_DIR"
              "value": "$(OUTPUT_BUCKET)/tf-r2.4.0/shapemask/conv/v3-32/$(JOB_NAME)"
            "envFrom":
            - "configMapRef":
                "name": "gcs-buckets"
            "image": "gcr.io/xl-ml-test/tensorflow:r2.4.0"
            "imagePullPolicy": "Always"
            "name": "train"
            "resources":
              "limits":
                "cloud-tpus.google.com/v3": 32
              "requests":
                "cpu": 2
                "memory": "20G"
            "volumeMounts":
            - "mountPath": "/dev/shm"
              "name": "dshm"
              "readOnly": false
          "initContainers":
          - "env":
            - "name": "POD_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.name"
            - "name": "POD_UID"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.uid"
            - "name": "POD_NAMESPACE"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.namespace"
            - "name": "JOB_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.labels['job-name']"
            - "name": "MODEL_DIR"
              "value": "$(OUTPUT_BUCKET)/tf-r2.4.0/shapemask/conv/v3-32/$(JOB_NAME)"
            - "name": "METRIC_CONFIG"
              "value": |
                {
                 "metric_collection_config": {
                  "default_aggregation_strategies": [
                   "final"
                  ],
                  "metric_to_aggregation_strategies": {
                   "examples_per_second": [
                    "average"
                   ]
                  },
                  "use_run_name_prefix": true,
                  "write_to_bigquery": true
                 },
                 "regression_test_config": {
                  "metric_success_conditions": {
                   "examples_per_second_average": {
                    "comparison": "greater_or_equal",
                    "success_threshold": {
                     "stddevs_from_mean": 2
                    }
                   },
                   "total_wall_time": {
                    "comparison": "less",
                    "success_threshold": {
                     "stddevs_from_mean": 5
                    },
                    "wait_for_n_points_of_history": 10
                   }
                  }
                 },
                 "test_name": "tf-r2.4.0-shapemask-conv-v3-32"
                }
            "envFrom":
            - "configMapRef":
                "name": "gcs-buckets"
            "image": "gcr.io/xl-ml-test/publisher:stable"
            "imagePullPolicy": "Always"
            "name": "publisher"
          "nodeSelector":
            "tpu-available": "true"
          "restartPolicy": "Never"
          "volumes":
          - "emptyDir":
              "medium": "Memory"
            "name": "dshm"
  "schedule": "0 8 * * *"
  "successfulJobsHistoryLimit": 1