---
pipeline:
  name: mti300ahrs_cron_daily_and_date_control
transform:
  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-date-cntl:v1.0.0
  cmd: ["/bin/bash"]
  env:
    # START_DATE and END_DATE indicate the date range (inclusive) to create the /Y/M/D folder structure
    # START_DATE must be set, format "YYYY-MM-DD"
    # END_DATE can be set or unset (remove line entirely to unset). If unset, end date will be yesterday. 
    OUT_PATH: /pfs/out
    START_DATE: "2020-01-01"
    END_DATE: "2020-01-12"
    SOURCE_TYPE: "mti300ahrs"
  stdin:
  - "#!/bin/bash"
  - "# If END_DATE unset, set to yesterday"
  - "[  -z $END_DATE ] && END_DATE=$(date -u -d '1 day ago' +%Y/%m/%d)"
  - "# Create daily folder structure from start date to end date"
  - for d in $(seq $(date -u +%s -d $START_DATE) +86400 $(date -u +%s -d $END_DATE)) ; do
  - date_path=$(date -u +%Y/%m/%d -d @$d)
  - echo $OUT_PATH/$SOURCE_TYPE/$date_path
  - mkdir -p $OUT_PATH/$SOURCE_TYPE/$date_path
  - touch $OUT_PATH/$SOURCE_TYPE/$date_path/empty
  - done
  - "# Finally, create a file listing the data years (for the metadata assignment pipelines)"
  - years=$(ls $OUT_PATH/$SOURCE_TYPE)
  - for year in $years; do
  - echo $year > $OUT_PATH/data_year_$year.txt
  - done
input:
  # This cron should be the central driver for daily scheduled updates, such as data ingest and metadata loaders.
  cron:
    name: tick
    spec: "0 7 * * *" # Run at 00:00 MST (07:00 GMT)
    overwrite: true
resource_requests:
  memory: 64M
  cpu: 0.5
autoscaling: true
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
    nodepool.neonscience.org/pipeline: "yes"
    cloud.google.com/gke-spot: "true"
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
pod_patch: |-
  [
    { "op": "replace",
      "path":"/containers/1/resources/requests/memory",
      "value":"64M"
    },
    { "op": "replace",
      "path": "/containers/1/resources/requests/cpu",
      "value": "0.1"
    }
  ]
