---
pipeline:
  name: tchain_date_gap_filler
transform:
  # image_pull_secrets:
  # - battelleecology-quay-read-all-pull-secret
image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-date-gap-fill:v1.0.0

  cmd:
  - /bin/bash
  stdin:
  - '#!/bin/bash'
  - python3 -m date_gap_filler.date_gap_filler_main
  env:
    LOG_LEVEL: DEBUG
    OUT_PATH: /pfs/out
    OUTPUT_DIRECTORIES: data,location,uncertainty_data,uncertainty_coef,flags
    DATA_SOURCE_TYPE_INDEX: '3'
    DATA_YEAR_INDEX: '4'
    DATA_MONTH_INDEX: '5'
    DATA_DAY_INDEX: '6'
    DATA_LOCATION_INDEX: '7'
    DATA_TYPE_INDEX: '8'
    LOCATION_SOURCE_TYPE_INDEX: '3'
    LOCATION_YEAR_INDEX: '4'
    LOCATION_MONTH_INDEX: '5'
    LOCATION_DAY_INDEX: '6'
    LOCATION_INDEX: '7'
    EMPTY_FILE_TYPE_INDEX: '4'
    LINK_TYPE: SYMLINK # options are COPY or SYMLINK
input:
  cross:
  - pfs:
      name: EMPTY_FILE_PATH
      repo: empty_files_tchain
      glob: /tchain
      empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK.
  - group:
    - pfs:
        name: DATA_PATH
        repo: tchain_merge_data_by_location
        # For full-scale daily processing, glob should be /tchain/(*/*/*). To limit to particular CFGLOCs, note the parentheses and enter something like /tchain/(*/*/*/(CFGLOC100476|CFGLOC100489|CFGLOC100490))
        glob: /tchain/(*/*/*)
        group_by: $1
        empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK.
    - join:
      - pfs:
          name: LOCATION_PATH
          repo: tchain_location_active_dates_assignment
          # For full-scale daily processing, glob should be /tchain/(*/*/*). To limit to particular CFGLOCs, note the parentheses and enter something like /tchain/((*/*/*)/(CFGLOC100476|CFGLOC100489|CFGLOC100490))
          glob: /tchain/(*/*/*)
          # For full-scale daily processing, joinOn be $1. When limiting to particular CFGLOCs, joinOn will be $2 to match parentheses around (*/*/*)
          joinOn: $1
          group_by: $1
          empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK.
      - pfs:
          name: DATE_LIMITER_PATH
          repo: tchain_date_gap_filler_limiter
          glob: /(*/*/*)
          joinOn: $1
          group_by: $1
          empty_files: true # This can remain true even if LINK_TYPE=COPY
parallelism_spec:
  constant: 1
resource_requests:
  memory: 200M
  cpu: 0.2
autoscaling: true
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
    nodepool.neonscience.org/pipeline: "yes"
    cloud.google.com/gke-spot: "true"
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
pod_patch: |-
  [
    { "op": "replace",
      "path":"/containers/1/resources/requests/memory",
      "value":"2G"
    },
    { "op": "replace",
      "path": "/containers/1/resources/requests/cpu",
      "value": "0.5"
    }
  ]
