---
pipeline:
  name: pressureAir_relHumidity_group
transform:
#  image_pull_secrets: [battelleecology-quay-read-all-pull-secret]
  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-filt-join:v1.0.0
  cmd: ["/bin/bash"]
  stdin:
  - "#!/bin/bash"
  - "python3 -m filter_joiner.filter_joiner_main"
  env:
    CONFIG: |
      ---
      # In Pachyderm root will be index 0, 'pfs' index 1, and the repo name index 2.
      # Metadata indices will typically begin at index 3.
      # Use unix-style glob pattern to select the desired directories in each repo 
      input_paths:
        - path:
            name: QUALITY_METRICS_PATH
            # Filter for data directory
            glob_pattern: /pfs/QUALITY_METRICS_PATH/*/*/*/*/ptb330a/*/quality_metrics/**
            # Join on group ID (already joined below by day)
            join_indices: [6]
            outer_join: true
        - path:
            name: STATISTICS_PATH
            # Filter for data directory
            glob_pattern: /pfs/STATISTICS_PATH/*/*/*/*/ptb330a/*/stats/**
            # Join on group ID(already joined below by day)
            join_indices: [6]
            outer_join: true
        - path:
            name: GROUP_PATH
            # Filter for data directory
            glob_pattern: /pfs/GROUP_PATH/*/*/*/*/ptb330a/*/location/**
            # Join on group ID(already joined below by day)
            join_indices: [6]
        - path:
            name: GROUP_PATH
            # Filter for data directory
            glob_pattern: /pfs/GROUP_PATH/*/*/*/*/group/**
            # Join on group ID(already joined below by day)
            join_indices: [6]
        - path:
            name: RELHUMIDITY_PATH
            # Filter for data directory
            glob_pattern: /pfs/RELHUMIDITY_PATH/*/*/*/*/**
            # Join on group ID(already joined below by day)
            join_indices: [6]
    OUT_PATH: /pfs/out
    LOG_LEVEL: INFO
    RELATIVE_PATH_INDEX: "3"
    LINK_TYPE: SYMLINK # options are COPY or SYMLINK
input:
  join:
  - pfs:
      name: QUALITY_METRICS_PATH
      repo: pressureAir_qm_group_and_compute
      glob: /(*)/(*)/(*)
      joinOn: $1/$2/$3
      empty_files: false # Make sure this is false for LINK_TYPE=COPY
      outer_join: true
  - pfs:
      name: STATISTICS_PATH
      repo: pressureAir_stats_group_and_compute
      glob: /(*)/(*)/(*)
      joinOn: $1/$2/$3
      empty_files: false # Make sure this is false for LINK_TYPE=COPY
      outer_join: true
  - pfs:
      name: GROUP_PATH
      repo: pressureAir_group_path_ptb330a
      glob: /(*)/(*)/(*)
      joinOn: $1/$2/$3
      empty_files: false # Make sure this is false for LINK_TYPE=COPY
  - pfs:
      name: RELHUMIDITY_PATH
      repo: pressureAir_group_path_relHumidity
      glob: /(*)/(*)/(*)
      joinOn: $1/$2/$3
      empty_files: false # Make sure this is false for LINK_TYPE=COPY
parallelism_spec:
  constant: 1
autoscaling: true
resource_requests:
  memory: 600M
  cpu: 0.4
resource_limits:
  memory: 2G
  cpu: 1.2
sidecar_resource_requests:
  memory: 2G
  cpu: 0.5
sidecar_resource_limits:
  memory: 4Gi
  cpu: 1.5
datum_set_spec:
  number: 1
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
    nodepool.neonscience.org/pipeline: "yes"
    cloud.google.com/gke-spot: "true"
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
