---
pipeline:
  name: presValiRegOutTurb_thresh_select_ts_pad
transform:
  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-thsh-slct-ts-pad:v2.1.0
  cmd:
  - sh
  - "-c"
  - |-
    /bin/bash <<'EOF'
    # Use bash-scrict mode. See http://redsymbol.net/articles/unofficial-bash-strict-mode/
    set -euo pipefail
    IFS=$'\n\t'
    # Refresh interim directories with each datum (otherwise they persist and cause probs)
    rm -rf /tmp/threshold_select
    rm -rf /tmp/threshold_selectCopy
    mkdir -p /tmp/threshold_select
    # Run first module - threshold_select
    Rscript ./flow.thsh.slct.R \
      DirIn=$REPO_LOCATIONS \
      DirOut=/tmp/threshold_select \
      DirErr=/pfs/out/errored_datums \
      FileThsh=$FILE_THRESHOLDS \
      "TermCtxt1=presDiff" \
      "DirSubCopy=data|location"
    # Copy output to another interim folder to destroy links (cannot daisy chain links from pfs input to output)
    cp -rL /tmp/threshold_select /tmp/threshold_selectCopy || : # Allow to fail without exit code (happens if step above produced no output) 
    rm -r -f /tmp/threshold_select 
    # Run second module - timeseries_padder
    python3 -m timeseries_padder.timeseries_padder.variable_pad_main --yearindex 3 --monthindex 4 --dayindex 5 --locindex 8 --subdirindex 9
    EOF
  env:
    DATA_PATH: /tmp/threshold_selectCopy
    OUT_PATH: /pfs/out
    LOG_LEVEL: INFO
    PAD_DIR: data
    COPY_DIR: none # Can be multiple, separated by commas without spaces. Directories other than the pad directory and threshold directory to copy to the output (e.g. location,flags). Set to something like 'none' if none other are desired.
    RELATIVE_PATH_INDEX: '3'
    PARALLELIZATION_INTERNAL: '1' # For threshold select module
output_branch: master
input:
  cross:
  - pfs:
      name: REPO_LOCATIONS
      repo: pressuretransducer_l0p_data
      glob: /presValiRegOutTurb/*/*/*
  - pfs:
      name: FILE_THRESHOLDS
      repo: presValiRegOutTurb_threshold
      glob: /thresholds.json
parallelism_spec:
  constant: 4
resource_requests:
  memory: 200M
  cpu: 1.2
resource_limits:
  memory: 1G
  cpu: 2
sidecar_resource_requests:
  memory: 1G
  cpu: 0.3
sidecar_resource_limits:
  memory: 2Gi
  cpu: 1.2
datum_set_spec:
  number: 1
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
    nodepool.neonscience.org/pipeline: "yes"
    cloud.google.com/gke-spot: "true"
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
