---
pipeline:
  name: relHumidity_pub_format_and_package
transform:
  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-pub-grp-pack:v3.1.1
  cmd:
  - sh
  - "-c"
  - |-
    /bin/bash <<'EOF'
    # Use bash-scrict mode. See http://redsymbol.net/articles/unofficial-bash-strict-mode/
    set -euo pipefail
    IFS=$'\n\t'
    # Refresh interim directories with each datum (otherwise they persist and cause probs)
    rm -rf $OUT_PATH_TRANSFORMER
    rm -rf $OUT_PATH_PACKAGER
    mkdir $OUT_PATH_TRANSFORMER
    mkdir $OUT_PATH_PACKAGER
    # Set some environment variables for the second module
    export DATA_PATH=$GROUPED_PATH
    export OUT_PATH=$OUT_PATH_TRANSFORMER
    # Run second module - pub_transformer (using environment variables below as input parameters)
    python3 -m pub_workbook_loader.pub_workbook_loader_main
    python3 -m pub_transformer.pub_transformer_main
    # Set some environment variables for the third module
    export OUT_PATH=$OUT_PATH_PACKAGER
    # Packager needs to be run at monthly glob. Get those paths.
    product_month_paths="${OUT_PATH_TRANSFORMER}/*/*/*"
    for path in $product_month_paths; do
      echo "Processing product-month path $path"
      export DATA_PATH=$path
      # Run third module - pub_packager (using environment variables below as input parameters)
      python3 -m pub_packager.pub_packager_main
    done
    # Clean up after 2nd module
    rm -rf $OUT_PATH_TRANSFORMER
    # Set some environment variables for the fourth module and do some cleanup'
    export OUT_PATH=$OUT_PATH_PUBFILES
    export IN_PATH=$OUT_PATH_PACKAGER
    export LOCATION_PATH=$GROUPED_PATH
    # Run fourth module - pub_files (using environment variables below as input parameters)'
    python3 -m pub_files.main
    EOF
  env:
    # Environment variables for 2nd (part A) module: pub_workbook_loader.
    OUT_PATH_WORKBOOK: "/tmp/pub_workbooks"
    PRODUCTS: NEON.DOM.SITE.DP1.00098.001,NEON.DOM.SITE.DP1.20271.001,NEON.DOM.SITE.DP1.20046.001 # Format: NEON.DOM.SITE.DPX.XXXXX.XXX,NEON.DOM.SITE.DPX.XXXXX.XXX,etc

    # Environment variables for 2nd module (part B): pub_transformer. 
    LOG_LEVEL: INFO
    PRODUCT_INDEX: '3' # input path index of the data product identifier
    YEAR_INDEX: '4' 
    DATA_TYPE_INDEX: '8' 
    GROUP_METADATA_DIR: group # Also shared with pub_transform
    DATA_PATH_PARSE_INDEX: '2'
    OUT_PATH_TRANSFORMER: "/tmp/pub_transformer"
    WORKBOOK_PATH: "/tmp/pub_workbooks"

    # Environment variables for 3rd module: pub_packager. Also uses PRODUCT_INDEX from pub_transformer.
    OUT_PATH_PACKAGER: "/tmp/pub_packager"
    # ERR_PATH can be changed, it is user specified
    ERR_PATH_PACKAGER: "/pfs/out/packager/errored_datums"
    PUBLOC_INDEX: '7' # input path index of the pub package location (typically the site)
    DATE_INDEX: '4' # Starting index of date in path (i.e. year index)
    DATE_INDEX_LENGTH: '2' # length of date index for pub package (should be 2 for monthly)
    SORT_INDEX: '10' # File name index corresponding to date field (delimiter = .)
    
    # Environment variables for 3rd module: pub_files.
    OUT_PATH_PUBFILES: "/pfs/out"
    RELATIVE_PATH_INDEX: '3'
    DB_SECRETS_PATH: /var/db_secret
    GITHUB_PEM_PATH: /var/github_secret/key
    GITHUB_APP_ID: '300002'
    GITHUB_INSTALLATION_ID: '34765458'
    GITHUB_HOST: https://api.github.com
    GITHUB_REPO_OWNER: NEONScience
    GITHUB_README_REPO: neon-metadata-docs
    GITHUB_README_PATH: readme/template.j2
    GITHUB_EML_REPO: neon-metadata-docs
    GITHUB_EML_BOILERPLATE_PATH: eml/neon_components/NEON_EML_Boilerplate.xml
    GITHUB_EML_CONTACT_PATH: eml/neon_components/neon_contact.xml
    GITHUB_EML_INTELLECTUAL_RIGHTS_PATH: eml/neon_components/neon_intellectualRights.xml
    GITHUB_EML_UNIT_TYPES_PATH: eml/neon_components/neon_unitTypes.xml
    GITHUB_EML_UNITS_PATH: eml/neon_components/NEON_units.txt
    GITHUB_BRANCH: main
    
  secrets:
    - name: pdr-secret
      mount_path: /var/db_secret
    - name: github-neonscience-app-secret
      mount_path: /var/github_secret

input: 
  group:
  - pfs: 
      name: GROUPED_PATH
      repo: relHumidity_pub_group
      # Glob must be product-monthly or product-site-monthly (use group to do product-site-monthly)
      # path structure is e.g. DP1.00098.001/2023/04/04/CPER
      glob: /(*/*/*)/*/(*)
      group_by: $1$2
parallelism_spec:
  constant: 10
autoscaling: true
resource_requests:
  memory: 700M
  cpu: 1
resource_limits:
  memory: 1.5G
  cpu: 1.8
sidecar_resource_requests:
  memory: 800M
  cpu: 0.3
sidecar_resource_limits:
  memory: 1.5G
  cpu: 1.2
datum_set_spec:
  number: 1
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
    nodepool.neonscience.org/pipeline: "yes"
    cloud.google.com/gke-spot: "true"
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
