---
pipeline:
  name: data_source_dualfan_site
transform:
  image_pull_secrets:
  - battelleecology-quay-read-all-pull-secret
  image: quay.io/battelleecology/genavro:v1.0.73
  cmd:
  - bash
  stdin:
  - "#!/usr/bin/env bash"
  - "# Split data source path"
  - echo "Checking $presto_data_source files"
  - for path in $(find -L $import_trigger -type f); do
  - '  echo "Processing $path"'
  - "  p=${path#/pfs}"
  - '  IFS="/"; arr=($p); unset IFS;'
  - "  year=${arr[2]}"
  - "  month=${arr[3]}"
  - "  day=${arr[4]}"
  - "  site=${arr[5]}"
  - "  export GEN_DATE=$year-$month-$day"
  - "  export GEN_SITE_NAME=$site"
  - "  export GEN_OUTPUT_DIR=/pfs/out/dualfan/$year/$month/$day"
  - "  export REQUESTS_CA_BUNDLE=/etc/pki/tls/cert.pem"
  - "  mkdir -p $GEN_OUTPUT_DIR"
  - "  /usr/src/app/genscript/genparquet.py --storesitename --codec gzip --truncperiod
    second"
  - done
  env:
    GEN_YAML_CONF: "/usr/src/app/genscript/configs/dualfan_streams.yaml"
    GEN_SCHEMA_FILE: "/usr/src/app/schemas/dualfan/dualfan.avsc"
    LOG_LEVEL: info
    REQUESTS_CA_BUNDLE: "/etc/pki/tls/cert.pem"
  secrets:
  - name: pachd-trino-secret
    key: TRINO_HOST
    env_var: PRESTO_HOST
  - name: pachd-trino-secret
    key: TRINO_PASSWORD
    env_var: PRESTO_PASSWORD
  - name: pachd-trino-secret
    key: TRINO_USER
    env_var: PRESTO_USER
input:
  pfs:
    name: import_trigger
    repo: import_trigger_dualfan
    glob: "/*/*/*"
output_branch: master
parallelism_spec:
  constant: 4
resource_requests:
  memory: 1G
  cpu: 0.7
autoscaling: true
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
    nodepool.neonscience.org/pipeline: "yes"
    cloud.google.com/gke-spot: "true"
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
pod_patch: |-
  [
    { "op": "replace",
      "path":"/containers/1/resources/requests/memory",
      "value":"2G"
    },
    { "op": "replace",
      "path": "/containers/1/resources/requests/cpu",
      "value": "0.5"
    }
  ]
