pipeline:
  name: pluvio_data_parser
transform:
  image_pull_secrets:
    - battelleecology-quay-read-all-pull-secret
  image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:sha-9c8f0d0
  cmd:
    - /bin/bash
  stdin:
    - '#!/bin/bash'
    - python3 -m pluvio_data_parser.pluvio_data_parser_main
  env:
    PARSED_START_INDEX: "3"
    PARSED_END_INDEX: "16"
    LOG_LEVEL: DEBUG 
    OUT_PATH: /pfs/out 
    PARSE_FIELD: serial_output
    RELATIVE_PATH_INDEX: "4"
    PARSED_SCHEMA_PATH: /usr/src/app/schemas/pluvio/pluvio_parsed.avsc 
    SOURCE_TYPE: 'pluvio_raw'
input:
  pfs:
    name: DATA_PATH
    repo: pluvio_data_source_kafka
    glob: /pluvio_raw/*/*/*
parallelism_spec:
  constant: 10
autoscaling: true
resource_requests:
  memory: 500M
  cpu: 0.5
resource_limits:
  memory: 1G
  cpu: 1.5
sidecar_resource_requests:
  memory: 2G
  cpu: 0.5
datum_set_spec:
  number: 1
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: 'true'
    nodepool.neonscience.org/pipeline: 'yes'
    cloud.google.com/gke-spot: 'true'
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
