apiVersion: batch/v1
kind: Job
metadata:
  name: geonames
  namespace: airflow-tls
  labels:
    name: geonames
    instance: geonames-job
    version: "1.0.0"
    component: batch
    part-of: jobs
spec:
  activeDeadlineSeconds: 1800  # must finish before this (30 min)
  ttlSecondsAfterFinished: 180  # deleted after this (3 min)
  backoffLimit: 3  # retries (up to 6), uses exponential back-off (10s, 20s, 40s, ...300s)
  # See https://kubernetes.io/docs/concepts/workloads/controllers/job/#parallel-jobs
  # See https://kubernetes.io/docs/concepts/workloads/controllers/job/#job-patterns
  completions: 10  # if none, then first completion signals success
  parallelism: 5  # 1 to many instances, only 1 if completions set to 1
  template:
    metadata:
      labels:
        name: geonames
        instance: geonames-job
        version: "1.0.0"
        component: batch
        part-of: jobs
    spec:
      restartPolicy: Never  # OnFailure
      nodeSelector:
        "beta.kubernetes.io/os": linux
      tolerations:
        - key: "kubernetes.azure.com/scalesetpriority"
          operator: "Equal"
          value: "spot"
          effect: "NoSchedule"
      affinity:
        nodeAffinity:
          # requiredDuringSchedulingIgnoredDuringExecution:
          #   nodeSelectorTerms:
          #   - matchExpressions:
          #     - key: kubernetes.azure.com/scalesetpriority
          #       operator: In
          #       values:
          #       - spot
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            preference:
              matchExpressions:
              - key: kubernetes.azure.com/scalesetpriority
                operator: In
                values:
                - spot            
      containers:
        - name: geonames
          image: rkoH1pVL.azurecr.io/geonames:latest
          imagePullPolicy: Always  # IfNotPresent
          env:
            - name: APP_LOG_KEY
              value: "rko.aks.geonames2lake.dev"
            - name: LOG_LEVEL
              value: "DEBUG"
            - name: ON_QUEUE
              value: "True"
            - name: QUEUE_NAME
              value: "geonames-tasks"
            - name: CENSUS_STATES_URI
              value: "https://www2.census.gov/programs-surveys/popest/geographies/2019/state-geocodes-v2019.xlsx"
            - name: CENSUS_COUNTIES_URI
              value: "https://www2.census.gov/geo/docs/maps-data/data/gazetteer/2019_Gazetteer/2019_Gaz_counties_national.zip"
            - name: GEONAMES_ZIPCODES_URI
              value: "http://download.geonames.org/export/zip/US.zip"
            - name: CENSUS_STATE_NAME
              value: "MasterData.StateTerritory"
            - name: CENSUS_COUNTY_NAME
              value: "MasterData.CountyProvince"
            - name: GEONAMES_ZIPCODE_NAME
              value: "MasterData.ZipCode"
            - name: LAKE_ACCOUNT_NAME
              value: "airflowstoragesandbox"
            - name: LAKE_ACCOUNT_KEY
              valueFrom:
                secretKeyRef:
                  name: az-file-secret
                  key: azurestorageaccountkey
            - name: STORE_CONNECTION_STRING
              valueFrom:
                secretKeyRef:
                  name: az-queue-secret
                  key: azure-queue-connectionstring
            - name: LAKE_CONTAINER_NAME
              value: "enterprisedata"
            - name: LAKE_BASE_PATH
              value: "Raw/Master Data/Geography/Brightspring"
          resources:
            requests:
              cpu: "500m"
              memory: "500Mi"
            limits:
              cpu: "2000m"
              memory: "2Gi"
