apiVersion: batch/v1
kind: Job
metadata:
  name: spark-on-eks
spec:
  template:
    spec:
      containers:
        - name: spark
          image: <REPO_NAME>/spark:v2.4.4
          command: [
            "/bin/sh",
            "-c",
            "/opt/spark/bin/spark-submit \
            --master k8s://https://10.100.0.1:443 \
            --deploy-mode cluster \
            --name spark-on-eks \
            --class ValueZones \
            --conf spark.executor.instances=4 \
            --conf spark.executor.memory=10G \
            --conf spark.executor.cores=2 \
            --conf spark.sql.shuffle.partitions=60 \
            --conf spark.kubernetes.container.image=<REPO_NAME>/<IMAGE_NAME>:<TAG> \
            --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
            --conf spark.kubernetes.container.image.pullPolicy=Always \
            --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
            local:///opt/spark/jars/spark-on-eks-assembly-v1.0.jar \
            \"s3a://nyc-tlc/trip data/yellow_tripdata_2018*.csv\" \
            \"s3a://nyc-tlc/trip data/green_tripdata_2018*.csv\" \
            \"s3a://nyc-tlc/misc/taxi _zone_lookup.csv\" \
            \"s3a://<YOUR_S3_BUCKET>\""
          ]
      serviceAccountName: spark
      restartPolicy: Never
  backoffLimit: 4