---
# CS4287-5287
# Author: Aniruddha Gokhale
# Created: Spring 2021
#
# For assignment #4
# this is the deployment pod for Spark workers
apiVersion: apps/v1
kind: Deployment         # We are testing the Deployment resource
metadata:
  name: spark-worker-deploy  # This will run the Spark worker
spec:                     # This is the specification where we can even put the number of replicas
  replicas: 5             # we run 5 workers
  selector:
    matchLabels:
      app: sparkWorkerApp          # Basically this is like the search string used to locate the pods
  minReadySeconds: 5  # if anything crashes before 5 secs, the deployment is not
                          # considered as ready and available. Default value is 0
  template:               # Specified info needed to run the pod and what runs in the pod
    metadata:
      labels:
        app: sparkWorkerApp        # some label to give to this pod (see the matching label above)
    spec:                 # actual specification
      hostname: spark-worker-host  # we need it to set the SPARK_LOCAL_IP
      containers:
        - name: spark-worker       # used to name container
          image: 129.114.25.80:5000/my-spark:latest   # this is the image in private registry
          ports:            # Spark worker port
            - containerPort: 7078  # worker will use this port instead of random port
            - containerPort: 7079  # used by block manager
            - containerPort: 8081  # GUI
          env:  # environment variables to pass

            - name: SPARK_LOCAL_IP
              value: "spark-worker-host"  # floating IP of your spark master machine

            - name: SPARK_NO_DAEMONIZE # so that the master runs in foreground
              value: "1"

            # the SPARK_HOME env set in docker image is not accessible for the command line
            # below. So had to set it here.
            - name: SPARK_HOME  
              value: "/spark-3.1.1-bin-hadoop3.2"

          imagePullPolicy: Always  # This forces the node to pull the image
          command: ["$(SPARK_HOME)/sbin/start-worker.sh"]
          args: ["spark://spark-master-svc:7077", "--properties-file", "$(SPARK_HOME)/conf/spark-worker.conf"]
...
