apiVersion: v1
kind: ConfigMap
metadata:
  #namespace: monitoring
  name: spark-worker-config
data:
  spark-defaults.conf: |
    spark.blockManager.port {{ .Values.spark_worker.config.ports.blockManager }}
    spark.broadcast.port {{ .Values.spark_worker.config.ports.broadcast }}
    spark.cleaner.ttl 900
    spark.cores.max 1
    spark.driver.extraClassPath {{ .Values.spark_worker.config.driver.extraClassPath }}
    spark.driver.extraJavaOptions -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/tmp/gc_driver.log -Xmx512m
    spark.driver.memory 512m
    spark.driver.port {{ .Values.spark_worker.config.ports.driver }}
    spark.eventLog.dir {{ .Values.spark_worker.config.eventLog_dir }}
    spark.executor.cores 1
    spark.executor.extraClassPath {{ .Values.spark_worker.config.executor.extraClassPath }}
    spark.executor.extraJavaOptions -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/tmp/gc_executor.log
    spark.executor.memory 512m
    spark.executor.port {{ .Values.spark_worker.config.ports.executor }}
    spark.fileserver.port {{ .Values.spark_worker.config.ports.fileserver }}
    spark.history.fs.logDirectory {{ .Values.spark_worker.config.history_fs_logDirectory }}
    spark.history.ui.port {{ .Values.spark_worker.config.ports.history_ui }}
    spark.python.worker.memory 16m
    spark.speculation {{ .Values.spark_worker.config.speculation }}
    spark.speculation.interval {{ .Values.spark_worker.config.speculation_interval }}
    spark.sql.shuffle.partitions 32
    spark.sql.ui.retainedExecutions 10
    spark.streaming.ui.retainedBatches 10
    spark.ui.retainedJobs 10
    spark.ui.retainedStages 10
    spark.worker.cleanup.enabled True
    spark.worker.ui.retainedDrivers 10
    spark.worker.ui.retainedExecutors 10


  spark-env.sh: |
    #!/usr/bin/env bash
    export SPARK_WORKER_PORT={{ .Values.spark_worker.config.ports.worker }}
    export SPARK_WORKER_WEBUI_PORT={{ .Values.spark_worker.config.ports.worker_web_ui }}
    export SPARK_WORKER_DIR={{ .Values.spark_worker.config.worker_dir }}
    export SPARK_WORKER_MEMORY={{ .Values.spark_worker.config.worker_memory }}
    export SPARK_WORKER_CORES={{ .Values.spark_worker.config.worker_cores }}
    export SPARK_LOG_DIR={{ .Values.spark_worker.config.log_dir }}
    export SPARK_DAEMON_JAVA_OPTS={{ .Values.spark_worker.config.daemon_java_opts }}
  start_spark_worker.sh: |
    #!/usr/bin/env bash
    . /etc/config/spark-env.sh
    export EXEC_CLASS=org.apache.spark.deploy.worker.Worker
    export INSTANCE_ID=1
    export SPARK_CLASSPATH=/opt/spark/current/conf/:/opt/spark/current/lib/spark-assembly-1.6.1-hadoop2.6.0.jar:/opt/spark/current/lib/datanucleus-core-3.2.10.jar:/opt/spark/current/lib/datanucleus-rdbms-3.2.9.jar:/opt/spark/current/lib/datanucleus-api-jdo-3.2.6.jar
    export log="$SPARK_LOG_DIR/spark-spark-"$EXEC_CLASS"-"$INSTANCE_ID"-${SPARK_LOCAL_IP}.out"
    export SPARK_HOME=/opt/spark/current

    /usr/bin/java -cp $SPARK_CLASSPATH $SPARK_DAEMON_JAVA_OPTS -Xms1g -Xmx1g "$EXEC_CLASS" --webui-port "$SPARK_WORKER_WEBUI_PORT" --port $SPARK_WORKER_PORT spark-master-svc:7077 2>&1 | tee $log

