apiVersion: v1
kind: ConfigMap
metadata:
  name: sparkhistory-cm
  namespace: mapr-configuration-cspaces
data:
  dep-blacklist.txt: ""
  docker.properties.template: |
    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    spark.mesos.executor.docker.image: <image built from `../external/docker/spark-mesos/Dockerfile`>
    spark.mesos.executor.docker.volumes: /usr/local/lib:/host/usr/local/lib:ro
    spark.mesos.executor.home: /opt/spark
  fairscheduler.xml.template: |
    <?xml version="1.0"?>

    <!--
       Licensed to the Apache Software Foundation (ASF) under one or more
       contributor license agreements.  See the NOTICE file distributed with
       this work for additional information regarding copyright ownership.
       The ASF licenses this file to You under the Apache License, Version 2.0
       (the "License"); you may not use this file except in compliance with
       the License.  You may obtain a copy of the License at

           http://www.apache.org/licenses/LICENSE-2.0

       Unless required by applicable law or agreed to in writing, software
       distributed under the License is distributed on an "AS IS" BASIS,
       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
       See the License for the specific language governing permissions and
       limitations under the License.
    -->

    <allocations>
      <pool name="production">
        <schedulingMode>FAIR</schedulingMode>
        <weight>1</weight>
        <minShare>2</minShare>
      </pool>
      <pool name="test">
        <schedulingMode>FIFO</schedulingMode>
        <weight>2</weight>
        <minShare>3</minShare>
      </pool>
    </allocations>
  log4j.properties: |-
    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    # Set everything to be logged to the console
    log4j.rootCategory=WARN, console
    log4j.appender.console=org.apache.log4j.ConsoleAppender
    log4j.appender.console.target=System.err
    log4j.appender.console.layout=org.apache.log4j.PatternLayout
    log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n

    # Set the default spark-shell log level to WARN. When running the spark-shell, the
    # log level for this class is used to overwrite the root logger's log level, so that
    # the user can have different defaults for the shell and regular Spark apps.
    log4j.logger.org.apache.spark.repl.Main=WARN

    # Settings to quiet third party logs that are too verbose
    log4j.logger.org.spark_project.jetty=WARN
    log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
    log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
    log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
    log4j.logger.org.apache.parquet=ERROR
    log4j.logger.parquet=ERROR

    # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
    log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
    log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
    log4j.logger.org.apache.hadoop.hive.conf.HiveConf=ERROR

    # SPARK-327: Settings to suppress the unnecessary warning message from MultiMechsAuthenticationHandler
    log4j.logger.org.apache.hadoop.security.authentication.server.MultiMechsAuthenticationHandler=ERROR
    log4j.logger.org.apache.hadoop.security.authentication.server.KerberosAuthHandler=ERROR
  log4j.properties.template: |-
    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    # Set everything to be logged to the console
    log4j.rootCategory=INFO, console
    log4j.appender.console=org.apache.log4j.ConsoleAppender
    log4j.appender.console.target=System.err
    log4j.appender.console.layout=org.apache.log4j.PatternLayout
    log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n

    # Set the default spark-shell log level to WARN. When running the spark-shell, the
    # log level for this class is used to overwrite the root logger's log level, so that
    # the user can have different defaults for the shell and regular Spark apps.
    log4j.logger.org.apache.spark.repl.Main=WARN

    # Settings to quiet third party logs that are too verbose
    log4j.logger.org.spark_project.jetty=WARN
    log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
    log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
    log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
    log4j.logger.org.apache.parquet=ERROR
    log4j.logger.parquet=ERROR

    # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
    log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
    log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
    log4j.logger.org.apache.hadoop.hive.conf.HiveConf=ERROR

    # SPARK-327: Settings to suppress the unnecessary warning message from MultiMechsAuthenticationHandler
    log4j.logger.org.apache.hadoop.security.authentication.server.MultiMechsAuthenticationHandler=ERROR
    log4j.logger.org.apache.hadoop.security.authentication.server.KerberosAuthHandler=ERROR
  metrics.properties: |
    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    #  syntax: [instance].sink|source.[name].[options]=[value]

    #  This file configures Spark's internal metrics system. The metrics system is
    #  divided into instances which correspond to internal components.
    #  Each instance can be configured to report its metrics to one or more sinks.
    #  Accepted values for [instance] are "master", "worker", "executor", "driver",
    #  and "applications". A wildcard "*" can be used as an instance name, in
    #  which case all instances will inherit the supplied property.
    #
    #  Within an instance, a "source" specifies a particular set of grouped metrics.
    #  there are two kinds of sources:
    #    1. Spark internal sources, like MasterSource, WorkerSource, etc, which will
    #    collect a Spark component's internal state. Each instance is paired with a
    #    Spark source that is added automatically.
    #    2. Common sources, like JvmSource, which will collect low level state.
    #    These can be added through configuration options and are then loaded
    #    using reflection.
    #
    #  A "sink" specifies where metrics are delivered to. Each instance can be
    #  assigned one or more sinks.
    #
    #  The sink|source field specifies whether the property relates to a sink or
    #  source.
    #
    #  The [name] field specifies the name of source or sink.
    #
    #  The [options] field is the specific property of this source or sink. The
    #  source or sink is responsible for parsing this property.
    #
    #  Notes:
    #    1. To add a new sink, set the "class" option to a fully qualified class
    #    name (see examples below).
    #    2. Some sinks involve a polling period. The minimum allowed polling period
    #    is 1 second.
    #    3. Wildcard properties can be overridden by more specific properties.
    #    For example, master.sink.console.period takes precedence over
    #    *.sink.console.period.
    #    4. A metrics specific configuration
    #    "spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties" should be
    #    added to Java properties using -Dspark.metrics.conf=xxx if you want to
    #    customize metrics system. You can also put the file in ${SPARK_HOME}/conf
    #    and it will be loaded automatically.
    #    5. The MetricsServlet sink is added by default as a sink in the master,
    #    worker and driver, and you can send HTTP requests to the "/metrics/json"
    #    endpoint to get a snapshot of all the registered metrics in JSON format.
    #    For master, requests to the "/metrics/master/json" and
    #    "/metrics/applications/json" endpoints can be sent separately to get
    #    metrics snapshots of the master instance and applications. This
    #    MetricsServlet does not have to be configured.

    ## List of available common sources and their properties.

    # org.apache.spark.metrics.source.JvmSource
    #   Note: Currently, JvmSource is the only available common source.
    #         It can be added to an instance by setting the "class" option to its
    #         fully qualified class name (see examples below).

    ## List of available sinks and their properties.

    # org.apache.spark.metrics.sink.ConsoleSink
    #   Name:   Default:   Description:
    #   period  10         Poll period
    #   unit    seconds    Unit of the poll period

    # org.apache.spark.metrics.sink.CSVSink
    #   Name:     Default:   Description:
    #   period    10         Poll period
    #   unit      seconds    Unit of the poll period
    #   directory /tmp       Where to store CSV files

    # org.apache.spark.metrics.sink.GangliaSink
    #   Name:     Default:   Description:
    #   host      NONE       Hostname or multicast group of the Ganglia server,
    #                        must be set
    #   port      NONE       Port of the Ganglia server(s), must be set
    #   period    10         Poll period
    #   unit      seconds    Unit of the poll period
    #   ttl       1          TTL of messages sent by Ganglia
    #   dmax      0          Lifetime in seconds of metrics (0 never expired)
    #   mode      multicast  Ganglia network mode ('unicast' or 'multicast')

    # org.apache.spark.metrics.sink.JmxSink

    # org.apache.spark.metrics.sink.MetricsServlet
    #   Name:     Default:   Description:
    #   path      VARIES*    Path prefix from the web server root
    #   sample    false      Whether to show entire set of samples for histograms
    #                        ('false' or 'true')
    #
    # * Default path is /metrics/json for all instances except the master. The
    #   master has two paths:
    #     /metrics/applications/json # App information
    #     /metrics/master/json       # Master information

    # org.apache.spark.metrics.sink.GraphiteSink
    #   Name:     Default:      Description:
    #   host      NONE          Hostname of the Graphite server, must be set
    #   port      NONE          Port of the Graphite server, must be set
    #   period    10            Poll period
    #   unit      seconds       Unit of the poll period
    #   prefix    EMPTY STRING  Prefix to prepend to every metric's name
    #   protocol  tcp           Protocol ("tcp" or "udp") to use

    # org.apache.spark.metrics.sink.StatsdSink
    #   Name:     Default:      Description:
    #   host      127.0.0.1     Hostname or IP of StatsD server
    #   port      8125          Port of StatsD server
    #   period    10            Poll period
    #   unit      seconds       Units of poll period
    #   prefix    EMPTY STRING  Prefix to prepend to metric name

    ## Examples
    # Enable JmxSink for all instances by class name
    *.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink

    # Enable ConsoleSink for all instances by class name
    #*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink

    # Enable StatsdSink for all instances by class name
    #*.sink.statsd.class=org.apache.spark.metrics.sink.StatsdSink
    #*.sink.statsd.prefix=spark

    # Polling period for the ConsoleSink
    #*.sink.console.period=10
    # Unit of the polling period for the ConsoleSink
    #*.sink.console.unit=seconds

    # Polling period for the ConsoleSink specific for the master instance
    #master.sink.console.period=15
    # Unit of the polling period for the ConsoleSink specific for the master
    # instance
    #master.sink.console.unit=seconds

    # Enable CsvSink for all instances by class name
    #*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink

    # Polling period for the CsvSink
    #*.sink.csv.period=1
    # Unit of the polling period for the CsvSink
    #*.sink.csv.unit=minutes

    # Polling directory for CsvSink
    #*.sink.csv.directory=/tmp/

    # Polling period for the CsvSink specific for the worker instance
    #worker.sink.csv.period=10
    # Unit of the polling period for the CsvSink specific for the worker instance
    #worker.sink.csv.unit=minutes

    # Enable Slf4jSink for all instances by class name
    #*.sink.slf4j.class=org.apache.spark.metrics.sink.Slf4jSink

    # Polling period for the Slf4JSink
    #*.sink.slf4j.period=1
    # Unit of the polling period for the Slf4jSink
    #*.sink.slf4j.unit=minutes

    # Enable JvmSource for instance master, worker, driver and executor
    master.source.jvm.class=org.apache.spark.metrics.source.JvmSource

    worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource

    driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource

    executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource
  metrics.properties.template: |
    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    #  syntax: [instance].sink|source.[name].[options]=[value]

    #  This file configures Spark's internal metrics system. The metrics system is
    #  divided into instances which correspond to internal components.
    #  Each instance can be configured to report its metrics to one or more sinks.
    #  Accepted values for [instance] are "master", "worker", "executor", "driver",
    #  and "applications". A wildcard "*" can be used as an instance name, in
    #  which case all instances will inherit the supplied property.
    #
    #  Within an instance, a "source" specifies a particular set of grouped metrics.
    #  there are two kinds of sources:
    #    1. Spark internal sources, like MasterSource, WorkerSource, etc, which will
    #    collect a Spark component's internal state. Each instance is paired with a
    #    Spark source that is added automatically.
    #    2. Common sources, like JvmSource, which will collect low level state.
    #    These can be added through configuration options and are then loaded
    #    using reflection.
    #
    #  A "sink" specifies where metrics are delivered to. Each instance can be
    #  assigned one or more sinks.
    #
    #  The sink|source field specifies whether the property relates to a sink or
    #  source.
    #
    #  The [name] field specifies the name of source or sink.
    #
    #  The [options] field is the specific property of this source or sink. The
    #  source or sink is responsible for parsing this property.
    #
    #  Notes:
    #    1. To add a new sink, set the "class" option to a fully qualified class
    #    name (see examples below).
    #    2. Some sinks involve a polling period. The minimum allowed polling period
    #    is 1 second.
    #    3. Wildcard properties can be overridden by more specific properties.
    #    For example, master.sink.console.period takes precedence over
    #    *.sink.console.period.
    #    4. A metrics specific configuration
    #    "spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties" should be
    #    added to Java properties using -Dspark.metrics.conf=xxx if you want to
    #    customize metrics system. You can also put the file in ${SPARK_HOME}/conf
    #    and it will be loaded automatically.
    #    5. The MetricsServlet sink is added by default as a sink in the master,
    #    worker and driver, and you can send HTTP requests to the "/metrics/json"
    #    endpoint to get a snapshot of all the registered metrics in JSON format.
    #    For master, requests to the "/metrics/master/json" and
    #    "/metrics/applications/json" endpoints can be sent separately to get
    #    metrics snapshots of the master instance and applications. This
    #    MetricsServlet does not have to be configured.

    ## List of available common sources and their properties.

    # org.apache.spark.metrics.source.JvmSource
    #   Note: Currently, JvmSource is the only available common source.
    #         It can be added to an instance by setting the "class" option to its
    #         fully qualified class name (see examples below).

    ## List of available sinks and their properties.

    # org.apache.spark.metrics.sink.ConsoleSink
    #   Name:   Default:   Description:
    #   period  10         Poll period
    #   unit    seconds    Unit of the poll period

    # org.apache.spark.metrics.sink.CSVSink
    #   Name:     Default:   Description:
    #   period    10         Poll period
    #   unit      seconds    Unit of the poll period
    #   directory /tmp       Where to store CSV files

    # org.apache.spark.metrics.sink.GangliaSink
    #   Name:     Default:   Description:
    #   host      NONE       Hostname or multicast group of the Ganglia server,
    #                        must be set
    #   port      NONE       Port of the Ganglia server(s), must be set
    #   period    10         Poll period
    #   unit      seconds    Unit of the poll period
    #   ttl       1          TTL of messages sent by Ganglia
    #   dmax      0          Lifetime in seconds of metrics (0 never expired)
    #   mode      multicast  Ganglia network mode ('unicast' or 'multicast')

    # org.apache.spark.metrics.sink.JmxSink

    # org.apache.spark.metrics.sink.MetricsServlet
    #   Name:     Default:   Description:
    #   path      VARIES*    Path prefix from the web server root
    #   sample    false      Whether to show entire set of samples for histograms
    #                        ('false' or 'true')
    #
    # * Default path is /metrics/json for all instances except the master. The
    #   master has two paths:
    #     /metrics/applications/json # App information
    #     /metrics/master/json       # Master information

    # org.apache.spark.metrics.sink.GraphiteSink
    #   Name:     Default:      Description:
    #   host      NONE          Hostname of the Graphite server, must be set
    #   port      NONE          Port of the Graphite server, must be set
    #   period    10            Poll period
    #   unit      seconds       Unit of the poll period
    #   prefix    EMPTY STRING  Prefix to prepend to every metric's name
    #   protocol  tcp           Protocol ("tcp" or "udp") to use

    # org.apache.spark.metrics.sink.StatsdSink
    #   Name:     Default:      Description:
    #   host      127.0.0.1     Hostname or IP of StatsD server
    #   port      8125          Port of StatsD server
    #   period    10            Poll period
    #   unit      seconds       Units of poll period
    #   prefix    EMPTY STRING  Prefix to prepend to metric name

    ## Examples
    # Enable JmxSink for all instances by class name
    *.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink

    # Enable ConsoleSink for all instances by class name
    #*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink

    # Enable StatsdSink for all instances by class name
    #*.sink.statsd.class=org.apache.spark.metrics.sink.StatsdSink
    #*.sink.statsd.prefix=spark

    # Polling period for the ConsoleSink
    #*.sink.console.period=10
    # Unit of the polling period for the ConsoleSink
    #*.sink.console.unit=seconds

    # Polling period for the ConsoleSink specific for the master instance
    #master.sink.console.period=15
    # Unit of the polling period for the ConsoleSink specific for the master
    # instance
    #master.sink.console.unit=seconds

    # Enable CsvSink for all instances by class name
    #*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink

    # Polling period for the CsvSink
    #*.sink.csv.period=1
    # Unit of the polling period for the CsvSink
    #*.sink.csv.unit=minutes

    # Polling directory for CsvSink
    #*.sink.csv.directory=/tmp/

    # Polling period for the CsvSink specific for the worker instance
    #worker.sink.csv.period=10
    # Unit of the polling period for the CsvSink specific for the worker instance
    #worker.sink.csv.unit=minutes

    # Enable Slf4jSink for all instances by class name
    #*.sink.slf4j.class=org.apache.spark.metrics.sink.Slf4jSink

    # Polling period for the Slf4JSink
    #*.sink.slf4j.period=1
    # Unit of the polling period for the Slf4jSink
    #*.sink.slf4j.unit=minutes

    # Enable JvmSource for instance master, worker, driver and executor
    master.source.jvm.class=org.apache.spark.metrics.source.JvmSource

    worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource

    driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource

    executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource
  slaves.template: |-
    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    # A Spark Worker will be started on each of the machines listed below.
    localhost
  spark-defaults.conf: |
    # Default system properties included when running spark-submit.
    # This is useful for setting default environmental settings.

    # Log effective Spark configuration at startup on INFO level
    spark.logConf                      true

    # Enable event logs for HistoryServer
    spark.eventLog.enabled             true
    spark.eventLog.dir                 maprfs:///apps/spark
    spark.history.fs.logDirectory      maprfs:///apps/spark

    # Default location for Warehouse, if not using Hive
    spark.sql.warehouse.dir            maprfs:///user/${system:user.name}/spark-warehouse

    # Fix for SPARK-7819
    spark.sql.hive.metastore.sharedPrefixes  com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,oracle.jdbc,com.mapr.fs.shim.LibraryLoader,com.mapr.security.JNISecurity,com.mapr.fs.jni,com.mapr.fs.ShimLoader

    spark.executor.memory              2g

    spark.yarn.historyServer.address spark-hs-container-d46d4fd9d-zvnlf.spark-hs-container-svc.internaltenant.svc.cluster.local:18480
    spark.history.ui.port 18080
    # SECURITY BLOCK
    # ALL SECURITY PROPERTIES MUST BE PLACED IN THIS BLOCK

    # ssl
    spark.ssl.historyServer.port 18480
    spark.ssl.enabled true
    spark.ssl.ui.enabled false
    spark.ssl.fs.enabled true
    spark.ssl.trustStore /opt/mapr/conf/ssl_truststore
    spark.ssl.keyStore /opt/mapr/conf/ssl_keystore
    spark.ssl.protocol TLSv1.2

    # - PAM
    spark.ui.filters  org.apache.spark.deploy.yarn.YarnProxyRedirectFilter, org.apache.spark.ui.filters.MultiauthWebUiFilter

    # - ACLS
    spark.acls.enable       false
    spark.admin.acls        mapr
    spark.admin.acls.groups mapr
    spark.ui.view.acls      mapruser1

    # - Authorization and Network Encryption
    spark.authenticate      true
    # - - This secret will be used only by local/standalone modes. YARN will override this with its own secret
    spark.authenticate.secret       changeMe
    spark.authenticate.enableSaslEncryption true
    spark.network.sasl.serverAlwaysEncrypt  true
    # - IO Encryption
    spark.io.encryption.enabled     true
    spark.io.encryption.keySizeBits 128

    # END OF THE SECURITY CONFIGURATION BLOCK
  spark-defaults.conf.template: |
    # Default system properties included when running spark-submit.
    # This is useful for setting default environmental settings.

    # Log effective Spark configuration at startup on INFO level
    spark.logConf                      true

    # Enable event logs for HistoryServer
    spark.eventLog.enabled             true
    spark.eventLog.dir                 maprfs:///apps/spark
    spark.history.fs.logDirectory      maprfs:///apps/spark

    # Default location for Warehouse, if not using Hive
    spark.sql.warehouse.dir            maprfs:///user/${system:user.name}/spark-warehouse

    # Fix for SPARK-7819
    spark.sql.hive.metastore.sharedPrefixes  com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,oracle.jdbc,com.mapr.fs.shim.LibraryLoader,com.mapr.security.JNISecurity,com.mapr.fs.jni,com.mapr.fs.ShimLoader

    spark.executor.memory              2g

    spark.ui.filters  org.apache.spark.deploy.yarn.YarnProxyRedirectFilter
  spark-env.cmd.template: |
    @echo off
    set HADOOP_HOME=%MAPR_HOME%\hadoop\hadoop-2.7.0
    set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop
    set PATH=%HADOOP_HOME%\bin;%PATH%

    for /f "delims=" %%A in ('%HADOOP_HOME%\bin\hadoop classpath') do set "TMP_DIST_PATH=%%A"
    set SPARK_DIST_CLASSPATH=%TMP_DIST_PATH:C:=%
    set SPARK_DIST_CLASSPATH=%SPARK_DIST_CLASSPATH:\=/%
    set SPARK_DIST_CLASSPATH=%SPARK_DIST_CLASSPATH%;%TMP_DIST_PATH%
  spark-env.sh: |
    #!/usr/bin/env bash

    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    # This file is sourced when running various Spark programs.
    # Copy it as spark-env.sh and edit that to configure Spark for your site.

    # Options read when launching programs locally with
    # ./bin/run-example or ./bin/spark-submit
    # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
    # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
    # - SPARK_PUBLIC_DNS, to set the public dns name of the driver program

    # Options read by executors and drivers running inside the cluster
    # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
    # - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program
    # - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data
    # - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos

    # Options read in YARN client/cluster mode
    # - SPARK_CONF_DIR, Alternate conf dir. (Default: ${SPARK_HOME}/conf)
    # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
    # - YARN_CONF_DIR, to point Spark towards YARN configuration files when you use YARN
    # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1).
    # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G)
    # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G)

    # Options for the daemons used in the standalone deploy mode
    # - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname
    # - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master
    # - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y")
    # - SPARK_WORKER_CORES, to set the number of cores to use on this machine
    # - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)
    # - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
    # - SPARK_WORKER_DIR, to set the working directory of worker processes
    # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
    # - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).
    # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
    # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
    # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
    # - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons
    # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers

    # Generic options for the daemons used in the standalone deploy mode
    # - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)
    # - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)
    # - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)
    # - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)
    # - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)
    # - SPARK_NO_DAEMONIZE  Run the proposed command in the foreground. It will not output a PID file.
    # Options for native BLAS, like Intel MKL, OpenBLAS, and so on.
    # You might get better performance to enable these options if using native BLAS (see SPARK-21305).
    # - MKL_NUM_THREADS=1        Disable multi-threading of Intel MKL
    # - OPENBLAS_NUM_THREADS=1   Disable multi-threading of OpenBLAS

    #########################################################################################################
    # Set MapR attributes and compute classpath
    #########################################################################################################

    # Set the spark attributes
    if [ -d "/opt/mapr/spark/spark-2.4.4" ]; then
            export SPARK_HOME=/opt/mapr/spark/spark-2.4.4
    fi

    # Load the hadoop version attributes
    source /opt/mapr/spark/spark-2.4.4/mapr-util/hadoop-version-picker.sh
    export HADOOP_HOME=$hadoop_home_dir
    export HADOOP_CONF_DIR=$hadoop_conf_dir

    # Enable mapr impersonation
    export MAPR_IMPERSONATION_ENABLED=1

    MAPR_HADOOP_CLASSPATH=`/opt/mapr/spark/spark-2.4.4/bin/mapr-classpath.sh`
    MAPR_HADOOP_JNI_PATH=`hadoop jnipath`
    MAPR_SPARK_CLASSPATH="$MAPR_HADOOP_CLASSPATH"

    SPARK_MAPR_HOME=/opt/mapr

    export SPARK_LIBRARY_PATH=$MAPR_HADOOP_JNI_PATH
    export LD_LIBRARY_PATH="$MAPR_HADOOP_JNI_PATH:$LD_LIBRARY_PATH"

    # Load the classpath generator script
    source /opt/mapr/spark/spark-2.4.4/mapr-util/generate-classpath.sh

    # Calculate hive jars to include in classpath
    generate_compatible_classpath "spark" "2.4.4" "hive"
    MAPR_HIVE_CLASSPATH=${generated_classpath}
    if [ ! -z "$MAPR_HIVE_CLASSPATH" ]; then
      MAPR_SPARK_CLASSPATH="$MAPR_SPARK_CLASSPATH:$MAPR_HIVE_CLASSPATH"
    fi

    # Calculate hbase jars to include in classpath
    generate_compatible_classpath "spark" "2.4.4" "hbase"
    MAPR_HBASE_CLASSPATH=${generated_classpath}
    if [ ! -z "$MAPR_HBASE_CLASSPATH" ]; then
      MAPR_SPARK_CLASSPATH="$MAPR_SPARK_CLASSPATH:$MAPR_HBASE_CLASSPATH"
      SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Dspark.driver.extraClassPath=$MAPR_HBASE_CLASSPATH"
    fi

    # Set executor classpath for MESOS. Uncomment following string if you want deploy spark jobs on Mesos
    #MAPR_MESOS_CLASSPATH=$MAPR_SPARK_CLASSPATH
    SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Dspark.executor.extraClassPath=$MAPR_HBASE_CLASSPATH:$MAPR_MESOS_CLASSPATH"

    # Set SPARK_DIST_CLASSPATH
    export SPARK_DIST_CLASSPATH=$MAPR_SPARK_CLASSPATH

    # Security status
    source /opt/mapr/conf/env.sh
    if [ "$MAPR_SECURITY_STATUS" = "true" ]; then
      SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Dhadoop.login=hybrid -Dmapr_sec_enabled=true -Djavax.security.auth.useSubjectCredsOnly=false"
    fi

    # scala
    export SCALA_VERSION=2.11
    export SPARK_SCALA_VERSION=$SCALA_VERSION
    export SCALA_HOME=/opt/mapr/spark/spark-2.4.4/scala
    export SCALA_LIBRARY_PATH=$SCALA_HOME/lib

    # Use a fixed identifier for pid files
    export SPARK_IDENT_STRING="mapr"

    #########################################################################################################
    #    :::CAUTION::: DO NOT EDIT ANYTHING ON OR ABOVE THIS LINE
    #########################################################################################################


    #
    # MASTER HA SETTINGS
    #
    #export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER  -Dspark.deploy.zookeeper.url=<zookeerper1:5181,zookeeper2:5181,..> -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false"


    # MEMORY SETTINGS
    export SPARK_DAEMON_MEMORY=1g
    export SPARK_WORKER_MEMORY=16g

    # Worker Directory
    export SPARK_WORKER_DIR=$SPARK_HOME/tmp

    # Environment variable for printing spark command everytime you run spark.Set to "1" to print.
    # export SPARK_PRINT_LAUNCH_COMMAND=1

    #UI
    export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djava.library.path=$SPARK_MAPR_HOME/lib"


    export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=maprfs:///apps/spark/$POD_NAMESPACE -Djava.library.path=$SPARK_MAPR_HOME/lib"
  spark-env.sh.template: |
    #!/usr/bin/env bash

    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    # This file is sourced when running various Spark programs.
    # Copy it as spark-env.sh and edit that to configure Spark for your site.

    # Options read when launching programs locally with
    # ./bin/run-example or ./bin/spark-submit
    # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
    # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
    # - SPARK_PUBLIC_DNS, to set the public dns name of the driver program

    # Options read by executors and drivers running inside the cluster
    # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
    # - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program
    # - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data
    # - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos

    # Options read in YARN client/cluster mode
    # - SPARK_CONF_DIR, Alternate conf dir. (Default: ${SPARK_HOME}/conf)
    # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
    # - YARN_CONF_DIR, to point Spark towards YARN configuration files when you use YARN
    # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1).
    # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G)
    # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G)

    # Options for the daemons used in the standalone deploy mode
    # - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname
    # - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master
    # - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y")
    # - SPARK_WORKER_CORES, to set the number of cores to use on this machine
    # - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)
    # - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
    # - SPARK_WORKER_DIR, to set the working directory of worker processes
    # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
    # - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).
    # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
    # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
    # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
    # - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons
    # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers

    # Generic options for the daemons used in the standalone deploy mode
    # - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)
    # - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)
    # - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)
    # - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)
    # - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)
    # - SPARK_NO_DAEMONIZE  Run the proposed command in the foreground. It will not output a PID file.
    # Options for native BLAS, like Intel MKL, OpenBLAS, and so on.
    # You might get better performance to enable these options if using native BLAS (see SPARK-21305).
    # - MKL_NUM_THREADS=1        Disable multi-threading of Intel MKL
    # - OPENBLAS_NUM_THREADS=1   Disable multi-threading of OpenBLAS
    export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=maprfs:///apps/spark -Djava.library.path=$SPARK_MAPR_HOME/lib"
  spark-jars.sh: |
    #!/bin/bash

    SPARK_INSTALLATION_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
    SPARK_JARS_CLASSPATH=$(find $SPARK_INSTALLATION_DIRECTORY/jars -name '*.jar' -printf '%p:' | sed 's/:$//')
