# This file defines a daemonset which runs automatically on all
# kubernetes nodes. It is used like so: kubectl create -f <this file_path>.
# The daemonset does the following:
#   - waits until nvidia-smi is mounted and available on PATH
#   - enables persistence mode on the nvidia driver
#   - allows all users to set the GPU clock speed
# In effect, this allows pods created without a privileged security context to
# set the GPU clock speeds
# This daemonset config does not define GPU resources, because otherwise it
# would consume them, leaving them unavailable to pods. Instead, it runs in
# privileged mode (so it can see all GPUs), and manually mounts the CUDA
# lib and bin directories.

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nvidia-add-unrestricted-permissions-dameon-set
spec:
  selector:
    matchLabels:
      name: nvidia-add-unrestricted-permissions
  template:
    metadata:
      labels:
        name: nvidia-add-unrestricted-permissions
    spec:
      containers:
      - name: nvidia-add-unrestricted-permissions
        image: nvidia/cuda:9.0-devel-ubuntu16.04
        securityContext:
          privileged: true
        command: [ "/bin/bash", "-c", "export PATH=$PATH:/usr/local/bin/nvidia/ && while [ ! $(type -p nvidia-smi) ]; do echo waiting for nvidia-smi to mount...; sleep 2; done && nvidia-smi -pm 1 && nvidia-smi --applications-clocks-permission=UNRESTRICTED && nvidia-smi --auto-boost-permission=UNRESTRICTED && tail -f /dev/null" ]
        volumeMounts:
          - name: nvidia-debug-tools
            mountPath: /usr/local/bin/nvidia
          - name: nvidia-libraries
            mountPath: /usr/local/nvidia/lib64
      volumes:
        - name: nvidia-debug-tools
          hostPath:
            path: /home/kubernetes/bin/nvidia/bin
        - name: nvidia-libraries
          hostPath:
            path: /home/kubernetes/bin/nvidia/lib64
