apiVersion: batch/v1
kind: Job
metadata:
  name: training
  namespace: sugiyama-demo
spec:
  # completions: 6
  # parallelism: 6
  template:
    spec:
      restartPolicy: Never
      initContainers:
      - name: get-code
        image: alpine/git
        command:
        - git
        - clone
        - https://github.com/IMOKURI/image-competition-baseline.git
        - /working
        volumeMounts:
        - mountPath: /working
          name: working
      containers:
      - name: training
        image: quay.io/yoshio_sugiyama/torch-image-baseline:1.9.0
        # imagePullPolicy: Always
        command:
        - python
        - train.py
        # - settings.debug=True
        resources:
          limits:
            nvidia.com/gpu: 1
            # nvidia.com/mig-3g.20gb: 1
        env:
          # - name: CUDA_VISIBLE_DEVICES
          #   value: "7"
          - name: XDG_CACHE_HOME
            value: /app/cache/.cache
          - name: XDG_CONFIG_HOME
            value: /app/cache/.config
          - name: http_proxy
            value: ""
          - name: https_proxy
            value: ""
        volumeMounts:
        - mountPath: /app/inputs
          name: nfs
          subPath: cassava/inputs
        - mountPath: /app/outputs
          name: nfs
          subPath: cassava/outputs
        - mountPath: /app/cache
          name: nfs
          subPath: cassava/cache
        - mountPath: /app/working
          name: working
        # https://github.com/kubernetes/kubernetes/issues/28272#issuecomment-875229618
        # ERROR: Unexpected bus error encountered in worker. This might be caused by insufficient shared memory (shm)
        # PyTorchはプロセス間のデータ共有に共有メモリを使用するため、torchのマルチプロセッシングを使用する場合
        # (マルチスレッドのデータローダなど)、コンテナが実行するデフォルトの共有メモリセグメントサイズでは十分ではない。
        - mountPath: /dev/shm
          name: shm
      volumes:
      - name: nfs
        persistentVolumeClaim:
          claimName: pvc-sv1
      - name: working
        # emptyDir: {}
        emptyDir:
          medium: Memory
      - name: shm
        emptyDir:
          medium: Memory
      # nodeSelector:
      #   nvidia.com/gpu.product: NVIDIA-A100-SXM4-40GB-MIG-3g.20gb
        # nvidia.com/gpu.product: Tesla-T4
