apiVersion: batch/v1
kind: Job
metadata:
  name: nemo-redux-detector
spec:
  template:
    spec:
      nodeSelector:
        springfield.uit.no/gpu-type: titan-xp
      containers:
      - name: nemo-redux-detector
        image: thomasjo/nemo-redux:detection-20210310
        imagePullPolicy: Always
        workingDir: /storage/nemo-redux
        command:
        - python
        - src/train_detector.py
        # I/O options.
        - --data-dir=/data/segmentation-resized/partitioned/combined
        - --output-dir=output/detection
        - --checkpoint-interval=5
        # Dataset options.
        # - --no-augmentation
        # - --normalize
        # Optimizer parameters.
        - --optimizer=adamw
        - --learning-rate=1e-5
        - --weight-decay=1e-4
        # - --optimizer=sgd
        # - --learning-rate=5e-3
        # - --weight-decay=5e-5
        # - --momentum=0.9
        # Learning rate scheduler parameters.
        - --lr-milestones=25,40,45
        - --lr-gamma=0.1
        # Stochasticity parameters.
        - --seed=42
        # Other options...
        # - --dropout-rate=0.2
        - --trainable-backbone-layers=3
        - --backbone-epochs=25
        - --max-epochs=50
        - --num-workers=0
        stdin: true
        tty: true
        envFrom:
        - secretRef:
            name: wandb
        volumeMounts:
        - name: storage
          mountPath: /storage
        - name: local-data
          mountPath: /data
        # - name: shm
        #   mountPath: /dev/shm
        resources:
          limits:
            memory: 42Gi
            cpu: 8000m
      volumes:
      - name: storage
        persistentVolumeClaim:
          claimName: storage
      - name: local-data
        hostPath:
          path: /home/tjo083/data
          type: Directory
      # - name: shm
      #   emptyDir:
      #     medium: Memory
      restartPolicy: OnFailure
  backoffLimit: 0
