apiVersion: v1
kind: ConfigMap
metadata:
  name: data-loader-script
data:
  run.sh: |
    #!/bin/sh
    set -x
    DATASET_PATH="/data"
    DATASET_TIMESTAMP="${DATASET_YEAR}${DATASET_MONTH}${DATASET_DAY}"
    if [ -z "${SKIP_DOWNLOADING}" ]; then
      echo "Downloading dataset..."
      mkdir -p ${DATASET_PATH}
      apk add wget
      for label in artists labels masters releases; do
        wget \
          --continue \
          -O ${DATASET_PATH}/discogs_${DATASET_TIMESTAMP}_${label}.xml.gz \
          https://discogs-data.s3-us-west-2.amazonaws.com/data/${DATASET_YEAR}/discogs_${DATASET_TIMESTAMP}_${label}.xml.gz;
      done;
    else
      echo "Skipping downloading dataset..."
    fi
    if [ -z "${LIMIT}" ]; then
      python3 -m maps data load --path ${DATASET_PATH} --workers 16
    else 
      python3 -m maps data load --limit ${LIMIT} --path ${DATASET_PATH} --workers 16
    fi
