download dataset with Repository() api

app.py

@@ -30,21 +30,39 @@ from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
 
+from huggingface_hub import hf_hub_download, Repository
+
+def download_dataset(repo_id, local_dir):
+    # Clone the repository
+    repo_id_full = f"https://huggingface.co/datasets/{repo_id}"
+    repo = Repository(local_dir=local_dir, clone_from=repo_id_full)
+    # Alternatively, you can download specific files using hf_hub_download
+    # file_path = hf_hub_download(repo_id, filename="your_file_name")
+
+# Usage
+#download_dataset("https://huggingface.co/datasets/open-llm-leaderboard/requests", "new/requests")
+#download_dataset("datasets/open-llm-leaderboard/results", "new/results")
+
+
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=TOKEN)
 
+
 try:
     print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
+    download_dataset(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH)
+    #snapshot_download(
+    #    repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    #)
+    
 except Exception:
     restart_space()
 try:
     print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
+    download_dataset(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH)
+    #snapshot_download(
+    #    repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    #)
 except Exception:
     restart_space()